zhuyijun/nacos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize dump all logic and optimize dump change interface (#11658)

Browse Source 
* optimize config dump all,do not select content in page query,single query config on md5 and ts updated.

* 1.optimize config dump all,do not select content in page query,single query config on md5 and ts updated.
2. remove convertDeleteConfig and convertConfigConfig ，use standard RowMapper instead.

* remove unecessary code

* use context param instead of adding plugin method params

* bug and test fix
This commit is contained in:
nov.lzf 2024-01-19 15:26:28 +08:00 committed by GitHub
parent 9722ad8dd5
commit 849393c4a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 629 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
config/src/main/java/com/alibaba/nacos/config/server/model/ConfigInfoStateWrapper.java
View File

@ -36,6 +36,8 @@ public class ConfigInfoStateWrapper implements Serializable {
private long lastModified;
private String md5;
public long getId() {
return id;
}
@ -86,11 +88,21 @@ public class ConfigInfoStateWrapper implements Serializable {
}
ConfigInfoStateWrapper that = (ConfigInfoStateWrapper) o;
return id == that.id && lastModified == that.lastModified && Objects.equals(dataId, that.dataId)
&& Objects.equals(group, that.group) && Objects.equals(tenant, that.tenant);
&& Objects.equals(group, that.group) && Objects.equals(tenant, that.tenant) && Objects.equals(md5,
that.md5);
}
@Override
public int hashCode() {
return Objects.hash(id, dataId, group, tenant, lastModified);
return Objects.hash(dataId, group, tenant);
}
public String getMd5() {
return md5;
}
public void setMd5(String md5) {
this.md5 = md5;
}
}

4
config/src/main/java/com/alibaba/nacos/config/server/service/ConfigCacheService.java
View File

@ -37,6 +37,7 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import static com.alibaba.nacos.config.server.constant.Constants.ENCODE_UTF8;
import static com.alibaba.nacos.config.server.constant.Constants.PERSIST_ENCODE;
import static com.alibaba.nacos.config.server.utils.LogUtil.DEFAULT_LOG;
import static com.alibaba.nacos.config.server.utils.LogUtil.DUMP_LOG;
import static com.alibaba.nacos.config.server.utils.LogUtil.FATAL_LOG;
@ -72,6 +73,7 @@ public class ConfigCacheService {
* @param group group string value.
* @param tenant tenant string value.
* @param content content string value.
* @param md5 md5 of persist.
* @param lastModifiedTs lastModifiedTs.
* @param type file type.
* @return dumpChange success or not.
@ -100,7 +102,7 @@ public class ConfigCacheService {
boolean newLastModified = lastModifiedTs > ConfigCacheService.getLastModifiedTs(groupKey);
if (md5 == null) {
md5 = MD5Utils.md5Hex(content, ENCODE_UTF8);
md5 = MD5Utils.md5Hex(content, PERSIST_ENCODE);
}
//check md5 & update local disk cache.

12
config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpChangeConfigWorker.java
View File

@ -18,7 +18,7 @@ package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.common.utils.MD5Utils;
import com.alibaba.nacos.config.server.constant.Constants;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.service.ConfigCacheService;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
@ -81,9 +81,9 @@ public class DumpChangeConfigWorker implements Runnable {
long deleteCursorId = 0L;
while (true) {
List<ConfigInfoWrapper> configDeleted = historyConfigInfoPersistService.findDeletedConfig(startTime,
List<ConfigInfoStateWrapper> configDeleted = historyConfigInfoPersistService.findDeletedConfig(startTime,
deleteCursorId, pageSize);
for (ConfigInfo configInfo : configDeleted) {
for (ConfigInfoStateWrapper configInfo : configDeleted) {
if (configInfoPersistService.findConfigInfoState(configInfo.getDataId(), configInfo.getGroup(),
configInfo.getTenant()) == null) {
ConfigCacheService.remove(configInfo.getDataId(), configInfo.getGroup(),
@ -107,9 +107,9 @@ public class DumpChangeConfigWorker implements Runnable {
long changeCursorId = 0L;
while (true) {
LogUtil.DEFAULT_LOG.info("Check changed configs from time {},lastMaxId={}", startTime, changeCursorId);
List<ConfigInfoWrapper> changeConfigs = configInfoPersistService.findChangeConfig(startTime,
List<ConfigInfoStateWrapper> changeConfigs = configInfoPersistService.findChangeConfig(startTime,
changeCursorId, pageSize);
for (ConfigInfoWrapper cf : changeConfigs) {
for (ConfigInfoStateWrapper cf : changeConfigs) {
final String groupKey = GroupKey2.getKey(cf.getDataId(), cf.getGroup(), cf.getTenant());
//check md5 & localtimestamp update local disk cache.
boolean newLastModified = cf.getLastModified() > ConfigCacheService.getLastModifiedTs(groupKey);
@ -151,7 +151,7 @@ public class DumpChangeConfigWorker implements Runnable {
} finally {
ConfigExecutor.scheduleConfigChangeTask(this, PropertyUtil.getDumpChangeWorkerInterval(),
TimeUnit.MILLISECONDS);
LogUtil.DEFAULT_LOG.info("Next dump change will scheduled after {} millseconds",
LogUtil.DEFAULT_LOG.info("Next dump change will scheduled after {} milliseconds",
PropertyUtil.getDumpChangeWorkerInterval());
}

7
config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java
View File

@ -53,7 +53,6 @@ import com.alibaba.nacos.sys.utils.TimerContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.Calendar;
@ -263,7 +262,7 @@ public abstract class DumpService {
Timestamp currentTime = new Timestamp(System.currentTimeMillis());
try {
dumpConfigInfo(dumpAllProcessor);
dumpAllConfigInfoOnStartup(dumpAllProcessor);
// update Beta cache
LogUtil.DEFAULT_LOG.info("start clear all config-info-beta.");
@ -324,12 +323,12 @@ public abstract class DumpService {
}
private void dumpConfigInfo(DumpAllProcessor dumpAllProcessor) throws IOException {
private void dumpAllConfigInfoOnStartup(DumpAllProcessor dumpAllProcessor) {
try {
LogUtil.DEFAULT_LOG.info("start clear all config-info.");
ConfigDiskServiceFactory.getInstance().clearAll();
dumpAllProcessor.process(new DumpAllTask());
dumpAllProcessor.process(new DumpAllTask(true));
} catch (Exception e) {
LogUtil.FATAL_LOG.error("dump config fail" + e.getMessage());
throw e;

106
config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessor.java
View File

@ -19,17 +19,24 @@ package com.alibaba.nacos.config.server.service.dump.processor;
import com.alibaba.nacos.common.task.NacosTask;
import com.alibaba.nacos.common.task.NacosTaskProcessor;
import com.alibaba.nacos.common.utils.MD5Utils;
import com.alibaba.nacos.config.server.constant.Constants;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.service.AggrWhitelist;
import com.alibaba.nacos.config.server.service.ClientIpWhiteList;
import com.alibaba.nacos.config.server.service.ConfigCacheService;
import com.alibaba.nacos.config.server.service.SwitchService;
import com.alibaba.nacos.config.server.service.dump.task.DumpAllTask;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
import com.alibaba.nacos.config.server.utils.GroupKey2;
import com.alibaba.nacos.config.server.utils.LogUtil;
import com.alibaba.nacos.config.server.utils.PropertyUtil;
import com.alibaba.nacos.persistence.model.Page;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static com.alibaba.nacos.config.server.constant.Constants.ENCODE_UTF8;
import static com.alibaba.nacos.config.server.utils.LogUtil.DEFAULT_LOG;
/**
@ -46,16 +53,59 @@ public class DumpAllProcessor implements NacosTaskProcessor {
@Override
public boolean process(NacosTask task) {
if (!(task instanceof DumpAllTask)) {
DEFAULT_LOG.error("[all-dump-error] ,invalid task type,DumpAllProcessor should process DumpAllTask type.");
return false;
}
DumpAllTask dumpAllTask = (DumpAllTask) task;
long currentMaxId = configInfoPersistService.findConfigMaxId();
long lastMaxId = 0;
ThreadPoolExecutor executorService = null;
if (dumpAllTask.isStartUp()) {
executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(),
Runtime.getRuntime().availableProcessors(), 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(PropertyUtil.getAllDumpPageSize() * 2),
r -> new Thread(r, "dump all executor"), new ThreadPoolExecutor.CallerRunsPolicy());
} else {
executorService = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(),
r -> new Thread(r, "dump all executor"), new ThreadPoolExecutor.CallerRunsPolicy());
}
DEFAULT_LOG.info("start dump all config-info...");
while (lastMaxId < currentMaxId) {
Page<ConfigInfoWrapper> page = configInfoPersistService.findAllConfigInfoFragment(lastMaxId, PAGE_SIZE);
long start = System.currentTimeMillis();
Page<ConfigInfoWrapper> page = configInfoPersistService.findAllConfigInfoFragment(lastMaxId,
PropertyUtil.getAllDumpPageSize(), dumpAllTask.isStartUp());
long dbTimeStamp = System.currentTimeMillis();
if (page == null || page.getPageItems() == null || page.getPageItems().isEmpty()) {
break;
}
for (ConfigInfoWrapper cf : page.getPageItems()) {
long id = cf.getId();
lastMaxId = Math.max(id, lastMaxId);
lastMaxId = Math.max(cf.getId(), lastMaxId);
//if not start up, page query will not return content, check md5 and lastModified first ,if changed ,get single content info to dump.
if (!dumpAllTask.isStartUp()) {
final String groupKey = GroupKey2.getKey(cf.getDataId(), cf.getGroup(), cf.getTenant());
boolean newLastModified = cf.getLastModified() > ConfigCacheService.getLastModifiedTs(groupKey);
//check md5 & update local disk cache.
String localContentMd5 = ConfigCacheService.getContentMd5(groupKey);
boolean md5Update = !localContentMd5.equals(cf.getMd5());
if (newLastModified || md5Update) {
LogUtil.DUMP_LOG.info("[dump-all] find change config {}, {}, md5={}", groupKey,
cf.getLastModified(), cf.getMd5());
cf = configInfoPersistService.findConfigInfo(cf.getDataId(), cf.getGroup(), cf.getTenant());
} else {
continue;
}
}
if (cf == null) {
continue;
}
if (cf.getDataId().equals(AggrWhitelist.AGGRIDS_METADATA)) {
AggrWhitelist.load(cf.getContent());
}
@ -67,22 +117,50 @@ public class DumpAllProcessor implements NacosTaskProcessor {
if (cf.getDataId().equals(SwitchService.SWITCH_META_DATA_ID)) {
SwitchService.load(cf.getContent());
}
ConfigCacheService.dump(cf.getDataId(), cf.getGroup(), cf.getTenant(), cf.getContent(),
cf.getLastModified(), cf.getType(), cf.getEncryptedDataKey());
final String content = cf.getContent();
final String md5 = MD5Utils.md5Hex(content, Constants.ENCODE);
LogUtil.DUMP_LOG.info("[dump-all-ok] {}, {}, length={}, md5={}",
GroupKey2.getKey(cf.getDataId(), cf.getGroup()), cf.getLastModified(), content.length(),
md5);
final String dataId = cf.getDataId();
final String group = cf.getGroup();
final String tenant = cf.getTenant();
final long lastModified = cf.getLastModified();
final String type = cf.getType();
final String encryptedDataKey = cf.getEncryptedDataKey();
executorService.execute(() -> {
final String md5Utf8 = MD5Utils.md5Hex(content, ENCODE_UTF8);
boolean result = ConfigCacheService.dumpWithMd5(dataId, group, tenant, content, md5Utf8,
lastModified, type, encryptedDataKey);
if (result) {
LogUtil.DUMP_LOG.info("[dump-all-ok] {}, {}, length={},md5UTF8={}",
GroupKey2.getKey(dataId, group), lastModified, content.length(), md5Utf8);
} else {
LogUtil.DUMP_LOG.info("[dump-all-error] {}", GroupKey2.getKey(dataId, group));
}
});
}
DEFAULT_LOG.info("[all-dump] {} / {}", lastMaxId, currentMaxId);
long diskStamp = System.currentTimeMillis();
DEFAULT_LOG.info("[all-dump] submit all task for {} / {}, dbTime={},diskTime={}", lastMaxId, currentMaxId,
(dbTimeStamp - start), (diskStamp - dbTimeStamp));
}
//wait all task are finished and then shutdown executor.
try {
int unfinishedTaskCount = 0;
while ((unfinishedTaskCount = executorService.getQueue().size() + executorService.getActiveCount()) > 0) {
DEFAULT_LOG.info("[all-dump] wait {} dump tasks to be finished", unfinishedTaskCount);
Thread.sleep(1000L);
}
executorService.shutdown();
} catch (Exception e) {
DEFAULT_LOG.error("[all-dump] wait dump tasks to be finished error", e);
}
DEFAULT_LOG.info("success to dump all config-info。");
return true;
}
static final int PAGE_SIZE = 1000;
final ConfigInfoPersistService configInfoPersistService;
}

13
config/src/main/java/com/alibaba/nacos/config/server/service/dump/task/DumpAllTask.java
View File

@ -26,6 +26,19 @@ import com.alibaba.nacos.common.task.AbstractDelayTask;
*/
public class DumpAllTask extends AbstractDelayTask {
private boolean startUp;
public DumpAllTask() {
}
public DumpAllTask(boolean startUp) {
this.startUp = startUp;
}
public boolean isStartUp() {
return startUp;
}
@Override
public void merge(AbstractDelayTask task) {
}

13
config/src/main/java/com/alibaba/nacos/config/server/service/repository/ConfigInfoPersistService.java
View File

@ -245,7 +245,7 @@ public interface ConfigInfoPersistService {
* @return config max id
*/
long findConfigMaxId();
/**
* Query configuration information by primary key ID.
*
@ -315,11 +315,12 @@ public interface ConfigInfoPersistService {
/**
* Query all config info.
*
* @param lastMaxId last max id
* @param pageSize page size
* @param lastMaxId last max id
* @param pageSize page size
* @param needContent need content or not.
* @return {@link Page} with {@link ConfigInfoWrapper} generation
*/
Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize);
Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize, boolean needContent);
/**
* Query config info.
@ -334,7 +335,7 @@ public interface ConfigInfoPersistService {
*/
Page<ConfigInfo> findConfigInfoLike4Page(final int pageNo, final int pageSize, final String dataId,
final String group, final String tenant, final Map<String, Object> configAdvanceInfo);
/**
* Query change config.order by id asc.
*
@ -343,7 +344,7 @@ public interface ConfigInfoPersistService {
* @param pageSize pageSize
* @return {@link ConfigInfoWrapper} list
*/
List<ConfigInfoWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId, final int pageSize);
List<ConfigInfoStateWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId, final int pageSize);
/**
* Query tag list.

6
config/src/main/java/com/alibaba/nacos/config/server/service/repository/ConfigRowMapperInjector.java
View File

@ -222,7 +222,11 @@ public class ConfigRowMapperInjector {
info.setGroup(rs.getString("group_id"));
info.setTenant(rs.getString("tenant_id"));
info.setLastModified(rs.getTimestamp("gmt_modified").getTime());
try {
info.setMd5(rs.getString("md5"));
} catch (SQLException e) {
// ignore
}
try {
info.setId(rs.getLong("id"));
} catch (SQLException e) {

15
config/src/main/java/com/alibaba/nacos/config/server/service/repository/HistoryConfigInfoPersistService.java
View File

@ -18,13 +18,12 @@ package com.alibaba.nacos.config.server.service.repository;
import com.alibaba.nacos.config.server.model.ConfigHistoryInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.persistence.repository.PaginationHelper;
import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
/**
* Database service, providing access to his_config_info in the database.
@ -41,14 +40,6 @@ public interface HistoryConfigInfoPersistService {
*/
<E> PaginationHelper<E> createPaginationHelper();
/**
* Convert delete config.
*
* @param list origin data
* @return {@link ConfigInfo} list
*/
List<ConfigInfoWrapper> convertDeletedConfig(List<Map<String, Object>> list);
//------------------------------------------insert---------------------------------------------//
/**
@ -81,9 +72,9 @@ public interface HistoryConfigInfoPersistService {
* @param startTime start time
* @param startId last max id
* @param size page size
* @return {@link ConfigInfo} list
* @return {@link ConfigInfoStateWrapper} list
*/
List<ConfigInfoWrapper> findDeletedConfig(final Timestamp startTime, final long startId, int size);
List<ConfigInfoStateWrapper> findDeletedConfig(final Timestamp startTime, final long startId, int size);
/**
* List configuration history change record.

10
config/src/main/java/com/alibaba/nacos/config/server/service/repository/embedded/EmbeddedConfigInfoPersistServiceImpl.java
View File

@ -48,6 +48,7 @@ import com.alibaba.nacos.persistence.repository.embedded.EmbeddedStorageContextH
import com.alibaba.nacos.persistence.repository.embedded.operate.DatabaseOperate;
import com.alibaba.nacos.plugin.datasource.MapperManager;
import com.alibaba.nacos.plugin.datasource.constants.CommonConstant;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.constants.TableConstant;
import com.alibaba.nacos.plugin.datasource.mapper.ConfigInfoMapper;
@ -788,10 +789,12 @@ public class EmbeddedConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
}
@Override
public Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize) {
public Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize,
boolean needContent) {
ConfigInfoMapper configInfoMapper = mapperManager.findMapper(dataSourceService.getDataSourceType(),
TableConstant.CONFIG_INFO);
MapperContext context = new MapperContext(0, pageSize);
context.putContextParameter(ContextConstant.NEED_CONTENT, String.valueOf(needContent));
context.putWhereParameter(FieldConstant.ID, lastMaxId);
MapperResult select = configInfoMapper.findAllConfigInfoFragment(context);
PaginationHelper<ConfigInfoWrapper> helper = createPaginationHelper();
@ -852,7 +855,8 @@ public class EmbeddedConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
}
@Override
public List<ConfigInfoWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId, final int pageSize) {
public List<ConfigInfoStateWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId,
final int pageSize) {
ConfigInfoMapper configInfoMapper = mapperManager.findMapper(dataSourceService.getDataSourceType(),
TableConstant.CONFIG_INFO);
@ -863,7 +867,7 @@ public class EmbeddedConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
MapperResult mapperResult = configInfoMapper.findChangeConfig(context);
return databaseOperate.queryMany(mapperResult.getSql(), mapperResult.getParamList().toArray(),
CONFIG_INFO_WRAPPER_ROW_MAPPER);
CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER);
}

46
config/src/main/java/com/alibaba/nacos/config/server/service/repository/embedded/EmbeddedHistoryConfigInfoPersistServiceImpl.java
View File

@ -19,20 +19,20 @@ package com.alibaba.nacos.config.server.service.repository.embedded;
import com.alibaba.nacos.common.notify.NotifyCenter;
import com.alibaba.nacos.common.utils.MD5Utils;
import com.alibaba.nacos.common.utils.StringUtils;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.persistence.configuration.condition.ConditionOnEmbeddedStorage;
import com.alibaba.nacos.config.server.constant.Constants;
import com.alibaba.nacos.config.server.model.ConfigHistoryInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.persistence.model.event.DerbyImportEvent;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.persistence.configuration.condition.ConditionOnEmbeddedStorage;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.persistence.model.event.DerbyImportEvent;
import com.alibaba.nacos.persistence.repository.PaginationHelper;
import com.alibaba.nacos.persistence.repository.embedded.EmbeddedPaginationHelperImpl;
import com.alibaba.nacos.persistence.repository.embedded.operate.DatabaseOperate;
import com.alibaba.nacos.persistence.repository.embedded.EmbeddedStorageContextHolder;
import com.alibaba.nacos.persistence.repository.embedded.operate.DatabaseOperate;
import com.alibaba.nacos.plugin.datasource.MapperManager;
import com.alibaba.nacos.plugin.datasource.constants.CommonConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
@ -45,14 +45,11 @@ import org.springframework.context.annotation.Conditional;
import org.springframework.stereotype.Service;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_DETAIL_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_LIST_ROW_MAPPER;
@ -91,24 +88,6 @@ public class EmbeddedHistoryConfigInfoPersistServiceImpl implements HistoryConfi
return new EmbeddedPaginationHelperImpl<>(databaseOperate);
}
@Override
public List<ConfigInfoWrapper> convertDeletedConfig(List<Map<String, Object>> list) {
List<ConfigInfoWrapper> configs = new ArrayList<>();
for (Map<String, Object> map : list) {
String dataId = (String) map.get("data_id");
String group = (String) map.get("group_id");
String tenant = (String) map.get("tenant_id");
long mTime = ((LocalDateTime) map.get("gmt_modified")).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
ConfigInfoWrapper config = new ConfigInfoWrapper();
config.setDataId(dataId);
config.setGroup(group);
config.setTenant(tenant);
config.setLastModified(mTime);
configs.add(config);
}
return configs;
}
@Override
public void insertConfigHistoryAtomic(long configHistoryId, ConfigInfo configInfo, String srcIp, String srcUser,
final Timestamp time, String ops) {
@ -116,7 +95,7 @@ public class EmbeddedHistoryConfigInfoPersistServiceImpl implements HistoryConfi
String tenantTmp = StringUtils.defaultEmptyIfBlank(configInfo.getTenant());
final String md5Tmp = MD5Utils.md5Hex(configInfo.getContent(), Constants.ENCODE);
String encryptedDataKey = StringUtils.defaultEmptyIfBlank(configInfo.getEncryptedDataKey());
HistoryConfigInfoMapper historyConfigInfoMapper = mapperManager.findMapper(
dataSourceService.getDataSourceType(), TableConstant.HIS_CONFIG_INFO);
final String sql = historyConfigInfoMapper.insert(
@ -141,7 +120,8 @@ public class EmbeddedHistoryConfigInfoPersistServiceImpl implements HistoryConfi
}
@Override
public List<ConfigInfoWrapper> findDeletedConfig(final Timestamp startTime, long lastMaxId, final int pageSize) {
public List<ConfigInfoStateWrapper> findDeletedConfig(final Timestamp startTime, long lastMaxId,
final int pageSize) {
HistoryConfigInfoMapper historyConfigInfoMapper = mapperManager.findMapper(
dataSourceService.getDataSourceType(), TableConstant.HIS_CONFIG_INFO);
MapperContext context = new MapperContext();
@ -150,10 +130,8 @@ public class EmbeddedHistoryConfigInfoPersistServiceImpl implements HistoryConfi
context.putWhereParameter(FieldConstant.LAST_MAX_ID, lastMaxId);
MapperResult mapperResult = historyConfigInfoMapper.findDeletedConfig(context);
List<Map<String, Object>> list = databaseOperate.queryMany(mapperResult.getSql(),
mapperResult.getParamList().toArray());
return convertDeletedConfig(list);
return databaseOperate.queryMany(mapperResult.getSql(), mapperResult.getParamList().toArray(),
CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER);
}
@Override

10
config/src/main/java/com/alibaba/nacos/config/server/service/repository/extrnal/ExternalConfigInfoPersistServiceImpl.java
View File

@ -42,6 +42,7 @@ import com.alibaba.nacos.persistence.repository.PaginationHelper;
import com.alibaba.nacos.persistence.repository.extrnal.ExternalStoragePaginationHelperImpl;
import com.alibaba.nacos.plugin.datasource.MapperManager;
import com.alibaba.nacos.plugin.datasource.constants.CommonConstant;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.constants.TableConstant;
import com.alibaba.nacos.plugin.datasource.mapper.ConfigInfoMapper;
@ -822,10 +823,12 @@ public class ExternalConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
}
@Override
public Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize) {
public Page<ConfigInfoWrapper> findAllConfigInfoFragment(final long lastMaxId, final int pageSize,
boolean needContent) {
ConfigInfoMapper configInfoMapper = mapperManager.findMapper(dataSourceService.getDataSourceType(),
TableConstant.CONFIG_INFO);
MapperContext context = new MapperContext(0, pageSize);
context.putContextParameter(ContextConstant.NEED_CONTENT, String.valueOf(needContent));
context.putWhereParameter(FieldConstant.ID, lastMaxId);
MapperResult select = configInfoMapper.findAllConfigInfoFragment(context);
PaginationHelper<ConfigInfoWrapper> helper = createPaginationHelper();
@ -896,7 +899,8 @@ public class ExternalConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
}
@Override
public List<ConfigInfoWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId, final int pageSize) {
public List<ConfigInfoStateWrapper> findChangeConfig(final Timestamp startTime, long lastMaxId,
final int pageSize) {
try {
ConfigInfoMapper configInfoMapper = mapperManager.findMapper(dataSourceService.getDataSourceType(),
TableConstant.CONFIG_INFO);
@ -908,7 +912,7 @@ public class ExternalConfigInfoPersistServiceImpl implements ConfigInfoPersistSe
MapperResult mapperResult = configInfoMapper.findChangeConfig(context);
return jt.query(mapperResult.getSql(), mapperResult.getParamList().toArray(),
CONFIG_INFO_WRAPPER_ROW_MAPPER);
CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER);
} catch (DataAccessException e) {
LogUtil.FATAL_LOG.error("[db-error] " + e, e);
throw e;

43
config/src/main/java/com/alibaba/nacos/config/server/service/repository/extrnal/ExternalHistoryConfigInfoPersistServiceImpl.java
View File

@ -18,17 +18,17 @@ package com.alibaba.nacos.config.server.service.repository.extrnal;
import com.alibaba.nacos.common.utils.MD5Utils;
import com.alibaba.nacos.common.utils.StringUtils;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.persistence.configuration.condition.ConditionOnExternalStorage;
import com.alibaba.nacos.config.server.constant.Constants;
import com.alibaba.nacos.config.server.model.ConfigHistoryInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.config.server.utils.LogUtil;
import com.alibaba.nacos.persistence.configuration.condition.ConditionOnExternalStorage;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.persistence.model.Page;
import com.alibaba.nacos.persistence.repository.PaginationHelper;
import com.alibaba.nacos.config.server.utils.LogUtil;
import com.alibaba.nacos.persistence.repository.extrnal.ExternalStoragePaginationHelperImpl;
import com.alibaba.nacos.plugin.datasource.MapperManager;
import com.alibaba.nacos.plugin.datasource.constants.CommonConstant;
@ -45,16 +45,12 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import java.math.BigInteger;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_DETAIL_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_LIST_ROW_MAPPER;
@ -90,26 +86,6 @@ public class ExternalHistoryConfigInfoPersistServiceImpl implements HistoryConfi
return new ExternalStoragePaginationHelperImpl<>(jt);
}
@Override
public List<ConfigInfoWrapper> convertDeletedConfig(List<Map<String, Object>> list) {
List<ConfigInfoWrapper> configs = new ArrayList<>();
for (Map<String, Object> map : list) {
BigInteger id = (BigInteger) map.get("nid");
String dataId = (String) map.get("data_id");
String group = (String) map.get("group_id");
String tenant = (String) map.get("tenant_id");
long mTime = ((LocalDateTime) map.get("gmt_modified")).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
ConfigInfoWrapper config = new ConfigInfoWrapper();
config.setId(id.longValue());
config.setDataId(dataId);
config.setGroup(group);
config.setTenant(tenant);
config.setLastModified(mTime);
configs.add(config);
}
return configs;
}
@Override
public void insertConfigHistoryAtomic(long id, ConfigInfo configInfo, String srcIp, String srcUser,
final Timestamp time, String ops) {
@ -145,7 +121,7 @@ public class ExternalHistoryConfigInfoPersistServiceImpl implements HistoryConfi
}
@Override
public List<ConfigInfoWrapper> findDeletedConfig(final Timestamp startTime, long startId, int pageSize) {
public List<ConfigInfoStateWrapper> findDeletedConfig(final Timestamp startTime, long startId, int pageSize) {
try {
HistoryConfigInfoMapper historyConfigInfoMapper = mapperManager.findMapper(
dataSourceService.getDataSourceType(), TableConstant.HIS_CONFIG_INFO);
@ -155,9 +131,8 @@ public class ExternalHistoryConfigInfoPersistServiceImpl implements HistoryConfi
context.putWhereParameter(FieldConstant.LAST_MAX_ID, startId);
MapperResult mapperResult = historyConfigInfoMapper.findDeletedConfig(context);
List<Map<String, Object>> list = jt.queryForList(mapperResult.getSql(),
mapperResult.getParamList().toArray());
return convertDeletedConfig(list);
return jt.query(mapperResult.getSql(), mapperResult.getParamList().toArray(),
CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER);
} catch (DataAccessException e) {
LogUtil.FATAL_LOG.error("[db-error] " + e, e);
throw e;

64
config/src/main/java/com/alibaba/nacos/config/server/utils/PropertyUtil.java
View File

@ -22,6 +22,13 @@ import org.slf4j.Logger;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Properties util.
*
@ -310,4 +317,61 @@ public class PropertyUtil implements ApplicationContextInitializer<ConfigurableA
public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
loadSetting();
}
private static final int MAX_DUMP_PAGE = 1000;
private static final int MIN_DUMP_PAGE = 50;
private static final int PAGE_MEMORY_DIVIDE_MB = 512;
private static AtomicInteger allDumpPageSize;
public static int getAllDumpPageSize() {
if (allDumpPageSize == null) {
allDumpPageSize = new AtomicInteger(initAllDumpPageSize());
}
return allDumpPageSize.get();
}
static int initAllDumpPageSize() {
long memLimitMB = getMemLimitMB();
//512MB->50 Page Size
int pageSize = (int) ((float) memLimitMB / PAGE_MEMORY_DIVIDE_MB) * MIN_DUMP_PAGE;
pageSize = Math.max(pageSize, MIN_DUMP_PAGE);
pageSize = Math.min(pageSize, MAX_DUMP_PAGE);
LOGGER.info("All dump page size is set to {} according to mem limit {} MB", pageSize, memLimitMB);
return pageSize;
}
public static long getMemLimitMB() {
Optional<Long> memoryLimit = findMemoryLimitFromFile();
if (memoryLimit.isPresent()) {
return memoryLimit.get();
}
memoryLimit = findMemoryLimitFromSystem();
return memoryLimit.get();
}
private static String limitMemoryFile;
private static Optional<Long> findMemoryLimitFromFile() {
if (limitMemoryFile == null) {
limitMemoryFile = EnvUtil.getProperty("memory_limit_file_path",
"/sys/fs/cgroup/memory/memory.limit_in_bytes");
}
File file = new File(limitMemoryFile);
try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
long memoryLimit = Long.parseLong(reader.readLine().trim());
return Optional.of(memoryLimit / 1024L / 1024L);
} catch (IOException | NumberFormatException ignored) {
return Optional.empty();
}
}
private static Optional<Long> findMemoryLimitFromSystem() {
long maxHeapSizeMb = Runtime.getRuntime().maxMemory() / 1024L / 1024L;
return Optional.of(maxHeapSizeMb);
}
}

39
config/src/test/java/com/alibaba/nacos/config/server/service/dump/DumpChangeConfigWorkerTest.java
View File

@ -17,6 +17,7 @@
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.common.utils.MD5Utils;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.service.ConfigCacheService;
import com.alibaba.nacos.config.server.service.dump.disk.ConfigDiskService;
@ -129,13 +130,13 @@ public class DumpChangeConfigWorkerTest {
PropertyUtil.setDumpChangeOn(true);
dumpChangeConfigWorker.setPageSize(3);
//mock delete first page
List<ConfigInfoWrapper> firstPageDeleted = new ArrayList<>();
List<ConfigInfoStateWrapper> firstPageDeleted = new ArrayList<>();
Timestamp startTime = dumpChangeConfigWorker.startTime;
String dataIdPrefix = "d12345";
firstPageDeleted.add(createConfigInfoWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
firstPageDeleted.add(createConfigInfoWrapper(dataIdPrefix, 2, startTime.getTime() + 2));
firstPageDeleted.add(createConfigInfoWrapper(dataIdPrefix, 3, startTime.getTime() + 3));
firstPageDeleted.add(createConfigInfoStateWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
firstPageDeleted.add(createConfigInfoStateWrapper(dataIdPrefix, 2, startTime.getTime() + 2));
firstPageDeleted.add(createConfigInfoStateWrapper(dataIdPrefix, 3, startTime.getTime() + 3));
//pre set cache for id1
preSetCache(dataIdPrefix, 1, System.currentTimeMillis());
Assert.assertEquals("encrykey" + 1,
@ -173,8 +174,8 @@ public class DumpChangeConfigWorkerTest {
Assert.assertEquals(startTime.getTime() - 1,
ConfigCacheService.getContentCache(GroupKey.getKeyTenant(dataIdPrefix + 1, "group" + 1, "tenant" + 1))
.getConfigCache().getLastModifiedTs());
List<ConfigInfoWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
List<ConfigInfoStateWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoStateWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
Mockito.when(configInfoPersistService.findChangeConfig(eq(startTime), eq(0L), eq(3))).thenReturn(firstChanged);
@ -211,8 +212,8 @@ public class DumpChangeConfigWorkerTest {
Assert.assertEquals(startTime.getTime() - 1,
ConfigCacheService.getContentCache(GroupKey.getKeyTenant(dataIdPrefix + 1, "group" + 1, "tenant" + 1))
.getConfigCache().getLastModifiedTs());
List<ConfigInfoWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
List<ConfigInfoStateWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoStateWrapper(dataIdPrefix, 1, startTime.getTime() + 1));
Mockito.when(configInfoPersistService.findChangeConfig(eq(startTime), eq(0L), eq(3))).thenReturn(firstChanged);
@ -250,8 +251,8 @@ public class DumpChangeConfigWorkerTest {
Assert.assertEquals(startTime.getTime() - 1,
ConfigCacheService.getContentCache(GroupKey.getKeyTenant(dataIdPrefix + 1, "group" + 1, "tenant" + 1))
.getConfigCache().getLastModifiedTs());
List<ConfigInfoWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoWrapper(dataIdPrefix, 1, startTime.getTime() - 2));
List<ConfigInfoStateWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoStateWrapper(dataIdPrefix, 1, startTime.getTime() - 2));
Mockito.when(configInfoPersistService.findChangeConfig(eq(startTime), eq(0L), eq(3))).thenReturn(firstChanged);
@ -290,8 +291,8 @@ public class DumpChangeConfigWorkerTest {
Assert.assertEquals(startTime.getTime() - 1,
ConfigCacheService.getContentCache(GroupKey.getKeyTenant(dataIdPrefix + 1, "group" + 1, "tenant" + 1))
.getConfigCache().getLastModifiedTs());
List<ConfigInfoWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoWrapper(dataIdPrefix, 1, startTime.getTime() - 1));
List<ConfigInfoStateWrapper> firstChanged = new ArrayList<>();
firstChanged.add(createConfigInfoStateWrapper(dataIdPrefix, 1, startTime.getTime() - 1));
Mockito.when(configInfoPersistService.findChangeConfig(eq(startTime), eq(0L), eq(3))).thenReturn(firstChanged);
@ -320,12 +321,24 @@ public class DumpChangeConfigWorkerTest {
MD5Utils.md5Hex("content" + id, "UTF-8"), timeStamp, "json", "encrykey" + id);
}
private ConfigInfoStateWrapper createConfigInfoStateWrapper(String dataIdPreFix, long id, long timeStamp) {
ConfigInfoStateWrapper configInfoWrapper = new ConfigInfoStateWrapper();
configInfoWrapper.setDataId(dataIdPreFix + id);
configInfoWrapper.setGroup("group" + id);
configInfoWrapper.setTenant("md5" + id);
configInfoWrapper.setTenant("tenant" + id);
configInfoWrapper.setId(id);
configInfoWrapper.setLastModified(timeStamp);
return configInfoWrapper;
}
private ConfigInfoWrapper createConfigInfoWrapper(String dataIdPreFix, long id, long timeStamp) {
ConfigInfoWrapper configInfoWrapper = new ConfigInfoWrapper();
configInfoWrapper.setDataId(dataIdPreFix + id);
configInfoWrapper.setGroup("group" + id);
configInfoWrapper.setTenant("tenant" + id);
configInfoWrapper.setMd5("md5" + id);
configInfoWrapper.setContent("content" + id);
configInfoWrapper.setTenant("tenant" + id);
configInfoWrapper.setId(id);
configInfoWrapper.setLastModified(timeStamp);
return configInfoWrapper;

12
config/src/test/java/com/alibaba/nacos/config/server/service/dump/DumpServiceTest.java
View File

@ -28,6 +28,7 @@ import com.alibaba.nacos.config.server.service.repository.ConfigInfoTagPersistSe
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.config.server.utils.ConfigExecutor;
import com.alibaba.nacos.config.server.utils.GroupKey;
import com.alibaba.nacos.config.server.utils.PropertyUtil;
import com.alibaba.nacos.core.cluster.ServerMemberManager;
import com.alibaba.nacos.core.namespace.repository.NamespacePersistService;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
@ -91,6 +92,8 @@ public class DumpServiceTest {
MockedStatic<ConfigExecutor> configExecutorMocked;
MockedStatic<PropertyUtil> propertyUtilMockedStatic;
private DumpService dumpService;
@Mock
@ -103,6 +106,10 @@ public class DumpServiceTest {
@Before
public void setUp() {
envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
propertyUtilMockedStatic = Mockito.mockStatic(PropertyUtil.class);
propertyUtilMockedStatic.when(() -> PropertyUtil.getAllDumpPageSize()).thenReturn(100);
propertyUtilMockedStatic.when(() -> PropertyUtil.getDumpChangeWorkerInterval()).thenReturn(1000 * 60L);
ReflectionTestUtils.setField(DynamicDataSource.getInstance(), "localDataSourceService", dataSourceService);
ReflectionTestUtils.setField(DynamicDataSource.getInstance(), "basicDataSourceService", dataSourceService);
dumpService = new ExternalDumpService(configInfoPersistService, namespacePersistService,
@ -116,6 +123,7 @@ public class DumpServiceTest {
public void after() {
envUtilMockedStatic.close();
configExecutorMocked.close();
propertyUtilMockedStatic.close();
}
@Test
@ -177,9 +185,11 @@ public class DumpServiceTest {
result.add(Arrays.asList(noDatum));
Mockito.when(mergeDatumService.splitList(anyList(), anyInt())).thenReturn(result);
Mockito.doNothing().when(mergeDatumService).executeConfigsMerge(anyList());
Mockito.when(configInfoPersistService.findConfigMaxId()).thenReturn(300L);
dumpService.dumpOperate();
// expect dump formal,beta,tag to be invoked.
// expect dump
Mockito.verify(configInfoPersistService, times(1)).findAllConfigInfoFragment(0, 100, true);
Mockito.verify(configInfoPersistService, times(1)).findConfigMaxId();
Mockito.verify(configInfoBetaPersistService, times(1)).configInfoBetaCount();
Mockito.verify(configInfoTagPersistService, times(1)).configInfoTagCount();

103
config/src/test/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllProcessorTest.java
View File

@ -27,6 +27,7 @@ import com.alibaba.nacos.config.server.service.repository.ConfigInfoBetaPersistS
import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
import com.alibaba.nacos.config.server.service.repository.ConfigInfoTagPersistService;
import com.alibaba.nacos.config.server.utils.GroupKey2;
import com.alibaba.nacos.config.server.utils.PropertyUtil;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
import com.alibaba.nacos.persistence.model.Page;
@ -42,8 +43,8 @@ import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.beans.BeanUtils;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@ -76,6 +77,8 @@ public class DumpAllProcessorTest extends TestCase {
MockedStatic<EnvUtil> envUtilMockedStatic;
private String mockMem = "tmpmocklimitfile.txt";
@Before
public void init() throws Exception {
dynamicDataSourceMockedStatic = Mockito.mockStatic(DynamicDataSource.class);
@ -92,6 +95,8 @@ public class DumpAllProcessorTest extends TestCase {
configInfoTagPersistService, null, null);
dumpAllProcessor = new DumpAllProcessor(configInfoPersistService);
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("memory_limit_file_path"),
eq("/sys/fs/cgroup/memory/memory.limit_in_bytes"))).thenReturn(mockMem);
}
@After
@ -118,7 +123,7 @@ public class DumpAllProcessorTest extends TestCase {
}
@Test
public void testDumpAll() throws IOException {
public void testDumpAllOnStartUp() throws Exception {
ConfigInfoWrapper configInfoWrapper1 = createNewConfig(1);
ConfigInfoWrapper configInfoWrapper2 = createNewConfig(2);
long timestamp = System.currentTimeMillis();
@ -132,12 +137,9 @@ public class DumpAllProcessorTest extends TestCase {
page.setPageItems(list);
Mockito.when(configInfoPersistService.findConfigMaxId()).thenReturn(2L);
Mockito.when(configInfoPersistService.findAllConfigInfoFragment(0, 1000)).thenReturn(page);
Mockito.when(configInfoPersistService.findAllConfigInfoFragment(0, PropertyUtil.getAllDumpPageSize(), true))
.thenReturn(page);
String groupKey1 = GroupKey2.getKey(configInfoWrapper1.getDataId(), configInfoWrapper1.getGroup(),
configInfoWrapper1.getTenant());
String groupKey2 = GroupKey2.getKey(configInfoWrapper2.getDataId(), configInfoWrapper2.getGroup(),
configInfoWrapper2.getTenant());
// For config 1, assign a latter time, to make sure that it would be updated.
// For config 2, assign an earlier time, to make sure that it is not be updated.
String md51 = MD5Utils.md5Hex(configInfoWrapper1.getContent(), "UTF-8");
@ -152,7 +154,8 @@ public class DumpAllProcessorTest extends TestCase {
configInfoWrapper2.getTenant(), configInfoWrapper2.getContent(), md52, earlierTimestamp, "json",
encryptedDataKey);
DumpAllTask dumpAllTask = new DumpAllTask();
DumpAllTask dumpAllTask = new DumpAllTask(true);
boolean process = dumpAllProcessor.process(dumpAllTask);
Assert.assertTrue(process);
@ -183,4 +186,88 @@ public class DumpAllProcessorTest extends TestCase {
configInfoWrapper2.getTenant());
Assert.assertEquals(configInfoWrapper2.getContent(), contentFromDisk2);
}
/**
* test dump all for all check task.
*
*/
@Test
public void testDumpAllOnCheckAll() throws Exception {
ConfigInfoWrapper configInfoWrapper1 = createNewConfig(1);
ConfigInfoWrapper configInfoWrapper2 = createNewConfig(2);
long timestamp = System.currentTimeMillis();
configInfoWrapper1.setLastModified(timestamp);
configInfoWrapper2.setLastModified(timestamp);
Page<ConfigInfoWrapper> page = new Page<>();
page.setTotalCount(2);
page.setPagesAvailable(2);
page.setPageNumber(1);
List<ConfigInfoWrapper> list = Arrays.asList(configInfoWrapper1, configInfoWrapper2);
page.setPageItems(list);
Mockito.when(configInfoPersistService.findConfigMaxId()).thenReturn(2L);
Mockito.when(configInfoPersistService.findAllConfigInfoFragment(0, PropertyUtil.getAllDumpPageSize(), false))
.thenReturn(page);
ConfigInfoWrapper configInfoWrapperSingle1 = new ConfigInfoWrapper();
BeanUtils.copyProperties(configInfoWrapper1, configInfoWrapperSingle1);
configInfoWrapperSingle1.setContent("content123456");
Mockito.when(
configInfoPersistService.findConfigInfo(configInfoWrapper1.getDataId(), configInfoWrapper1.getGroup(),
configInfoWrapper1.getTenant())).thenReturn(configInfoWrapperSingle1);
ConfigInfoWrapper configInfoWrapperSingle2 = new ConfigInfoWrapper();
BeanUtils.copyProperties(configInfoWrapper2, configInfoWrapperSingle2);
configInfoWrapperSingle2.setContent("content123456222");
Mockito.when(
configInfoPersistService.findConfigInfo(configInfoWrapper2.getDataId(), configInfoWrapper2.getGroup(),
configInfoWrapper2.getTenant())).thenReturn(configInfoWrapperSingle2);
// For config 1, assign a latter time, to make sure that it would not be updated.
// For config 2, assign an earlier time, to make sure that it would be updated.
String md51 = MD5Utils.md5Hex(configInfoWrapper1.getContent(), "UTF-8");
String md52 = MD5Utils.md5Hex(configInfoWrapper2.getContent(), "UTF-8");
long latterTimestamp = timestamp + 999;
long earlierTimestamp = timestamp - 999;
String encryptedDataKey = "testEncryptedDataKey";
ConfigCacheService.dumpWithMd5(configInfoWrapper1.getDataId(), configInfoWrapper1.getGroup(),
configInfoWrapper1.getTenant(), configInfoWrapper1.getContent(), md51, latterTimestamp, "json",
encryptedDataKey);
ConfigCacheService.dumpWithMd5(configInfoWrapper2.getDataId(), configInfoWrapper2.getGroup(),
configInfoWrapper2.getTenant(), configInfoWrapper2.getContent(), md52, earlierTimestamp, "json",
encryptedDataKey);
DumpAllTask dumpAllTask = new DumpAllTask(false);
boolean process = dumpAllProcessor.process(dumpAllTask);
Assert.assertTrue(process);
//Check cache
CacheItem contentCache1 = ConfigCacheService.getContentCache(
GroupKey2.getKey(configInfoWrapper1.getDataId(), configInfoWrapper1.getGroup(),
configInfoWrapper1.getTenant()));
// check if config1 is not updated
Assert.assertEquals(md51, contentCache1.getConfigCache().getMd5Utf8());
Assert.assertEquals(latterTimestamp, contentCache1.getConfigCache().getLastModifiedTs());
//check disk
String contentFromDisk1 = ConfigDiskServiceFactory.getInstance()
.getContent(configInfoWrapper1.getDataId(), configInfoWrapper1.getGroup(),
configInfoWrapper1.getTenant());
Assert.assertEquals(configInfoWrapper1.getContent(), contentFromDisk1);
//Check cache
CacheItem contentCache2 = ConfigCacheService.getContentCache(
GroupKey2.getKey(configInfoWrapper2.getDataId(), configInfoWrapper2.getGroup(),
configInfoWrapper2.getTenant()));
// check if config2 is updated
Assert.assertEquals(MD5Utils.md5Hex(configInfoWrapperSingle2.getContent(), "UTF-8"),
contentCache2.getConfigCache().getMd5Utf8());
Assert.assertEquals(configInfoWrapper2.getLastModified(), contentCache2.getConfigCache().getLastModifiedTs());
//check disk
String contentFromDisk2 = ConfigDiskServiceFactory.getInstance()
.getContent(configInfoWrapper2.getDataId(), configInfoWrapper2.getGroup(),
configInfoWrapper2.getTenant());
Assert.assertEquals(configInfoWrapperSingle2.getContent(), contentFromDisk2);
}
}

22
config/src/test/java/com/alibaba/nacos/config/server/service/repository/embedded/EmbeddedConfigInfoPersistServiceImplTest.java
View File

@ -579,6 +579,14 @@ public class EmbeddedConfigInfoPersistServiceImplTest {
return configAllInfo;
}
private ConfigInfoStateWrapper createMockConfigInfoStateWrapper(long mockId) {
ConfigInfoStateWrapper configAllInfo = new ConfigInfoStateWrapper();
configAllInfo.setDataId("test" + mockId + ".yaml");
configAllInfo.setGroup("test");
configAllInfo.setLastModified(System.currentTimeMillis());
return configAllInfo;
}
private ConfigInfo createMockConfigInfo(long mockId) {
ConfigInfo configInfo = new ConfigInfo();
configInfo.setDataId("test" + mockId + ".yaml");
@ -784,17 +792,17 @@ public class EmbeddedConfigInfoPersistServiceImplTest {
public void testFindChangeConfig() {
//mock page list
List<ConfigInfoWrapper> result = new ArrayList<>();
result.add(createMockConfigInfoWrapper(0));
result.add(createMockConfigInfoWrapper(1));
result.add(createMockConfigInfoWrapper(2));
List<ConfigInfoStateWrapper> result = new ArrayList<>();
result.add(createMockConfigInfoStateWrapper(0));
result.add(createMockConfigInfoStateWrapper(1));
result.add(createMockConfigInfoStateWrapper(2));
Timestamp startTime = new Timestamp(System.currentTimeMillis() - 1000L);
long lastMaxId = 10000L;
int pageSize = 30;
when(databaseOperate.queryMany(anyString(), eq(new Object[] {startTime, lastMaxId, pageSize}),
eq(CONFIG_INFO_WRAPPER_ROW_MAPPER))).thenReturn(result);
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER))).thenReturn(result);
List<ConfigInfoWrapper> configInfo4List = embeddedConfigInfoPersistService.findChangeConfig(startTime,
List<ConfigInfoStateWrapper> configInfo4List = embeddedConfigInfoPersistService.findChangeConfig(startTime,
lastMaxId, pageSize);
Assert.assertEquals(result.size(), configInfo4List.size());
}
@ -1019,7 +1027,7 @@ public class EmbeddedConfigInfoPersistServiceImplTest {
int pageSize = 100;
//execute return mock obj
Page<ConfigInfoWrapper> returnConfigPage = embeddedConfigInfoPersistService.findAllConfigInfoFragment(lastId,
pageSize);
pageSize, true);
//expect check
Assert.assertEquals(mockConfigs, returnConfigPage.getPageItems());

52
config/src/test/java/com/alibaba/nacos/config/server/service/repository/embedded/EmbeddedHistoryConfigInfoPersistServiceImplTest.java
View File

@ -18,7 +18,7 @@ package com.alibaba.nacos.config.server.service.repository.embedded;
import com.alibaba.nacos.config.server.model.ConfigHistoryInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
import com.alibaba.nacos.persistence.model.Page;
@ -35,17 +35,11 @@ import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.math.BigInteger;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_DETAIL_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_LIST_ROW_MAPPER;
import static org.mockito.ArgumentMatchers.anyString;
@ -137,42 +131,38 @@ public class EmbeddedHistoryConfigInfoPersistServiceImplTest {
public void testFindDeletedConfig() {
//mock query list return
Map<String, Object> mockObj1 = new HashMap<>();
mockObj1.put("nid", new BigInteger("1234"));
mockObj1.put("data_id", "data_id1");
mockObj1.put("group_id", "group_id1");
mockObj1.put("tenant_id", "tenant_id1");
LocalDateTime now = LocalDateTime.of(LocalDate.now(), LocalTime.now());
mockObj1.put("gmt_modified", now);
List<Map<String, Object>> list = new ArrayList<>();
ConfigInfoStateWrapper mockObj1 = new ConfigInfoStateWrapper();
mockObj1.setDataId("data_id1");
mockObj1.setGroup("group_id1");
mockObj1.setTenant("tenant_id1");
mockObj1.setMd5("md51");
mockObj1.setLastModified(System.currentTimeMillis());
List<ConfigInfoStateWrapper> list = new ArrayList<>();
list.add(mockObj1);
Map<String, Object> mockObj2 = new HashMap<>();
mockObj2.put("nid", new BigInteger("12345"));
mockObj2.put("data_id", "data_id2");
mockObj2.put("group_id", "group_id2");
mockObj2.put("tenant_id", "tenant_id2");
LocalDateTime now2 = LocalDateTime.of(LocalDate.now(), LocalTime.now());
mockObj2.put("gmt_modified", now2);
ConfigInfoStateWrapper mockObj2 = new ConfigInfoStateWrapper();
mockObj2.setDataId("data_id2");
mockObj2.setGroup("group_id2");
mockObj2.setTenant("tenant_id2");
mockObj2.setMd5("md52");
list.add(mockObj2);
int pageSize = 1233;
long startId = 23456;
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
Mockito.when(databaseOperate.queryMany(anyString(), eq(new Object[] {timestamp, startId, pageSize})))
.thenReturn(list);
Mockito.when(databaseOperate.queryMany(anyString(), eq(new Object[] {timestamp, startId, pageSize}),
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER))).thenReturn(list);
//execute
List<ConfigInfoWrapper> deletedConfig = embeddedHistoryConfigInfoPersistService.findDeletedConfig(timestamp,
startId, pageSize);
List<ConfigInfoStateWrapper> deletedConfig = embeddedHistoryConfigInfoPersistService.findDeletedConfig(
timestamp, startId, pageSize);
//expect verify
Assert.assertEquals("data_id1", deletedConfig.get(0).getDataId());
Assert.assertEquals("group_id1", deletedConfig.get(0).getGroup());
Assert.assertEquals("tenant_id1", deletedConfig.get(0).getTenant());
Assert.assertEquals(now.toInstant(ZoneOffset.ofHours(8)).toEpochMilli(),
deletedConfig.get(0).getLastModified());
Assert.assertEquals(mockObj1.getLastModified(), deletedConfig.get(0).getLastModified());
Assert.assertEquals("data_id2", deletedConfig.get(1).getDataId());
Assert.assertEquals("group_id2", deletedConfig.get(1).getGroup());
Assert.assertEquals("tenant_id2", deletedConfig.get(1).getTenant());
Assert.assertEquals(now2.toInstant(ZoneOffset.ofHours(8)).toEpochMilli(),
deletedConfig.get(1).getLastModified());
Assert.assertEquals(mockObj2.getLastModified(), deletedConfig.get(1).getLastModified());
}
@Test

28
config/src/test/java/com/alibaba/nacos/config/server/service/repository/extrnal/ExternalConfigInfoPersistServiceImplTest.java
View File

@ -642,6 +642,14 @@ public class ExternalConfigInfoPersistServiceImplTest {
return configAllInfo;
}
private ConfigInfoStateWrapper createMockConfigInfoStateWrapper(long mockId) {
ConfigInfoStateWrapper configAllInfo = new ConfigInfoStateWrapper();
configAllInfo.setDataId("test" + mockId + ".yaml");
configAllInfo.setGroup("test");
configAllInfo.setLastModified(System.currentTimeMillis());
return configAllInfo;
}
private ConfigInfo createMockConfigInfo(long mockId) {
ConfigInfo configInfo = new ConfigInfo();
configInfo.setDataId("test" + mockId + ".yaml");
@ -900,17 +908,17 @@ public class ExternalConfigInfoPersistServiceImplTest {
public void testFindChangeConfig() {
//mock page list
List<ConfigInfoWrapper> result = new ArrayList<>();
result.add(createMockConfigInfoWrapper(0));
result.add(createMockConfigInfoWrapper(1));
result.add(createMockConfigInfoWrapper(2));
List<ConfigInfoStateWrapper> result = new ArrayList<>();
result.add(createMockConfigInfoStateWrapper(0));
result.add(createMockConfigInfoStateWrapper(1));
result.add(createMockConfigInfoStateWrapper(2));
Timestamp startTime = new Timestamp(System.currentTimeMillis() - 1000L);
long lastMaxId = 10000L;
int pageSize = 30;
when(jdbcTemplate.query(anyString(), eq(new Object[] {startTime, lastMaxId, pageSize}),
eq(CONFIG_INFO_WRAPPER_ROW_MAPPER))).thenReturn(result);
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER))).thenReturn(result);
List<ConfigInfoWrapper> configInfo4List = externalConfigInfoPersistService.findChangeConfig(startTime,
List<ConfigInfoStateWrapper> configInfo4List = externalConfigInfoPersistService.findChangeConfig(startTime,
lastMaxId, pageSize);
Assert.assertEquals(result.size(), configInfo4List.size());
}
@ -922,9 +930,9 @@ public class ExternalConfigInfoPersistServiceImplTest {
int pageSize = 30;
//mock page list
when(jdbcTemplate.query(anyString(), eq(new Object[] {startTime, lastMaxId, pageSize}),
eq(CONFIG_INFO_WRAPPER_ROW_MAPPER))).thenThrow(new CannotAcquireLockException("mock ex"));
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER))).thenThrow(new CannotAcquireLockException("mock ex"));
try {
List<ConfigInfoWrapper> configInfo4List = externalConfigInfoPersistService.findChangeConfig(startTime,
List<ConfigInfoStateWrapper> configInfo4List = externalConfigInfoPersistService.findChangeConfig(startTime,
lastMaxId, pageSize);
Assert.assertTrue(false);
} catch (Exception e) {
@ -1256,7 +1264,7 @@ public class ExternalConfigInfoPersistServiceImplTest {
int pageSize = 100;
//execute return mock obj
Page<ConfigInfoWrapper> returnConfigPage = externalConfigInfoPersistService.findAllConfigInfoFragment(lastId,
pageSize);
pageSize, true);
//expect check
Assert.assertEquals(mockConfigs, returnConfigPage.getPageItems());
@ -1264,7 +1272,7 @@ public class ExternalConfigInfoPersistServiceImplTest {
when(jdbcTemplate.query(anyString(), eq(new Object[] {lastId}), eq(CONFIG_INFO_WRAPPER_ROW_MAPPER))).thenThrow(
new CannotGetJdbcConnectionException("mock fail"));
try {
externalConfigInfoPersistService.findAllConfigInfoFragment(lastId, pageSize);
externalConfigInfoPersistService.findAllConfigInfoFragment(lastId, pageSize, true);
Assert.assertTrue(false);
} catch (Exception e) {
Assert.assertEquals("mock fail", e.getMessage());

54
config/src/test/java/com/alibaba/nacos/config/server/service/repository/extrnal/ExternalHistoryConfigInfoPersistServiceImplTest.java
View File

@ -18,7 +18,7 @@ package com.alibaba.nacos.config.server.service.repository.extrnal;
import com.alibaba.nacos.config.server.model.ConfigHistoryInfo;
import com.alibaba.nacos.config.server.model.ConfigInfo;
import com.alibaba.nacos.config.server.model.ConfigInfoWrapper;
import com.alibaba.nacos.config.server.model.ConfigInfoStateWrapper;
import com.alibaba.nacos.config.server.service.sql.ExternalStorageUtils;
import com.alibaba.nacos.config.server.utils.TestCaseUtils;
import com.alibaba.nacos.persistence.datasource.DataSourceService;
@ -39,17 +39,11 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.transaction.support.TransactionTemplate;
import java.math.BigInteger;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_DETAIL_ROW_MAPPER;
import static com.alibaba.nacos.config.server.service.repository.ConfigRowMapperInjector.HISTORY_LIST_ROW_MAPPER;
import static org.mockito.ArgumentMatchers.anyString;
@ -150,44 +144,42 @@ public class ExternalHistoryConfigInfoPersistServiceImplTest {
public void testFindDeletedConfig() {
//mock query list return
Map<String, Object> mockObj1 = new HashMap<>();
mockObj1.put("nid", new BigInteger("1234"));
mockObj1.put("data_id", "data_id1");
mockObj1.put("group_id", "group_id1");
mockObj1.put("tenant_id", "tenant_id1");
LocalDateTime now = LocalDateTime.of(LocalDate.now(), LocalTime.now());
mockObj1.put("gmt_modified", now);
List<Map<String, Object>> list = new ArrayList<>();
ConfigInfoStateWrapper mockObj1 = new ConfigInfoStateWrapper();
mockObj1.setDataId("data_id1");
mockObj1.setGroup("group_id1");
mockObj1.setTenant("tenant_id1");
mockObj1.setMd5("md51");
mockObj1.setLastModified(System.currentTimeMillis());
List<ConfigInfoStateWrapper> list = new ArrayList<>();
list.add(mockObj1);
Map<String, Object> mockObj2 = new HashMap<>();
mockObj2.put("nid", new BigInteger("12345"));
mockObj2.put("data_id", "data_id2");
mockObj2.put("group_id", "group_id2");
mockObj2.put("tenant_id", "tenant_id2");
LocalDateTime now2 = LocalDateTime.of(LocalDate.now(), LocalTime.now());
mockObj2.put("gmt_modified", now2);
ConfigInfoStateWrapper mockObj2 = new ConfigInfoStateWrapper();
mockObj2.setDataId("data_id2");
mockObj2.setGroup("group_id2");
mockObj2.setTenant("tenant_id2");
mockObj2.setMd5("md52");
list.add(mockObj2);
int pageSize = 1233;
long startId = 23456;
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
Mockito.when(jdbcTemplate.queryForList(anyString(), eq(timestamp), eq(startId), eq(pageSize))).thenReturn(list);
Mockito.when(jdbcTemplate.query(anyString(), eq(new Object[] {timestamp, startId, pageSize}),
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER))).thenReturn(list);
//execute
List<ConfigInfoWrapper> deletedConfig = externalHistoryConfigInfoPersistService.findDeletedConfig(timestamp,
startId, pageSize);
List<ConfigInfoStateWrapper> deletedConfig = externalHistoryConfigInfoPersistService.findDeletedConfig(
timestamp, startId, pageSize);
//expect verify
Assert.assertEquals("data_id1", deletedConfig.get(0).getDataId());
Assert.assertEquals("group_id1", deletedConfig.get(0).getGroup());
Assert.assertEquals("tenant_id1", deletedConfig.get(0).getTenant());
Assert.assertEquals(now.toInstant(ZoneOffset.ofHours(8)).toEpochMilli(),
deletedConfig.get(0).getLastModified());
Assert.assertEquals(mockObj1.getLastModified(), deletedConfig.get(0).getLastModified());
Assert.assertEquals("data_id2", deletedConfig.get(1).getDataId());
Assert.assertEquals("group_id2", deletedConfig.get(1).getGroup());
Assert.assertEquals("tenant_id2", deletedConfig.get(1).getTenant());
Assert.assertEquals(now2.toInstant(ZoneOffset.ofHours(8)).toEpochMilli(),
deletedConfig.get(1).getLastModified());
Assert.assertEquals(mockObj2.getLastModified(), deletedConfig.get(1).getLastModified());
//mock exception
Mockito.when(jdbcTemplate.queryForList(anyString(), eq(timestamp), eq(startId), eq(pageSize)))
Mockito.when(jdbcTemplate.query(anyString(), eq(new Object[] {timestamp, startId, pageSize}),
eq(CONFIG_INFO_STATE_WRAPPER_ROW_MAPPER)))
.thenThrow(new CannotGetJdbcConnectionException("conn error"));
try {

94
config/src/test/java/com/alibaba/nacos/config/server/utils/PropertyUtilTest.java
View File

@ -17,43 +17,113 @@
package com.alibaba.nacos.config.server.utils;
import com.alibaba.nacos.sys.env.EnvUtil;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.security.util.FieldUtils;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.io.File;
import java.lang.reflect.Field;
import static org.mockito.ArgumentMatchers.eq;
@RunWith(SpringJUnit4ClassRunner.class)
public class PropertyUtilTest {
@Mock
private ConfigurableEnvironment configurableEnvironment;
MockedStatic<EnvUtil> envUtilMockedStatic;
private String mockMem = "tmpmocklimitfile.txt";
@Before
public void setUp() {
envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("memory_limit_file_path"),
eq("/sys/fs/cgroup/memory/memory.limit_in_bytes"))).thenReturn(mockMem);
}
@After
public void after() {
envUtilMockedStatic.close();
File file = new File(mockMem);
if (file.exists()) {
file.delete();
}
}
@Test
public void testGetPropertyV1() {
MockedStatic<EnvUtil> envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
EnvUtil.setEnvironment(configurableEnvironment);
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("test"))).thenReturn("test");
Assert.assertEquals("test", new PropertyUtil().getProperty("test"));
envUtilMockedStatic.close();
}
@Test
public void testGetPropertyV2() {
MockedStatic<EnvUtil> envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
EnvUtil.setEnvironment(configurableEnvironment);
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("test"), eq("default"))).thenReturn("default");
Assert.assertEquals("default", new PropertyUtil().getProperty("test", "default"));
envUtilMockedStatic.close();
}
}
private void clearAllDumpFiled() throws Exception {
Field allDumpPageSizeFiled = FieldUtils.getField(PropertyUtil.class, "allDumpPageSize");
allDumpPageSizeFiled.setAccessible(true);
allDumpPageSizeFiled.set(null, null);
}
@Test
public void testGetAllDumpPageSize() throws Exception {
clearAllDumpFiled();
File file = new File(mockMem);
//2G pageSize between 50 to 1000
long gb2 = 2L * 1024L * 1024L * 1024L;
FileUtils.writeStringToFile(file, String.valueOf(gb2));
int allDumpPageSizeNormal = PropertyUtil.getAllDumpPageSize();
//expect 2*2*50
Assert.assertEquals(200, allDumpPageSizeNormal);
clearAllDumpFiled();
// 12G pageSize over 1000
long gb12 = 12L * 1024L * 1024L * 1024L;
FileUtils.writeStringToFile(file, String.valueOf(gb12));
int allDumpPageSizeOverMax = PropertyUtil.getAllDumpPageSize();
Assert.assertEquals(1000, allDumpPageSizeOverMax);
clearAllDumpFiled();
//100MB
long mb100 = 100L * 1024L * 1024L;
FileUtils.writeStringToFile(file, String.valueOf(mb100));
int allDumpPageSizeUnderMin = PropertyUtil.getAllDumpPageSize();
Assert.assertEquals(50, allDumpPageSizeUnderMin);
}
@Test
public void testGetAllDumpPageSizeWithJvmArgs() throws Exception {
File file = new File(mockMem);
if (file.exists()) {
file.delete();
}
int allDumpPageSizeUnderMin = PropertyUtil.initAllDumpPageSize();
long maxMem = Runtime.getRuntime().maxMemory();
long pageSize = maxMem / 1024 / 1024 / 512 * 50;
if (pageSize < 50) {
Assert.assertEquals(50, allDumpPageSizeUnderMin);
} else if (pageSize > 1000) {
Assert.assertEquals(1000, allDumpPageSizeUnderMin);
} else {
Assert.assertEquals(pageSize, allDumpPageSizeUnderMin);
}
}
}

28
plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/constants/ContextConstant.java Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright 1999-2023 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.plugin.datasource.constants;
/**
* Datasource plugin common constant.
*
* @author zunfei.lzf
*/
public class ContextConstant {
public static final String NEED_CONTENT = "needContent";
}

16
plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/derby/ConfigInfoMapperByDerby.java
View File

@ -19,6 +19,7 @@ package com.alibaba.nacos.plugin.datasource.impl.derby;
import com.alibaba.nacos.common.utils.CollectionUtils;
import com.alibaba.nacos.common.utils.NamespaceUtil;
import com.alibaba.nacos.common.utils.StringUtils;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.DataSourceConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.mapper.AbstractMapper;
@ -77,8 +78,7 @@ public class ConfigInfoMapperByDerby extends AbstractMapper implements ConfigInf
+ " ( SELECT id FROM config_info WHERE tenant_id LIKE ? ORDER BY id OFFSET " + context.getStartRow()
+ " ROWS FETCH NEXT " + context.getPageSize() + " ROWS ONLY ) "
+ "g, config_info t WHERE g.id = t.id ";
return new MapperResult(sql,
CollectionUtils.list(context.getWhereParameter(FieldConstant.TENANT_ID)));
return new MapperResult(sql, CollectionUtils.list(context.getWhereParameter(FieldConstant.TENANT_ID)));
}
@Override
@ -92,11 +92,11 @@ public class ConfigInfoMapperByDerby extends AbstractMapper implements ConfigInf
@Override
public MapperResult findAllConfigInfoFragment(MapperContext context) {
return new MapperResult(
"SELECT id,data_id,group_id,tenant_id,app_name,content,md5,gmt_modified,type FROM config_info WHERE id > ? "
+ "ORDER BY id ASC OFFSET " + context.getStartRow() + " ROWS FETCH NEXT "
+ context.getPageSize() + " ROWS ONLY",
String contextParameter = context.getContextParameter(ContextConstant.NEED_CONTENT);
boolean needContent = contextParameter != null && Boolean.parseBoolean(contextParameter);
return new MapperResult("SELECT id,data_id,group_id,tenant_id,app_name," + (needContent ? "content," : "")
+ "md5,gmt_modified,type FROM config_info WHERE id > ? " + "ORDER BY id ASC OFFSET "
+ context.getStartRow() + " ROWS FETCH NEXT " + context.getPageSize() + " ROWS ONLY",
CollectionUtils.list(context.getWhereParameter(FieldConstant.ID)));
}
@ -276,7 +276,7 @@ public class ConfigInfoMapperByDerby extends AbstractMapper implements ConfigInf
public String getDataSource() {
return DataSourceConstant.DERBY;
}
@Override
public MapperResult findChangeConfig(MapperContext context) {
String sql =

13
plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/mysql/ConfigInfoMapperByMySql.java
View File

@ -19,6 +19,7 @@ package com.alibaba.nacos.plugin.datasource.impl.mysql;
import com.alibaba.nacos.common.utils.CollectionUtils;
import com.alibaba.nacos.common.utils.NamespaceUtil;
import com.alibaba.nacos.common.utils.StringUtils;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.DataSourceConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.mapper.AbstractMapper;
@ -83,9 +84,6 @@ public class ConfigInfoMapperByMySql extends AbstractMapper implements ConfigInf
@Override
public MapperResult findAllConfigInfoBaseFetchRows(MapperContext context) {
String appName = (String) context.getWhereParameter(FieldConstant.APP_NAME);
final String tenantId = (String) context.getWhereParameter(FieldConstant.TENANT_ID);
String sql = "SELECT t.id,data_id,group_id,content,md5"
+ " FROM ( SELECT id FROM config_info ORDER BY id LIMIT ?,? ) "
+ " g, config_info t WHERE g.id = t.id ";
@ -94,10 +92,11 @@ public class ConfigInfoMapperByMySql extends AbstractMapper implements ConfigInf
@Override
public MapperResult findAllConfigInfoFragment(MapperContext context) {
String sql = "SELECT id,data_id,group_id,tenant_id,app_name,content,md5,gmt_modified,type,encrypted_data_key "
+ "FROM config_info WHERE id > ? ORDER BY id ASC LIMIT " + context.getStartRow() + ","
+ context.getPageSize();
String contextParameter = context.getContextParameter(ContextConstant.NEED_CONTENT);
boolean needContent = contextParameter != null && Boolean.parseBoolean(contextParameter);
String sql = "SELECT id,data_id,group_id,tenant_id,app_name," + (needContent ? "content," : "")
+ "md5,gmt_modified,type,encrypted_data_key FROM config_info WHERE id > ? ORDER BY id ASC LIMIT "
+ context.getStartRow() + "," + context.getPageSize();
return new MapperResult(sql, CollectionUtils.list(context.getWhereParameter(FieldConstant.ID)));
}

2
plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/mapper/ConfigInfoMapper.java
View File

@ -135,7 +135,7 @@ public interface ConfigInfoMapper extends Mapper {
* id,data_id,group_id,tenant_id,app_name,content,md5,gmt_modified,type,encrypted_data_key FROM config_info WHERE id
* > ? ORDER BY id ASC LIMIT startRow,pageSize
*
* @param context The context of startRow, pageSize
* @param context The context of startRow, pageSize
* @return The sql of querying all config info.
*/
MapperResult findAllConfigInfoFragment(MapperContext context);

26
plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/model/MapperContext.java
View File

@ -31,6 +31,8 @@ public class MapperContext {
private final Map<String, Object> updateParamMap;
private final Map<String, String> contextParamMap;
private int startRow;
private int pageSize;
@ -38,11 +40,11 @@ public class MapperContext {
public MapperContext() {
this.whereParamMap = new HashMap<>();
this.updateParamMap = new HashMap<>();
this.contextParamMap = new HashMap<>();
}
public MapperContext(int startRow, int pageSize) {
this.whereParamMap = new HashMap<>();
this.updateParamMap = new HashMap<>();
this();
this.startRow = startRow;
this.pageSize = pageSize;
}
@ -67,6 +69,26 @@ public class MapperContext {
this.whereParamMap.put(key, value);
}
/**
* Returns the value to which the key is mapped, it will return the context param.
*
* @param key The key whose associated value is to be returned
* @return The value to which the key is mapped
*/
public String getContextParameter(String key) {
return contextParamMap.get(key);
}
/**
* Associates the value with the key in this map, it will contain the context parameter.
*
* @param key Key with which the value is to be associated
* @param value Value to be associated with the specified key
*/
public void putContextParameter(String key, String value) {
this.contextParamMap.put(key, value);
}
/**
* Returns the value to which the key is mapped, it will return the UPDATE parameter in the SQL statement.
*

12
plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/derby/ConfigInfoMapperByDerbyTest.java
View File

@ -16,6 +16,7 @@
package com.alibaba.nacos.plugin.datasource.impl.derby;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.DataSourceConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.constants.TableConstant;
@ -149,12 +150,23 @@ public class ConfigInfoMapperByDerbyTest {
@Test
public void testFindAllConfigInfoFragment() {
//with content
context.putContextParameter(ContextConstant.NEED_CONTENT, "true");
MapperResult mapperResult = configInfoMapperByDerby.findAllConfigInfoFragment(context);
Assert.assertEquals(mapperResult.getSql(),
"SELECT id,data_id,group_id,tenant_id,app_name,content,md5,gmt_modified,type FROM config_info "
+ "WHERE id > ? ORDER BY id ASC OFFSET " + startRow + " ROWS FETCH NEXT " + pageSize
+ " ROWS ONLY");
Assert.assertArrayEquals(mapperResult.getParamList().toArray(), new Object[] {id});
//with out content
context.putContextParameter(ContextConstant.NEED_CONTENT, "false");
MapperResult mapperResult2 = configInfoMapperByDerby.findAllConfigInfoFragment(context);
Assert.assertEquals(mapperResult2.getSql(),
"SELECT id,data_id,group_id,tenant_id,app_name,md5,gmt_modified,type FROM config_info "
+ "WHERE id > ? ORDER BY id ASC OFFSET " + startRow + " ROWS FETCH NEXT " + pageSize
+ " ROWS ONLY");
Assert.assertArrayEquals(mapperResult2.getParamList().toArray(), new Object[] {id});
}
@Test

16
plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/mysql/ConfigInfoMapperByMySqlTest.java
View File

@ -17,6 +17,7 @@
package com.alibaba.nacos.plugin.datasource.impl.mysql;
import com.alibaba.nacos.common.utils.NamespaceUtil;
import com.alibaba.nacos.plugin.datasource.constants.ContextConstant;
import com.alibaba.nacos.plugin.datasource.constants.DataSourceConstant;
import com.alibaba.nacos.plugin.datasource.constants.FieldConstant;
import com.alibaba.nacos.plugin.datasource.constants.TableConstant;
@ -149,11 +150,22 @@ public class ConfigInfoMapperByMySqlTest {
@Test
public void testFindAllConfigInfoFragment() {
//with content
context.putContextParameter(ContextConstant.NEED_CONTENT, "true");
MapperResult mapperResult = configInfoMapperByMySql.findAllConfigInfoFragment(context);
Assert.assertEquals(mapperResult.getSql(),
Assert.assertEquals(
"SELECT id,data_id,group_id,tenant_id,app_name,content,md5,gmt_modified,type,encrypted_data_key "
+ "FROM config_info WHERE id > ? ORDER BY id ASC LIMIT " + startRow + "," + pageSize);
+ "FROM config_info WHERE id > ? ORDER BY id ASC LIMIT " + startRow + "," + pageSize,
mapperResult.getSql());
Assert.assertArrayEquals(mapperResult.getParamList().toArray(), new Object[] {id});
context.putContextParameter(ContextConstant.NEED_CONTENT, "false");
MapperResult mapperResult2 = configInfoMapperByMySql.findAllConfigInfoFragment(context);
Assert.assertEquals("SELECT id,data_id,group_id,tenant_id,app_name,md5,gmt_modified,type,encrypted_data_key "
+ "FROM config_info WHERE id > ? ORDER BY id ASC LIMIT " + startRow + "," + pageSize,
mapperResult2.getSql());
Assert.assertArrayEquals(mapperResult2.getParamList().toArray(), new Object[] {id});
}
@Test