zhuyijun/nacos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ISSUE#12359]Refactoring the historical configuration cleanup strategy with SPI (#12367)

Browse Source 
* 历史配置清理逻辑SPI改造

* 添加单元测试

* 单元测试fix

* 单元测试fix

* 优化spi实现

* 优化日志打印
This commit is contained in:
Sunrisea 2024-07-23 16:42:29 +08:00 committed by GitHub
parent 197795a854
commit 7e577811a7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 485 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
config/src/main/java/com/alibaba/nacos/config/server/service/dump/DefaultHistoryConfigCleaner.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.config.server.utils.TimeUtils;
import com.alibaba.nacos.sys.env.EnvUtil;
import com.alibaba.nacos.sys.utils.ApplicationUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import static com.alibaba.nacos.config.server.utils.LogUtil.FATAL_LOG;
/**
* The type Default history config cleaner.
*
* @author Sunrisea
*/
public class DefaultHistoryConfigCleaner implements HistoryConfigCleaner {
private static final Logger LOGGER = LoggerFactory.getLogger(DefaultHistoryConfigCleaner.class);
private HistoryConfigInfoPersistService historyConfigInfoPersistService;
private int retentionDays = 30;
@Override
public void cleanHistoryConfig() {
Timestamp startTime = getBeforeStamp(TimeUtils.getCurrentTime(), 24 * getRetentionDays());
int pageSize = 1000;
LOGGER.warn("clearConfigHistory, getBeforeStamp:{}, pageSize:{}", startTime, pageSize);
getHistoryConfigInfoPersistService().removeConfigHistory(startTime, pageSize);
}
private HistoryConfigInfoPersistService getHistoryConfigInfoPersistService() {
if (historyConfigInfoPersistService == null) {
historyConfigInfoPersistService = ApplicationUtils.getBean(HistoryConfigInfoPersistService.class);
}
return historyConfigInfoPersistService;
}
private Timestamp getBeforeStamp(Timestamp date, int step) {
Calendar cal = Calendar.getInstance();
cal.setTime(date);
cal.add(Calendar.HOUR_OF_DAY, -step);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return Timestamp.valueOf(format.format(cal.getTime()));
}
private int getRetentionDays() {
String val = EnvUtil.getProperty("nacos.config.retention.days");
if (null == val) {
return retentionDays;
}
int tmp = 0;
try {
tmp = Integer.parseInt(val);
if (tmp > 0) {
retentionDays = tmp;
}
} catch (NumberFormatException nfe) {
FATAL_LOG.error("read nacos.config.retention.days wrong", nfe);
}
return retentionDays;
}
@Override
public String getName() {
return "nacos";
}
}

71
config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java
View File

@ -44,7 +44,6 @@ import com.alibaba.nacos.config.server.utils.ConfigExecutor;
import com.alibaba.nacos.config.server.utils.GroupKey2;
import com.alibaba.nacos.config.server.utils.LogUtil;
import com.alibaba.nacos.config.server.utils.PropertyUtil;
import com.alibaba.nacos.config.server.utils.TimeUtils;
import com.alibaba.nacos.core.cluster.ServerMemberManager;
import com.alibaba.nacos.core.namespace.repository.NamespacePersistService;
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
@ -54,14 +53,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import static com.alibaba.nacos.config.server.utils.LogUtil.DUMP_LOG;
import static com.alibaba.nacos.config.server.utils.LogUtil.FATAL_LOG;
/**
* Dump data service.
@ -194,29 +190,31 @@ public abstract class DumpService {
*/
protected abstract void init() throws Throwable;
void clearConfigHistory() {
LOGGER.warn("clearConfigHistory start");
if (canExecute()) {
try {
Timestamp startTime = getBeforeStamp(TimeUtils.getCurrentTime(), 24 * getRetentionDays());
int pageSize = 1000;
LOGGER.warn("clearConfigHistory, getBeforeStamp:{}, pageSize:{}", startTime, pageSize);
historyConfigInfoPersistService.removeConfigHistory(startTime, pageSize);
} catch (Throwable e) {
LOGGER.error("clearConfigHistory error : {}", e.toString());
}
}
}
/**
* config history clear.
*/
class ConfigHistoryClear implements Runnable {
private HistoryConfigCleaner historyConfigCleaner;
public ConfigHistoryClear(HistoryConfigCleaner historyConfigCleaner) {
this.historyConfigCleaner = historyConfigCleaner;
}
@Override
public void run() {
clearConfigHistory();
LOGGER.warn("clearHistoryConfig get scheduled");
if (canExecute()) {
try {
LOGGER.warn("clearHistoryConfig is enable in current context, try to run cleaner");
historyConfigCleaner.cleanHistoryConfig();
LOGGER.warn("history config cleaner successfully");
} catch (Throwable e) {
LOGGER.error("clearConfigHistory error : {}", e.toString());
}
} else {
LOGGER.warn("clearHistoryConfig is disable in current context");
}
}
}
@ -316,7 +314,10 @@ public abstract class DumpService {
}
ConfigExecutor.scheduleConfigTask(new ConfigHistoryClear(), 10, 10, TimeUnit.MINUTES);
HistoryConfigCleaner cleaner = HistoryConfigCleanerManager.getHistoryConfigCleaner(
HistoryConfigCleanerConfig.getInstance().getActiveHistoryConfigCleaner());
ConfigExecutor.scheduleConfigTask(new ConfigHistoryClear(cleaner), 10, 10, TimeUnit.MINUTES);
} finally {
TimerContext.end(dumpFileContext, LogUtil.DUMP_LOG);
}
@ -335,34 +336,6 @@ public abstract class DumpService {
}
}
private Timestamp getBeforeStamp(Timestamp date, int step) {
Calendar cal = Calendar.getInstance();
cal.setTime(date);
// before 6 hour
cal.add(Calendar.HOUR_OF_DAY, -step);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return Timestamp.valueOf(format.format(cal.getTime()));
}
private int getRetentionDays() {
String val = EnvUtil.getProperty("nacos.config.retention.days");
if (null == val) {
return retentionDays;
}
int tmp = 0;
try {
tmp = Integer.parseInt(val);
if (tmp > 0) {
retentionDays = tmp;
}
} catch (NumberFormatException nfe) {
FATAL_LOG.error("read nacos.config.retention.days wrong", nfe);
}
return retentionDays;
}
/**
* dump operation.
*

36
config/src/main/java/com/alibaba/nacos/config/server/service/dump/HistoryConfigCleaner.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
/**
* The interface History config cleaner.
* @author Sunrisea
*/
public interface HistoryConfigCleaner {
/**
* Clean history config.
*/
public void cleanHistoryConfig();
/**
* Gets name.
*
* @return the name
*/
public String getName();
}

79
config/src/main/java/com/alibaba/nacos/config/server/service/dump/HistoryConfigCleanerConfig.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.api.utils.StringUtils;
import com.alibaba.nacos.core.config.AbstractDynamicConfig;
import com.alibaba.nacos.sys.env.EnvUtil;
/**
* The type History config cleaner config.
* @author Sunrisea
*/
public class HistoryConfigCleanerConfig extends AbstractDynamicConfig {
private static final String HISTORY_CONFIG_CLEANER = "historyConfigCleaner";
private static final HistoryConfigCleanerConfig INSTANCE = new HistoryConfigCleanerConfig();
private String activeHistoryConfigCleaner = "nacos";
private HistoryConfigCleanerConfig() {
super(HISTORY_CONFIG_CLEANER);
resetConfig();
}
/**
* Gets instance.
*
* @return the instance
*/
public static HistoryConfigCleanerConfig getInstance() {
return INSTANCE;
}
@Override
protected void getConfigFromEnv() {
activeHistoryConfigCleaner = EnvUtil.getProperty("nacos.config.history.clear.name", String.class, "nacos");
if (StringUtils.isBlank(activeHistoryConfigCleaner)) {
activeHistoryConfigCleaner = "nacos";
}
}
/**
* Gets active history config cleaner.
*
* @return the active history config cleaner
*/
public String getActiveHistoryConfigCleaner() {
return activeHistoryConfigCleaner;
}
/**
* Sets active history config cleaner.
*
* @param activeHistoryConfigCleaner the active history config cleaner
*/
public void setActiveHistoryConfigCleaner(String activeHistoryConfigCleaner) {
this.activeHistoryConfigCleaner = activeHistoryConfigCleaner;
}
@Override
protected String printConfig() {
return "activeHistoryConfigCleaner{ " + "activeHistoryConfigCleaner=" + activeHistoryConfigCleaner + "}";
}
}

48
config/src/main/java/com/alibaba/nacos/config/server/service/dump/HistoryConfigCleanerManager.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.common.spi.NacosServiceLoader;
import java.util.HashMap;
/**
* The type History config cleaner manager.
*
* @author Sunrisea
*/
public class HistoryConfigCleanerManager {
private static HashMap<String, HistoryConfigCleaner> historyConfigCleanerMap = new HashMap<String, HistoryConfigCleaner>();
static {
NacosServiceLoader.load(HistoryConfigCleaner.class).forEach(historyConfigCleaner -> {
historyConfigCleanerMap.put(historyConfigCleaner.getName(), historyConfigCleaner);
});
historyConfigCleanerMap.put("nacos", new DefaultHistoryConfigCleaner());
}
/**
* Gets history config cleaner.
*
* @param name the name
* @return the history config cleaner
*/
public static HistoryConfigCleaner getHistoryConfigCleaner(String name) {
return historyConfigCleanerMap.getOrDefault(name, historyConfigCleanerMap.get("nacos"));
}
}

102
config/src/test/java/com/alibaba/nacos/config/server/service/dump/DefaultHistoryConfigCleanerTest.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.config.server.service.repository.HistoryConfigInfoPersistService;
import com.alibaba.nacos.config.server.utils.ConfigExecutor;
import com.alibaba.nacos.sys.env.EnvUtil;
import com.alibaba.nacos.sys.utils.ApplicationUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.lang.reflect.Method;
import java.sql.Timestamp;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
@ExtendWith(SpringExtension.class)
public class DefaultHistoryConfigCleanerTest {
private DefaultHistoryConfigCleaner defaultHistoryConfigCleaner = new DefaultHistoryConfigCleaner();
@Mock
private HistoryConfigInfoPersistService historyConfigInfoPersistService;
MockedStatic<ApplicationUtils> applicationUtilsMockedStatic;
MockedStatic<ConfigExecutor> configExecutorMocked;
MockedStatic<EnvUtil> envUtilMockedStatic;
/**
* Sets up.
*/
@BeforeEach
public void setUp() {
applicationUtilsMockedStatic = Mockito.mockStatic(ApplicationUtils.class);
applicationUtilsMockedStatic.when(() -> ApplicationUtils.getBean(HistoryConfigInfoPersistService.class))
.thenReturn(historyConfigInfoPersistService);
configExecutorMocked = Mockito.mockStatic(ConfigExecutor.class);
envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
}
/**
* End.
*/
@AfterEach
public void end() {
applicationUtilsMockedStatic.close();
configExecutorMocked.close();
envUtilMockedStatic.close();
}
@Test
public void test() {
HistoryConfigCleaner configCleaner = HistoryConfigCleanerManager.getHistoryConfigCleaner("nacos");
assertEquals(configCleaner.getName(), "nacos");
}
@Test
public void testCleanHistoryConfig() throws Exception {
defaultHistoryConfigCleaner.cleanHistoryConfig();
Mockito.verify(historyConfigInfoPersistService, Mockito.times(1))
.removeConfigHistory(any(Timestamp.class), anyInt());
}
@Test
public void testGetRetentionDays() throws Exception {
Method method = DefaultHistoryConfigCleaner.class.getDeclaredMethod("getRetentionDays");
method.setAccessible(true);
envUtilMockedStatic.when(() -> EnvUtil.getProperty("nacos.config.retention.days")).thenReturn("-1");
assertEquals((int) method.invoke(defaultHistoryConfigCleaner), 30);
envUtilMockedStatic.when(() -> EnvUtil.getProperty("nacos.config.retention.days")).thenReturn("30");
assertEquals((int) method.invoke(defaultHistoryConfigCleaner), 30);
envUtilMockedStatic.when(() -> EnvUtil.getProperty("nacos.config.retention.days")).thenReturn("1");
assertEquals((int) method.invoke(defaultHistoryConfigCleaner), 1);
}
}

22
config/src/test/java/com/alibaba/nacos/config/server/service/dump/DumpServiceTest.java
View File

@ -44,7 +44,6 @@ import org.mockito.Mockito;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.util.ReflectionTestUtils;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -54,6 +53,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
@ -64,6 +64,9 @@ class DumpServiceTest {
private static final String TAG_TABLE_NAME = "config_info_tag";
@Mock
DefaultHistoryConfigCleaner defaultHistoryConfigCleaner = new DefaultHistoryConfigCleaner();
@Mock
ConfigInfoPersistService configInfoPersistService;
@ -95,6 +98,8 @@ class DumpServiceTest {
MockedStatic<PropertyUtil> propertyUtilMockedStatic;
MockedStatic<HistoryConfigCleanerManager> historyConfigCleanerManagerMockedStatic;
@Mock
private DataSourceService dataSourceService;
@ -115,6 +120,9 @@ class DumpServiceTest {
dumpService = new ExternalDumpService(configInfoPersistService, namespacePersistService, historyConfigInfoPersistService,
configInfoAggrPersistService, configInfoBetaPersistService, configInfoTagPersistService, mergeDatumService, memberManager);
configExecutorMocked = Mockito.mockStatic(ConfigExecutor.class);
historyConfigCleanerManagerMockedStatic = Mockito.mockStatic(HistoryConfigCleanerManager.class);
historyConfigCleanerManagerMockedStatic.when(() -> HistoryConfigCleanerManager.getHistoryConfigCleaner(anyString()))
.thenReturn(defaultHistoryConfigCleaner);
}
@ -123,6 +131,7 @@ class DumpServiceTest {
envUtilMockedStatic.close();
configExecutorMocked.close();
propertyUtilMockedStatic.close();
historyConfigCleanerManagerMockedStatic.close();
}
@Test
@ -204,16 +213,19 @@ class DumpServiceTest {
configExecutorMocked.verify(
() -> ConfigExecutor.scheduleConfigChangeTask(any(DumpChangeConfigWorker.class), anyLong(), eq(TimeUnit.MILLISECONDS)),
times(1));
configExecutorMocked.verify(() -> ConfigExecutor.scheduleConfigTask(any(DumpService.ConfigHistoryClear.class), anyLong(), anyLong(),
eq(TimeUnit.MINUTES)), times(1));
configExecutorMocked.verify(
() -> ConfigExecutor.scheduleConfigTask(any(DumpService.ConfigHistoryClear.class), anyLong(), anyLong(),
eq(TimeUnit.MINUTES)), times(1)
);
}
@Test
void clearHistory() {
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("nacos.config.retention.days"))).thenReturn("10");
Mockito.when(memberManager.isFirstIp()).thenReturn(true);
dumpService.clearConfigHistory();
Mockito.verify(historyConfigInfoPersistService, times(1)).removeConfigHistory(any(Timestamp.class), anyInt());
DumpService.ConfigHistoryClear configHistoryClear = dumpService.new ConfigHistoryClear(defaultHistoryConfigCleaner);
configHistoryClear.run();
Mockito.verify(defaultHistoryConfigCleaner, times(1)).cleanHistoryConfig();
}
@Test

58
config/src/test/java/com/alibaba/nacos/config/server/service/dump/HistoryConfigCleanerConfigTest.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import com.alibaba.nacos.sys.env.EnvUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@ExtendWith(SpringExtension.class)
class HistoryConfigCleanerConfigTest {
MockedStatic<EnvUtil> envUtilMockedStatic;
@BeforeEach
public void before() {
envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
}
@Test
public void test() {
envUtilMockedStatic.when(() -> EnvUtil.getProperty(anyString(), any(), anyString())).thenReturn("test");
HistoryConfigCleanerConfig historyConfigCleanerConfig = HistoryConfigCleanerConfig.getInstance();
historyConfigCleanerConfig.getConfigFromEnv();
assertEquals("test", historyConfigCleanerConfig.getActiveHistoryConfigCleaner());
envUtilMockedStatic.when(() -> EnvUtil.getProperty(anyString(), any(), anyString())).thenReturn(null);
historyConfigCleanerConfig.getConfigFromEnv();
assertEquals("nacos", historyConfigCleanerConfig.getActiveHistoryConfigCleaner());
}
@AfterEach
public void after() {
envUtilMockedStatic.close();
}
}

32
config/src/test/java/com/alibaba/nacos/config/server/service/dump/HistoryConfigCleanerManagerTest.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright 1999-2024 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.config.server.service.dump;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class HistoryConfigCleanerManagerTest {
@Test
public void testHistoryConfigCleanerManangerTest() {
HistoryConfigCleaner cleaner = HistoryConfigCleanerManager.getHistoryConfigCleaner(
HistoryConfigCleanerConfig.getInstance().getActiveHistoryConfigCleaner());
assertEquals(cleaner.getName(), "nacos");
}
}