zhuyijun/nacos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use ThreadUtil and EnvUtil replace Runtime.availableProcessor (#5388)

Browse Source
This commit is contained in:
杨翊 SionYang 2021-04-16 17:32:14 +08:00 committed by GitHub
parent ec2528f475
commit 73eea9c0bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 246 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
client/src/main/java/com/alibaba/nacos/client/config/impl/ClientWorker.java
View File

@ -77,7 +77,6 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@ -423,14 +422,11 @@ public class ClientWorker implements Closeable {
agent = new ConfigRpcTransportClient(properties, serverListManager);
ScheduledExecutorService executorService = Executors
.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setName("com.alibaba.nacos.client.Worker");
t.setDaemon(true);
return t;
}
.newScheduledThreadPool(ThreadUtils.getSuitableThreadCount(1), r -> {
Thread t = new Thread(r);
t.setName("com.alibaba.nacos.client.Worker");
t.setDaemon(true);
return t;
});
agent.setExecutor(executorService);
agent.start();

5
client/src/main/java/com/alibaba/nacos/client/naming/utils/UtilAndComs.java
View File

@ -16,6 +16,7 @@
package com.alibaba.nacos.client.naming.utils;
import com.alibaba.nacos.common.utils.ThreadUtils;
import com.alibaba.nacos.common.utils.VersionUtils;
/**
@ -54,10 +55,10 @@ public class UtilAndComs {
public static final String NACOS_NAMING_LOG_LEVEL = "com.alibaba.nacos.naming.log.level";
public static final int DEFAULT_CLIENT_BEAT_THREAD_COUNT =
Runtime.getRuntime().availableProcessors() > 1 ? Runtime.getRuntime().availableProcessors() / 2 : 1;
ThreadUtils.getSuitableThreadCount(1) > 1 ? ThreadUtils.getSuitableThreadCount(1) / 2 : 1;
public static final int DEFAULT_POLLING_THREAD_COUNT =
Runtime.getRuntime().availableProcessors() > 1 ? Runtime.getRuntime().availableProcessors() / 2 : 1;
ThreadUtils.getSuitableThreadCount(1) > 1 ? ThreadUtils.getSuitableThreadCount(1) / 2 : 1;
public static final String HTTP = "http://";

4
cmdb/src/main/java/com/alibaba/nacos/cmdb/utils/CmdbExecutor.java
View File

@ -20,6 +20,7 @@ import com.alibaba.nacos.cmdb.CmdbApp;
import com.alibaba.nacos.common.executor.ExecutorFactory;
import com.alibaba.nacos.common.executor.NameThreadFactory;
import com.alibaba.nacos.core.utils.ClassUtils;
import com.alibaba.nacos.sys.env.EnvUtil;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -33,8 +34,7 @@ import java.util.concurrent.TimeUnit;
public class CmdbExecutor {
private static final ScheduledExecutorService GLOBAL_EXECUTOR = ExecutorFactory.Managed
.newScheduledExecutorService(ClassUtils.getCanonicalName(CmdbApp.class),
Runtime.getRuntime().availableProcessors(),
.newScheduledExecutorService(ClassUtils.getCanonicalName(CmdbApp.class), EnvUtil.getAvailableProcessors(),
new NameThreadFactory("com.alibaba.nacos.cmdb.global.executor"));
public static void scheduleCmdbTask(Runnable runnable, long delay, TimeUnit unit) {

4
common/src/main/java/com/alibaba/nacos/common/http/HttpClientConfig.java
View File

@ -16,6 +16,8 @@
package com.alibaba.nacos.common.http;
import com.alibaba.nacos.common.utils.ThreadUtils;
import java.util.concurrent.TimeUnit;
/**
@ -164,7 +166,7 @@ public class HttpClientConfig {
private boolean contentCompressionEnabled = true;
private int ioThreadCount = Runtime.getRuntime().availableProcessors();
private int ioThreadCount = ThreadUtils.getSuitableThreadCount(1);
private String userAgent;

6
common/src/main/java/com/alibaba/nacos/common/remote/client/grpc/GrpcClient.java
View File

@ -31,6 +31,7 @@ import com.alibaba.nacos.common.remote.client.Connection;
import com.alibaba.nacos.common.remote.client.RpcClient;
import com.alibaba.nacos.common.remote.client.RpcClientStatus;
import com.alibaba.nacos.common.utils.LoggerUtils;
import com.alibaba.nacos.common.utils.ThreadUtils;
import com.alibaba.nacos.common.utils.VersionUtils;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@ -245,9 +246,8 @@ public abstract class GrpcClient extends RpcClient {
public Connection connectToServer(ServerInfo serverInfo) {
try {
if (grpcExecutor == null) {
grpcExecutor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 8,
Runtime.getRuntime().availableProcessors() * 8, 10L, TimeUnit.SECONDS,
int threadNumber = ThreadUtils.getSuitableThreadCount(8);
grpcExecutor = new ThreadPoolExecutor(threadNumber, threadNumber, 10L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(10000),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("nacos-grpc-client-executor-%d")
.build());

16
core/src/main/java/com/alibaba/nacos/core/utils/GlobalExecutor.java
View File

@ -18,6 +18,7 @@ package com.alibaba.nacos.core.utils;
import com.alibaba.nacos.common.executor.ExecutorFactory;
import com.alibaba.nacos.common.executor.NameThreadFactory;
import com.alibaba.nacos.sys.env.EnvUtil;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.LinkedBlockingQueue;
@ -39,19 +40,18 @@ public class GlobalExecutor {
private static final ScheduledExecutorService DISTRO_EXECUTOR = ExecutorFactory.Managed
.newScheduledExecutorService(ClassUtils.getCanonicalName(GlobalExecutor.class),
Runtime.getRuntime().availableProcessors() * 2,
new NameThreadFactory("com.alibaba.nacos.core.protocal.distro"));
EnvUtil.getAvailableProcessors(2), new NameThreadFactory("com.alibaba.nacos.core.protocal.distro"));
public static final ThreadPoolExecutor sdkRpcExecutor = new ThreadPoolExecutor(
Runtime.getRuntime().availableProcessors() * RemoteUtils.getRemoteExecutorTimesOfProcessors(),
Runtime.getRuntime().availableProcessors() * RemoteUtils.getRemoteExecutorTimesOfProcessors(), 60L,
TimeUnit.SECONDS, new LinkedBlockingQueue<>(RemoteUtils.getRemoteExecutorQueueSize()),
EnvUtil.getAvailableProcessors(RemoteUtils.getRemoteExecutorTimesOfProcessors()),
EnvUtil.getAvailableProcessors(RemoteUtils.getRemoteExecutorTimesOfProcessors()), 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(RemoteUtils.getRemoteExecutorQueueSize()),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("nacos-grpc-executor-%d").build());
public static final ThreadPoolExecutor clusterRpcExecutor = new ThreadPoolExecutor(
Runtime.getRuntime().availableProcessors() * RemoteUtils.getRemoteExecutorTimesOfProcessors(),
Runtime.getRuntime().availableProcessors() * RemoteUtils.getRemoteExecutorTimesOfProcessors(), 60L,
TimeUnit.SECONDS, new LinkedBlockingQueue<>(RemoteUtils.getRemoteExecutorQueueSize()),
EnvUtil.getAvailableProcessors(RemoteUtils.getRemoteExecutorTimesOfProcessors()),
EnvUtil.getAvailableProcessors(RemoteUtils.getRemoteExecutorTimesOfProcessors()), 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(RemoteUtils.getRemoteExecutorQueueSize()),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("nacos-cluster-grpc-executor-%d").build());
public static void runWithoutThread(Runnable runnable) {

2
naming/src/main/java/com/alibaba/nacos/naming/cluster/ServerListManager.java
View File

@ -196,7 +196,7 @@ public class ServerListManager extends MemberChangeListener {
return;
}
int weight = Runtime.getRuntime().availableProcessors() / 2;
int weight = EnvUtil.getAvailableProcessors(0.5);
if (weight <= 0) {
weight = 1;
}

4
naming/src/main/java/com/alibaba/nacos/naming/healthcheck/TcpSuperSenseProcessor.java
View File

@ -22,6 +22,7 @@ import com.alibaba.nacos.naming.misc.GlobalExecutor;
import com.alibaba.nacos.naming.misc.Loggers;
import com.alibaba.nacos.naming.misc.SwitchDomain;
import com.alibaba.nacos.naming.monitor.MetricsMonitor;
import com.alibaba.nacos.sys.env.EnvUtil;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -73,8 +74,7 @@ public class TcpSuperSenseProcessor implements HealthCheckProcessor, Runnable {
/**
* this value has been carefully tuned, do not modify unless you're confident.
*/
private static final int NIO_THREAD_COUNT =
Runtime.getRuntime().availableProcessors() <= 1 ? 1 : Runtime.getRuntime().availableProcessors() / 2;
private static final int NIO_THREAD_COUNT = EnvUtil.getAvailableProcessors(0.5);
/**
* because some hosts doesn't support keep-alive connections, disabled temporarily.

7
naming/src/main/java/com/alibaba/nacos/naming/healthcheck/v2/processor/TcpHealthCheckProcessor.java
View File

@ -25,6 +25,7 @@ import com.alibaba.nacos.naming.misc.GlobalExecutor;
import com.alibaba.nacos.naming.misc.Loggers;
import com.alibaba.nacos.naming.misc.SwitchDomain;
import com.alibaba.nacos.naming.monitor.MetricsMonitor;
import com.alibaba.nacos.sys.env.EnvUtil;
import org.springframework.stereotype.Component;
import java.net.ConnectException;
@ -63,8 +64,7 @@ public class TcpHealthCheckProcessor implements HealthCheckProcessorV2, Runnable
/**
* this value has been carefully tuned, do not modify unless you're confident.
*/
private static final int NIO_THREAD_COUNT =
Runtime.getRuntime().availableProcessors() <= 1 ? 1 : Runtime.getRuntime().availableProcessors() / 2;
private static final int NIO_THREAD_COUNT = EnvUtil.getAvailableProcessors(0.5);
/**
* because some hosts doesn't support keep-alive connections, disabled temporarily.
@ -193,7 +193,8 @@ public class TcpHealthCheckProcessor implements HealthCheckProcessorV2, Runnable
key.channel().close();
} else {
// not terminate request, ignore
SRV_LOG.warn("Tcp check ok, but the connected server responses some msg. Connection won't be closed.");
SRV_LOG.warn(
"Tcp check ok, but the connected server responses some msg. Connection won't be closed.");
}
}
} catch (ConnectException e) {

7
naming/src/main/java/com/alibaba/nacos/naming/misc/GlobalExecutor.java
View File

@ -20,6 +20,7 @@ import com.alibaba.nacos.common.executor.ExecutorFactory;
import com.alibaba.nacos.common.executor.NameThreadFactory;
import com.alibaba.nacos.core.utils.ClassUtils;
import com.alibaba.nacos.naming.NamingApp;
import com.alibaba.nacos.sys.env.EnvUtil;
import java.util.Collection;
import java.util.List;
@ -48,13 +49,11 @@ public class GlobalExecutor {
private static final long SERVER_STATUS_UPDATE_PERIOD = TimeUnit.SECONDS.toMillis(5);
public static final int DEFAULT_THREAD_COUNT =
Runtime.getRuntime().availableProcessors() <= 1 ? 1 : Runtime.getRuntime().availableProcessors() / 2;
public static final int DEFAULT_THREAD_COUNT = EnvUtil.getAvailableProcessors(0.5);
private static final ScheduledExecutorService NAMING_TIMER_EXECUTOR = ExecutorFactory.Managed
.newScheduledExecutorService(ClassUtils.getCanonicalName(NamingApp.class),
Runtime.getRuntime().availableProcessors() * 2,
new NameThreadFactory("com.alibaba.nacos.naming.timer"));
EnvUtil.getAvailableProcessors(2), new NameThreadFactory("com.alibaba.nacos.naming.timer"));
private static final ScheduledExecutorService SERVER_STATUS_EXECUTOR = ExecutorFactory.Managed
.newSingleScheduledExecutorService(ClassUtils.getCanonicalName(NamingApp.class),

5
naming/src/main/java/com/alibaba/nacos/naming/misc/HttpClientManager.java
View File

@ -25,6 +25,7 @@ import com.alibaba.nacos.common.http.client.NacosAsyncRestTemplate;
import com.alibaba.nacos.common.http.client.NacosRestTemplate;
import com.alibaba.nacos.common.utils.ExceptionUtil;
import com.alibaba.nacos.common.utils.ThreadUtils;
import com.alibaba.nacos.sys.env.EnvUtil;
import org.slf4j.Logger;
import java.util.concurrent.TimeUnit;
@ -149,8 +150,8 @@ public class HttpClientManager {
@Override
protected HttpClientConfig buildHttpClientConfig() {
return HttpClientConfig.builder().setConnectionTimeToLive(500, TimeUnit.MILLISECONDS)
.setMaxConnTotal(Runtime.getRuntime().availableProcessors() * 2)
.setMaxConnPerRoute(Runtime.getRuntime().availableProcessors()).setMaxRedirects(0).build();
.setMaxConnTotal(EnvUtil.getAvailableProcessors(2))
.setMaxConnPerRoute(EnvUtil.getAvailableProcessors()).setMaxRedirects(0).build();
}
@Override

2
naming/src/main/java/com/alibaba/nacos/naming/misc/UtilsAndCommons.java
View File

@ -36,7 +36,7 @@ import java.util.Map;
* @author nacos
* @author jifengnan
*/
@SuppressWarnings("PMD.ThreadPoolCreationRule")
@SuppressWarnings("PMD.ThreadPoolCreationle")
public class UtilsAndCommons {
// ********************** Nacos HTTP Context ************************ \\

2
sys/src/main/java/com/alibaba/nacos/sys/env/Constants.java vendored
View File

@ -63,4 +63,6 @@ public interface Constants {
String NACOS_SERVER_HEADER = "Nacos-Server";
String REQUEST_PATH_SEPARATOR = "-->";
String AVAILABLE_PROCESSORS_BASIC = "nacos.core.sys.basic.processors";
}

45
sys/src/main/java/com/alibaba/nacos/sys/env/EnvUtil.java vendored
View File

@ -20,6 +20,7 @@ import com.alibaba.nacos.common.JustForTest;
import com.alibaba.nacos.common.utils.IoUtils;
import com.alibaba.nacos.common.utils.Objects;
import com.alibaba.nacos.common.utils.StringUtils;
import com.alibaba.nacos.common.utils.ThreadUtils;
import com.alibaba.nacos.sys.utils.DiskUtils;
import com.alibaba.nacos.sys.utils.InetUtils;
import com.sun.management.OperatingSystemMXBean;
@ -390,4 +391,48 @@ public class EnvUtil {
return new InputStreamResource(inputStream);
}
/**
* Get available processor numbers from environment.
*
* <p>
* If there are setting of {@code nacos.core.sys.basic.processors} in config/JVM/system, use it.
* If no setting, use the one time {@code ThreadUtils.getSuitableThreadCount()}.
* </p>
*
* @return available processor numbers from environment, will not lower than 1.
*/
public static int getAvailableProcessors() {
int result = getProperty(Constants.AVAILABLE_PROCESSORS_BASIC, int.class,
ThreadUtils.getSuitableThreadCount(1));
return result > 0 ? result : 1;
}
/**
* Get a multiple time of available processor numbers from environment.
*
* @param multiple multiple of available processor numbers
* @return available processor numbers from environment, will not lower than 1.
*/
public static int getAvailableProcessors(int multiple) {
if (multiple < 1) {
throw new IllegalArgumentException("processors multiple must upper than 1");
}
Integer processor = getProperty(Constants.AVAILABLE_PROCESSORS_BASIC, Integer.class);
return null != processor && processor > 0 ? processor * multiple : ThreadUtils.getSuitableThreadCount(multiple);
}
/**
* Get a scale of available processor numbers from environment.
*
* @param scale scale from 0 to 1.
* @return available processor numbers from environment, will not lower than 1.
*/
public static int getAvailableProcessors(double scale) {
if (scale < 0 || scale > 1) {
throw new IllegalArgumentException("processors scale must between 0 and 1");
}
double result = getProperty(Constants.AVAILABLE_PROCESSORS_BASIC, int.class,
ThreadUtils.getSuitableThreadCount(1)) * scale;
return result > 1 ? (int) result : 1;
}
}

63
sys/src/test/java/com/alibaba/nacos/sys/env/EnvUtilWithConfigTest.java vendored Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright 1999-2020 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.sys.env;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import static org.junit.Assert.assertEquals;
@RunWith(SpringRunner.class)
@ActiveProfiles("test")
@SpringBootTest(classes = EnvUtilWithConfigTest.class)
public class EnvUtilWithConfigTest {
private static final int SETTING_PROCESSORS = 10;
@Autowired
private Environment environment;
@Before
public void setUp() {
EnvUtil.setEnvironment((ConfigurableEnvironment) environment);
}
@Test
public void testGetAvailableProcessors() {
int actual = EnvUtil.getAvailableProcessors();
assertEquals(SETTING_PROCESSORS, actual);
}
@Test
public void testGetAvailableProcessorsWithMultiple() {
int actual = EnvUtil.getAvailableProcessors(2);
assertEquals(SETTING_PROCESSORS * 2, actual);
}
@Test
public void testGetAvailableProcessorsWithScale() {
int actual = EnvUtil.getAvailableProcessors(0.5);
assertEquals((int) (SETTING_PROCESSORS * 0.5), actual);
}
}

65
sys/src/test/java/com/alibaba/nacos/sys/env/EnvUtilWithoutConfigTest.java vendored Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright 1999-2020 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.nacos.sys.env;
import com.alibaba.nacos.common.utils.ThreadUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.Environment;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import static org.junit.Assert.assertEquals;
@RunWith(SpringRunner.class)
@ActiveProfiles("empty")
@SpringBootTest(classes = EnvUtilWithConfigTest.class)
public class EnvUtilWithoutConfigTest {
@Autowired
private Environment environment;
@Before
public void setUp() {
EnvUtil.setEnvironment((ConfigurableEnvironment) environment);
}
@Test
public void testGetAvailableProcessors() {
int expected = ThreadUtils.getSuitableThreadCount(1);
int actual = EnvUtil.getAvailableProcessors();
assertEquals(expected, actual);
}
@Test
public void testGetAvailableProcessorsWithMultiple() {
int expected = ThreadUtils.getSuitableThreadCount(2);
int actual = EnvUtil.getAvailableProcessors(2);
assertEquals(expected, actual);
}
@Test
public void testGetAvailableProcessorsWithScale() {
int expected = ThreadUtils.getSuitableThreadCount(1);
int actual = EnvUtil.getAvailableProcessors(0.5);
assertEquals((int) (expected * 0.5), actual);
}
}

15
sys/src/test/resources/application-empty.properties Normal file
View File

@ -0,0 +1,15 @@
#
# Copyright 1999-2018 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

17
sys/src/test/resources/application-test.properties Normal file
View File

@ -0,0 +1,17 @@
#
# Copyright 1999-2018 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
nacos.core.sys.basic.processors=10

2
sys/src/test/resources/application.properties
View File

@ -14,4 +14,4 @@
# limitations under the License.
#
name=test-1
name=test-1