From 1b55e68fd41972878ff8d823f7dfcea6a23eeb1f Mon Sep 17 00:00:00 2001 From: zhanglong Date: Sat, 13 Jun 2020 19:05:22 +0800 Subject: [PATCH] Upgrade the database driver version of Mysql to 8 # 3036 #2993 #1868 --- .../ExternalDataSourceProperties.java | 125 +++++++++++++++ .../ExternalDataSourceServiceImpl.java | 151 +++++------------- distribution/bin/startup.cmd | 2 +- distribution/bin/startup.sh | 2 +- pom.xml | 2 +- 5 files changed, 168 insertions(+), 114 deletions(-) create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceProperties.java diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceProperties.java b/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceProperties.java new file mode 100644 index 000000000..e0b7d3afa --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceProperties.java @@ -0,0 +1,125 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.alibaba.nacos.config.server.service.datasource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.collections.CollectionUtils; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.core.env.Environment; + +import com.google.common.base.Preconditions; +import com.zaxxer.hikari.HikariDataSource; + +/** + * Properties of external DataSource + * + * @author Nacos + */ +public class ExternalDataSourceProperties { + + private final static String JDBC_DRIVER_NAME = "com.mysql.cj.jdbc.Driver"; + public static final long CONNECTION_TIMEOUT_MS = 3000L; + public static final long VALIDATION_TIMEOUT = 10L; + public static final String TEST_QUERY = "SELECT 1 FROM dual"; + public static final int DEFAULT_MAX_POOL_SIZE = 20; + public static final int DEFAULT_MINIMUM_IDLE = 50; + + private Integer num; + private List url = new ArrayList<>(); + private List user = new ArrayList<>(); + private List password = new ArrayList<>(); + private List maxPoolSize = new ArrayList<>(); + private List minIdle = new ArrayList<>(); + + public void setNum(Integer num) { + this.num = num; + } + + public void setUrl(List url) { + this.url = url; + } + + public void setUser(List user) { + this.user = user; + } + + public void setPassword(List password) { + this.password = password; + } + + public void setMaxPoolSize(List maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } + + public void setMinIdle(List minIdle) { + this.minIdle = minIdle; + } + + /** + * + * @param environment + * {@link Environment} + * @param callback + * Callback function when constructing data source + * @return List of {@link HikariDataSource} + */ + List build(Environment environment, Callback callback) { + List dataSources = new ArrayList<>(); + Binder.get(environment).bind("db", Bindable.ofInstance(this)); + Preconditions.checkArgument(Objects.nonNull(num), "db.num is null"); + Preconditions.checkArgument(CollectionUtils.isNotEmpty(user), "db.user or db.user.[index] is null"); + Preconditions.checkArgument(CollectionUtils.isNotEmpty(password), "db.password or db.password.[index] is null"); + for (int index = 0; index < num; index++) { + int currentSize = index + 1; + Preconditions.checkArgument(url.size() >= currentSize, "db.url.%s is null", index); + HikariDataSource ds = new HikariDataSource(); + ds.setDriverClassName(JDBC_DRIVER_NAME); + ds.setJdbcUrl(url.get(index).trim()); + ds.setUsername(defaultIfNull(user, index, user.get(0)).trim()); + ds.setPassword(defaultIfNull(password, index, password.get(0)).trim()); + ds.setConnectionTimeout(CONNECTION_TIMEOUT_MS); + ds.setMaximumPoolSize(defaultIfNull(maxPoolSize, index, DEFAULT_MAX_POOL_SIZE)); + ds.setMinimumIdle(defaultIfNull(minIdle, index, DEFAULT_MINIMUM_IDLE)); + // Check the connection pool every 10 minutes + ds.setValidationTimeout(TimeUnit.MINUTES.toMillis(VALIDATION_TIMEOUT)); + ds.setConnectionTestQuery(TEST_QUERY); + dataSources.add(ds); + callback.accept(ds); + } + Preconditions.checkArgument(CollectionUtils.isNotEmpty(dataSources), "no datasource available"); + return dataSources; + } + + static T defaultIfNull(List collection, int index, T defaultValue) { + try { + return collection.get(index); + } catch (IndexOutOfBoundsException e) { + return defaultValue; + } + } + + interface Callback { + + /** + * Perform custom logic + * @param t + */ + void accept(T t); + } + +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceServiceImpl.java b/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceServiceImpl.java index 43d7da6ab..b289b8232 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceServiceImpl.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/datasource/ExternalDataSourceServiceImpl.java @@ -15,13 +15,19 @@ */ package com.alibaba.nacos.config.server.service.datasource; -import com.alibaba.nacos.common.utils.ConvertUtils; -import com.alibaba.nacos.common.utils.StringUtils; -import com.alibaba.nacos.config.server.monitor.MetricsMonitor; -import com.alibaba.nacos.config.server.utils.ConfigExecutor; -import com.alibaba.nacos.config.server.utils.PropertyUtil; -import com.alibaba.nacos.core.utils.ApplicationUtils; -import com.zaxxer.hikari.HikariDataSource; +import static com.alibaba.nacos.config.server.service.repository.RowMapperManager.CONFIG_INFO4BETA_ROW_MAPPER; +import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; +import static com.alibaba.nacos.config.server.utils.LogUtil.fatalLog; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.sql.DataSource; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DataAccessException; @@ -30,18 +36,13 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.transaction.support.TransactionTemplate; -import javax.annotation.PostConstruct; -import javax.sql.DataSource; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.alibaba.nacos.config.server.service.repository.RowMapperManager.CONFIG_INFO4BETA_ROW_MAPPER; -import static com.alibaba.nacos.config.server.utils.LogUtil.defaultLog; -import static com.alibaba.nacos.config.server.utils.LogUtil.fatalLog; +import com.alibaba.nacos.common.utils.ConvertUtils; +import com.alibaba.nacos.common.utils.StringUtils; +import com.alibaba.nacos.config.server.monitor.MetricsMonitor; +import com.alibaba.nacos.config.server.utils.ConfigExecutor; +import com.alibaba.nacos.config.server.utils.PropertyUtil; +import com.alibaba.nacos.core.utils.ApplicationUtils; +import com.zaxxer.hikari.HikariDataSource; /** * Base data source @@ -52,9 +53,7 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { private static final Logger log = LoggerFactory.getLogger( ExternalDataSourceServiceImpl.class); - private static final String DEFAULT_MYSQL_DRIVER = "com.mysql.jdbc.Driver"; - private static final String MYSQL_HIGH_LEVEL_DRIVER = "com.mysql.cj.jdbc.Driver"; - private static String JDBC_DRIVER_NAME; + private final static String JDBC_DRIVER_NAME="com.mysql.cj.jdbc.Driver"; /** * JDBC执行超时时间, 单位秒 @@ -78,24 +77,13 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { private volatile int masterIndex; private static Pattern ipPattern = Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"); - static { - try { - Class.forName(MYSQL_HIGH_LEVEL_DRIVER); - JDBC_DRIVER_NAME = MYSQL_HIGH_LEVEL_DRIVER; - log.info("Use Mysql 8 as the driver"); - } catch (ClassNotFoundException e) { - log.info("Use Mysql as the driver"); - JDBC_DRIVER_NAME = DEFAULT_MYSQL_DRIVER; - } - } - @PostConstruct + + @Override public void init() { queryTimeout = ConvertUtils.toInt(System.getProperty("QUERYTIMEOUT"), 3); jt = new JdbcTemplate(); - /** - * 设置最大记录数，防止内存膨胀 - */ + // Set the maximum number of records to prevent memory expansion jt.setMaxRows(50000); jt.setQueryTimeout(queryTimeout); @@ -103,20 +91,18 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { testMasterJT.setQueryTimeout(queryTimeout); testMasterWritableJT = new JdbcTemplate(); - /** - * 防止login接口因为主库不可用而rt太长 - */ + // Prevent the login interface from being too long because the main library is not available testMasterWritableJT.setQueryTimeout(1); - /** - * 数据库健康检测 - */ + + // Database health check + testJTList = new ArrayList(); isHealthList = new ArrayList(); tm = new DataSourceTransactionManager(); tjt = new TransactionTemplate(tm); /** - * 事务的超时时间需要与普通操作区分开 + * Transaction timeout needs to be distinguished from ordinary operations */ tjt.setTimeout(TRANSACTION_QUERY_TIMEOUT); if (PropertyUtil.isUseExternalDB()) { @@ -136,73 +122,20 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { @Override public synchronized void reload() throws IOException { - List dblist = new ArrayList<>(); try { - String val = null; - val = ApplicationUtils.getProperty("db.num"); - if (null == val) { - throw new IllegalArgumentException("db.num is null"); - } - int dbNum = Integer.parseInt(val.trim()); - - for (int i = 0; i < dbNum; i++) { - HikariDataSource ds = new HikariDataSource(); - ds.setDriverClassName(JDBC_DRIVER_NAME); - - val = ApplicationUtils.getProperty("db.url." + i); - if (null == val) { - fatalLog.error("db.url." + i + " is null"); - throw new IllegalArgumentException("db.url." + i + " is null"); - } - ds.setJdbcUrl(val.trim()); - - val = ApplicationUtils.getProperty("db.user." + i, ApplicationUtils.getProperty("db.user")); - if (null == val) { - fatalLog.error("db.user." + i + " is null"); - throw new IllegalArgumentException("db.user." + i + " is null"); - } - ds.setUsername(val.trim()); - - val = ApplicationUtils.getProperty("db.password." + i, ApplicationUtils.getProperty("db.password")); - if (null == val) { - fatalLog.error("db.password." + i + " is null"); - throw new IllegalArgumentException("db.password." + i + " is null"); - } - ds.setPassword(val.trim()); - - val = ApplicationUtils.getProperty("db.maxPoolSize." + i, ApplicationUtils.getProperty("db.maxPoolSize")); - ds.setMaximumPoolSize(Integer.parseInt(defaultIfNull(val, "20"))); - - val = ApplicationUtils.getProperty("db.minIdle." + i, ApplicationUtils.getProperty("db.minIdle")); - ds.setMinimumIdle(Integer.parseInt(defaultIfNull(val, "50"))); - - ds.setConnectionTimeout(3000L); - - // 每10分钟检查一遍连接池 - ds.setValidationTimeout(TimeUnit.MINUTES.toMillis(10L)); - ds.setConnectionTestQuery("SELECT 1 FROM dual"); - - dblist.add(ds); - - JdbcTemplate jdbcTemplate = new JdbcTemplate(); - jdbcTemplate.setQueryTimeout(queryTimeout); - jdbcTemplate.setDataSource(ds); - - testJTList.add(jdbcTemplate); - isHealthList.add(Boolean.TRUE); - } - - if (dblist == null || dblist.size() == 0) { - throw new RuntimeException("no datasource available"); - } - - dataSourceList = dblist; + dataSourceList = new ExternalDataSourceProperties() + .build(ApplicationUtils.getEnvironment(), (dataSource) -> { + JdbcTemplate jdbcTemplate = new JdbcTemplate(); + jdbcTemplate.setQueryTimeout(queryTimeout); + jdbcTemplate.setDataSource(dataSource); + testJTList.add(jdbcTemplate); + isHealthList.add(Boolean.TRUE); + }); new SelectMasterTask().run(); new CheckDBHealthTask().run(); } catch (RuntimeException e) { fatalLog.error(DB_LOAD_ERROR_MSG, e); throw new IOException(e); - } finally { } } @@ -210,9 +143,7 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { public boolean checkMasterWritable() { testMasterWritableJT.setDataSource(jt.getDataSource()); - /** - * 防止login接口因为主库不可用而rt太长 - */ + // Prevent the login interface from being too long because the main library is not available testMasterWritableJT.setQueryTimeout(1); String sql = " SELECT @@read_only "; @@ -255,13 +186,11 @@ public class ExternalDataSourceServiceImpl implements DataSourceService { for (int i = 0; i < isHealthList.size(); i++) { if (!isHealthList.get(i)) { if (i == masterIndex) { - /** - * 主库不健康 - */ + // The master is unhealthy return "DOWN:" + getIpFromUrl(dataSourceList.get(i).getJdbcUrl()); } else { /** - * 从库不健康 + * The slave is unhealthy */ return "WARN:" + getIpFromUrl(dataSourceList.get(i).getJdbcUrl()); } diff --git a/distribution/bin/startup.cmd b/distribution/bin/startup.cmd index 8e79e6dd6..aad58e01e 100755 --- a/distribution/bin/startup.cmd +++ b/distribution/bin/startup.cmd @@ -64,7 +64,7 @@ if %FUNCTION_MODE% == "naming" ( set "JAVA_OPT=%JAVA_OPT% -Dnacos.functionMode=naming" ) -set "JAVA_OPT=%JAVA_OPT% -Dloader.path=%BASE_DIR%/plugins/health,%BASE_DIR%/plugins/cmdb,%BASE_DIR%/plugins/mysql" +set "JAVA_OPT=%JAVA_OPT% -Dloader.path=%BASE_DIR%/plugins/health,%BASE_DIR%/plugins/cmdb" set "JAVA_OPT=%JAVA_OPT% -Dnacos.home=%BASE_DIR%" set "JAVA_OPT=%JAVA_OPT% -jar %BASE_DIR%\target\%SERVER%.jar" diff --git a/distribution/bin/startup.sh b/distribution/bin/startup.sh index 558442f8c..52b345c1b 100644 --- a/distribution/bin/startup.sh +++ b/distribution/bin/startup.sh @@ -113,7 +113,7 @@ else JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M" fi -JAVA_OPT="${JAVA_OPT} -Dloader.path=${BASE_DIR}/plugins/health,${BASE_DIR}/plugins/cmdb,${BASE_DIR}/plugins/mysql" +JAVA_OPT="${JAVA_OPT} -Dloader.path=${BASE_DIR}/plugins/health,${BASE_DIR}/plugins/cmdb" JAVA_OPT="${JAVA_OPT} -Dnacos.home=${BASE_DIR}" JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/${SERVER}.jar" JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" diff --git a/pom.xml b/pom.xml index f6e7f98f2..7f17159c8 100644 --- a/pom.xml +++ b/pom.xml @@ -141,7 +141,7 @@ 4.5 4.1.3 1.7.17 - 5.1.34 + 8.0.13 10.14.2.0 2.1 1.0