Support setcpuaffinity on linux/bsd

Currently, there are several types of threads/child processes of a
redis server. Sometimes we need deeply optimise the performance of
redis, so we would like to isolate threads/processes.

There were some discussion about cpu affinity cases in the issue:
https://github.com/antirez/redis/issues/2863

So implement cpu affinity setting by redis.conf in this patch, then
we can config server_cpulist/bio_cpulist/aof_rewrite_cpulist/
bgsave_cpulist by cpu list.

Examples of cpulist in redis.conf:
server_cpulist 0-7:2      means cpu affinity 0,2,4,6
bio_cpulist 1,3           means cpu affinity 1,3
aof_rewrite_cpulist 8-11  means cpu affinity 8,9,10,11
bgsave_cpulist 1,10-11    means cpu affinity 1,10,11

Test on linux/freebsd, both work fine.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
This commit is contained in:
zhenwei pi 2020-05-02 20:05:39 +08:00
parent 365316aa59
commit 1a0deab2a5
12 changed files with 180 additions and 1 deletions

View File

@ -1780,3 +1780,18 @@ rdb-save-incremental-fsync yes
# Maximum number of set/hash/zset/list fields that will be processed from
# the main dictionary scan
# active-defrag-max-scan-fields 1000
# Redis server/IO threads, bio threads, aof rewrite child process, and bgsave
# child process cpu affinity list config. syntax of cpu list looks like taskset
# command. serveral examples:
# set redis server/io threads to cpu affinity 0,2,4,6
# server_cpulist 0-7:2
#
# set bio threads to cpu affinity 1,3
# bio_cpulist 1,3
#
# set aof rewrite child process to cpu affinity 8,9,10,11
# aof_rewrite_cpulist 8-11
#
# set bgsave child process to cpu affinity 1,10,11
# bgsave_cpulist 1,10-11

View File

@ -206,7 +206,7 @@ endif
REDIS_SERVER_NAME=redis-server
REDIS_SENTINEL_NAME=redis-sentinel
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o
REDIS_CLI_NAME=redis-cli
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o
REDIS_BENCHMARK_NAME=redis-benchmark

View File

@ -1596,6 +1596,7 @@ int rewriteAppendOnlyFileBackground(void) {
/* Child */
redisSetProcTitle("redis-aof-rewrite");
redisSetCpuAffinity(server.aof_rewrite_cpulist);
snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite");

View File

@ -166,6 +166,8 @@ void *bioProcessBackgroundJobs(void *arg) {
break;
}
redisSetCpuAffinity(server.bio_cpulist);
/* Make the thread killable at any time, so that bioKillThreads()
* can work reliably. */
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);

View File

@ -2133,6 +2133,10 @@ standardConfig configs[] = {
createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
createStringConfig("server_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.server_cpulist, NULL, NULL, NULL),
createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL),
createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL),
createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL),
/* Enum Configs */
createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL),

View File

@ -244,4 +244,10 @@ int pthread_setname_np(const char *name);
#endif
#endif
/* Check if we can use setcpuaffinity(). */
#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__)
#define USE_SETCPUAFFINITY
void setcpuaffinity(const char *cpulist);
#endif
#endif

View File

@ -2843,6 +2843,7 @@ void *IOThreadMain(void *myid) {
snprintf(thdname, sizeof(thdname), "io_thd_%ld", id);
redis_set_thread_title(thdname);
redisSetCpuAffinity(server.server_cpulist);
while(1) {
/* Wait for start */

View File

@ -1351,6 +1351,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
/* Child */
redisSetProcTitle("redis-rdb-bgsave");
redisSetCpuAffinity(server.bgsave_cpulist);
retval = rdbSave(filename,rsi);
if (retval == C_OK) {
sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
@ -2458,6 +2459,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
rioInitWithFd(&rdb,server.rdb_pipe_write);
redisSetProcTitle("redis-rdb-to-slaves");
redisSetCpuAffinity(server.bgsave_cpulist);
retval = rdbSaveRioWithEOFMark(&rdb,NULL,rsi);
if (retval == C_OK && rioFlush(&rdb) == 0)

View File

@ -4850,6 +4850,14 @@ void redisSetProcTitle(char *title) {
#endif
}
void redisSetCpuAffinity(const char *cpulist) {
#ifdef USE_SETCPUAFFINITY
setcpuaffinity(cpulist);
#else
UNUSED(cpulist);
#endif
}
/*
* Check whether systemd or upstart have been used to start redis.
*/
@ -5118,6 +5126,7 @@ int main(int argc, char **argv) {
serverLog(LL_WARNING,"WARNING: You specified a maxmemory value that is less than 1MB (current value is %llu bytes). Are you sure this is what you really want?", server.maxmemory);
}
redisSetCpuAffinity(server.server_cpulist);
aeSetBeforeSleepProc(server.el,beforeSleep);
aeSetAfterSleepProc(server.el,afterSleep);
aeMain(server.el);

View File

@ -1433,6 +1433,11 @@ struct redisServer {
int tls_replication;
int tls_auth_clients;
redisTLSContextConfig tls_ctx_config;
/* cpu affinity */
char *server_cpulist; /* cpu affinity list of redis server main/io thread. */
char *bio_cpulist; /* cpu affinity list of bio thread. */
char *aof_rewrite_cpulist; /* cpu affinity list of aof rewrite process. */
char *bgsave_cpulist; /* cpu affinity list of bgsave process. */
};
typedef struct pubsubPattern {
@ -1585,6 +1590,7 @@ void exitFromChild(int retcode);
size_t redisPopcount(void *s, long count);
void redisSetProcTitle(char *title);
int redisCommunicateSystemd(const char *sd_notify_msg);
void redisSetCpuAffinity(const char *cpulist);
/* networking.c -- Networking and Client related operations */
client *createClient(connection *conn);

129
src/setcpuaffinity.c Normal file
View File

@ -0,0 +1,129 @@
/* ==========================================================================
* setproctitle.c - Linux/BSD setcpuaffinity.
* --------------------------------------------------------------------------
* Copyright (C) 2020 zhenwei pi
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to permit
* persons to whom the Software is furnished to do so, subject to the
* following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
* ==========================================================================
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#ifdef __linux__
#include <sched.h>
#endif
#ifdef __FreeBSD__
#include <sys/param.h>
#include <sys/cpuset.h>
#endif
#include "config.h"
#ifdef USE_SETCPUAFFINITY
static const char *next_token(const char *q, int sep) {
if (q)
q = strchr(q, sep);
if (q)
q++;
return q;
}
static int next_num(const char *str, char **end, int *result) {
if (!str || *str == '\0' || !isdigit(*str))
return -1;
*result = strtoul(str, end, 10);
if (str == *end)
return -1;
return 0;
}
/* set current thread cpu affinity to cpu list, this function works like
* taskset command (actually cpulist parsing logic reference to util-linux).
* example of this function: "0,2,3", "0,2-3", "0-20:2". */
void setcpuaffinity(const char *cpulist) {
const char *p, *q;
char *end = NULL;
#ifdef __linux__
cpu_set_t cpuset;
#endif
#ifdef __FreeBSD__
cpuset_t cpuset;
#endif
if (!cpulist)
return;
CPU_ZERO(&cpuset);
q = cpulist;
while (p = q, q = next_token(q, ','), p) {
int a, b, s;
const char *c1, *c2;
if (next_num(p, &end, &a) != 0)
return;
b = a;
s = 1;
p = end;
c1 = next_token(p, '-');
c2 = next_token(p, ',');
if (c1 != NULL && (c2 == NULL || c1 < c2)) {
if (next_num(c1, &end, &b) != 0)
return;
c1 = end && *end ? next_token(end, ':') : NULL;
if (c1 != NULL && (c2 == NULL || c1 < c2)) {
if (next_num(c1, &end, &s) != 0)
return;
if (s == 0)
return;
}
}
if ((a > b))
return;
while (a <= b) {
CPU_SET(a, &cpuset);
a += s;
}
}
if (end && *end)
return;
#ifdef __linux__
sched_setaffinity(0, sizeof(cpuset), &cpuset);
#endif
#ifdef __FreeBSD__
cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
#endif
}
#endif /* USE_SETCPUAFFINITY */

View File

@ -94,6 +94,10 @@ start_server {tags {"introspection"}} {
slaveof
bind
requirepass
server_cpulist
bio_cpulist
aof_rewrite_cpulist
bgsave_cpulist
}
set configs {}