diff --git a/redis.conf b/redis.conf index e7a01eec..87d34eef 100644 --- a/redis.conf +++ b/redis.conf @@ -292,6 +292,26 @@ appendfsync everysec # "no" that is the safest pick from the point of view of durability. no-appendfsync-on-rewrite no +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size will growth by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (or if no rewrite happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a precentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + #################################### DISK STORE ############################### # When disk store is active Redis works as an on-disk database, where memory diff --git a/src/aof.c b/src/aof.c index cd409a0b..78387cd3 100644 --- a/src/aof.c +++ b/src/aof.c @@ -8,6 +8,8 @@ #include #include +void aofUpdateCurrentSize(void); + /* Called when the user switches from "appendonly yes" to "appendonly no" * at runtime using the CONFIG command. */ void stopAppendOnly(void) { @@ -19,15 +21,15 @@ void stopAppendOnly(void) { server.appendseldb = -1; server.appendonly = 0; /* rewrite operation in progress? kill it, wait child exit */ - if (server.bgsavechildpid != -1) { + if (server.bgrewritechildpid != -1) { int statloc; - if (kill(server.bgsavechildpid,SIGKILL) != -1) + if (kill(server.bgrewritechildpid,SIGKILL) != -1) wait3(&statloc,0,NULL); /* reset the buffer accumulating changes while the child saves */ sdsfree(server.bgrewritebuf); server.bgrewritebuf = sdsempty(); - server.bgsavechildpid = -1; + server.bgrewritechildpid = -1; } } @@ -82,6 +84,7 @@ void flushAppendOnlyFile(void) { } sdsfree(server.aofbuf); server.aofbuf = sdsempty(); + server.appendonly_current_size += nwritten; /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have * childs performing heavy I/O on disk. */ @@ -221,6 +224,7 @@ int loadAppendOnlyFile(char *filename) { long loops = 0; if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) { + server.appendonly_current_size = 0; fclose(fp); return REDIS_ERR; } @@ -299,6 +303,7 @@ int loadAppendOnlyFile(char *filename) { freeFakeClient(fakeClient); server.appendonly = appendonly; stopLoading(); + aofUpdateCurrentSize(); return REDIS_OK; readerr: @@ -611,9 +616,10 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(redisClient *c) { if (server.bgrewritechildpid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - return; - } - if (rewriteAppendOnlyFileBackground() == REDIS_OK) { + } else if (server.bgsavechildpid != -1) { + server.aofrewrite_scheduled = 1; + addReplyStatus(c,"Background append only file rewriting started"); + } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { addReply(c,shared.err); @@ -627,6 +633,21 @@ void aofRemoveTempFile(pid_t childpid) { unlink(tmpfile); } +/* Update the server.appendonly_current_size filed explicitly using stat(2) + * to check the size of the file. This is useful after a rewrite or after + * a restart, normally the size is updated just adding the write length + * to the current lenght, that is much faster. */ +void aofUpdateCurrentSize(void) { + struct redis_stat sb; + + if (redis_fstat(server.appendfd,&sb) == -1) { + redisLog(REDIS_WARNING,"Unable to check the AOF length: %s", + strerror(errno)); + } else { + server.appendonly_current_size = sb.st_size; + } +} + /* A background append only file rewriting (BGREWRITEAOF) terminated its work. * Handle this. */ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { @@ -667,6 +688,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(fd); server.appendseldb = -1; /* Make sure it will issue SELECT */ redisLog(REDIS_NOTICE,"The new append only file was selected for future appends."); + aofUpdateCurrentSize(); } else { /* If append only is disabled we just generate a dump in this * format. Why not? */ diff --git a/src/config.c b/src/config.c index 98fdb15d..73fd10e4 100644 --- a/src/config.c +++ b/src/config.c @@ -231,6 +231,18 @@ void loadServerConfig(char *filename) { err = "argument must be 'no', 'always' or 'everysec'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") && + argc == 2) + { + server.auto_aofrewrite_perc = atoi(argv[2]); + if (server.auto_aofrewrite_perc < 0) { + err = "Invalid negative percentage for AOF auto rewrite"; + goto loaderr; + } + } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && + argc == 2) + { + server.auto_aofrewrite_min_size = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { server.requirepass = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { diff --git a/src/rdb.c b/src/rdb.c index 0d4940d2..d9dac659 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1039,6 +1039,9 @@ void bgsaveCommand(redisClient *c) { if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) { addReplyError(c,"Background save already in progress"); return; + } else if (server.bgrewritechildpid != -1) { + addReplyError(c,"Can't BGSAVE while AOF log rewriting is in progress"); + return; } if (rdbSaveBackground(server.dbfilename) == REDIS_OK) { addReplyStatus(c,"Background saving started"); diff --git a/src/redis.c b/src/redis.c index dc8a0032..9243d554 100644 --- a/src/redis.c +++ b/src/redis.c @@ -633,6 +633,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if ((server.maxidletime && !(loops % 100)) || server.bpop_blocked_clients) closeTimedoutClients(); + /* Start a scheduled AOF rewrite if this was requested by the user while + * a BGSAVE was in progress. */ + if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1 && + server.aofrewrite_scheduled) + { + rewriteAppendOnlyFileBackground(); + } + /* Check if a background saving or AOF rewrite in progress terminated. */ if (server.bgsavechildpid != -1 || server.bgrewritechildpid != -1) { int statloc; @@ -667,9 +675,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } } } else if (!server.ds_enabled) { - /* If there is not a background saving in progress check if - * we have to save now */ time_t now = time(NULL); + + /* If there is not a background saving/rewrite in progress check if + * we have to save/rewrite now */ for (j = 0; j < server.saveparamslen; j++) { struct saveparam *sp = server.saveparams+j; @@ -681,6 +690,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { break; } } + + /* Trigger an AOF rewrite if needed */ + if (server.auto_aofrewrite_perc && + server.appendonly_current_size > server.auto_aofrewrite_min_size) + { + int growth = (server.appendonly_current_size*100/ + server.auto_aofrewrite_base_size); + if (growth >= server.auto_aofrewrite_perc) { + redisLog(REDIS_NOTICE,"Starting automatic rewriting of AOF on %d growth",growth); + rewriteAppendOnlyFileBackground(); + } + } } /* Expire a few keys per cycle, only if this is a master. @@ -828,6 +849,10 @@ void initServerConfig() { server.appendonly = 0; server.appendfsync = APPENDFSYNC_EVERYSEC; server.no_appendfsync_on_rewrite = 0; + server.auto_aofrewrite_perc = REDIS_AUTO_AOFREWRITE_PERC; + server.auto_aofrewrite_min_size = REDIS_AUTO_AOFREWRITE_MIN_SIZE; + server.auto_aofrewrite_base_size = 0; + server.aofrewrite_scheduled = 0; server.lastfsync = time(NULL); server.appendfd = -1; server.appendseldb = -1; /* Make sure the first time will not match */ diff --git a/src/redis.h b/src/redis.h index 4249985f..ce62d226 100644 --- a/src/redis.h +++ b/src/redis.h @@ -50,6 +50,8 @@ #define REDIS_SHARED_INTEGERS 10000 #define REDIS_REPLY_CHUNK_BYTES (5*1500) /* 5 TCP packets with default MTU */ #define REDIS_MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ +#define REDIS_AUTO_AOFREWRITE_PERC 100 +#define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024) /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@ -555,6 +557,11 @@ struct redisServer { int appendonly; int appendfsync; int no_appendfsync_on_rewrite; + int auto_aofrewrite_perc; /* Rewrite AOF if % growth is > M and... */ + off_t auto_aofrewrite_min_size; /* the AOF file is at least N bytes. */ + off_t auto_aofrewrite_base_size;/* AOF size on latest startup or rewrite. */ + off_t appendonly_current_size; /* AOF current size. */ + int aofrewrite_scheduled; /* Rewrite once BGSAVE terminates. */ int shutdown_asap; int activerehashing; char *requirepass;