diff --git a/src/search_local/index_read/index_read.cc b/src/search_local/index_read/index_read.cc index 8daba30..1e0d401 100644 --- a/src/search_local/index_read/index_read.cc +++ b/src/search_local/index_read/index_read.cc @@ -67,10 +67,7 @@ static CAgentProcess* agentProcess; static CAgentListenPkg* agentListener; static CSearchProcess* searchProcess; static CPollThread* updateThread; - -extern MemPool listNodePool; extern MemPool skipNodePool; - pthread_mutex_t mutex; SyncIndexTimer *globalSyncIndexTimer; int stop = 0; @@ -116,7 +113,6 @@ static int ServicePreRun(int log_level, bool deam, string log_path) log_debug("start service verison %s build at %s %s", SEARCH_VERSION_STR, __DATE__, __TIME__); pthread_mutex_init(&mutex, NULL); - listNodePool.SetMemInfo("listnode", sizeof(ListNode)); skipNodePool.SetMemInfo("skipnode", sizeof(SkipListNode)); if (SIG_ERR == signal(SIGTERM, catch_signal)) @@ -142,11 +138,6 @@ static int ServicePreRun(int log_level, bool deam, string log_path) return -RT_INIT_ERR; } - if (!TernaryTree::Instance()->Init(global_cfg.suggestPath)){ - log_error("int suggest words error"); - return -RT_INIT_ERR; - } - CConfig* DTCCacheConfig = new CConfig(); if (cache_file == NULL || DTCCacheConfig->ParseConfig(cache_file, "search_cache")) { log_error("no cache config or config file [%s] is error", cache_file); @@ -177,7 +168,6 @@ void ServicePostRun(string str_pid_file) { SearchConf::Instance()->Destroy(); DataManager::Instance()->Destroy(); SplitManager::Instance()->Destroy(); - TernaryTree::Instance()->Destroy(); search_delete_pid(str_pid_file); } diff --git a/src/search_local/index_read/index_sync/sequence_search_index.cc b/src/search_local/index_read/index_sync/sequence_search_index.cc index 63a263f..0951fdd 100644 --- a/src/search_local/index_read/index_sync/sequence_search_index.cc +++ b/src/search_local/index_read/index_sync/sequence_search_index.cc @@ -403,16 +403,10 @@ vector split(const string& str, const string& delim) { } std::string getPath(const char *bind_addr){ - string str = bind_addr; - vector res = split(str, ":"); - res = split(res[1], "/"); - string s = "/tmp/domain_socket/rocks_direct_"; - s.append(res[0]); - s.append(".sock"); + string s = "/tmp/domain_socket/rocks_direct_20000.sock"; return s; } - void SearchRocksDBIndex::buildTableFieldMap(CConfig* _DTCTableConfig){ int fieldCount = _DTCTableConfig->GetIntVal("TABLE_DEFINE", "FieldCount", 0); diff --git a/src/search_local/index_read/index_tbl_op.h b/src/search_local/index_read/index_tbl_op.h index 4ba4a48..966ecbe 100644 --- a/src/search_local/index_read/index_tbl_op.h +++ b/src/search_local/index_read/index_tbl_op.h @@ -23,7 +23,6 @@ #include "chash.h" #include "search_conf.h" -#include "search_list.h" #include "search_util.h" #include "json/value.h" #include diff --git a/src/search_local/index_read/lirs_cache.cc b/src/search_local/index_read/lirs_cache.cc deleted file mode 100644 index 1d24bf8..0000000 --- a/src/search_local/index_read/lirs_cache.cc +++ /dev/null @@ -1,813 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: lirs_cache.h - * - * Description: lirs cache class definition. - * - * Version: 1.0 - * Created: 22/04/2019 10:02:05 PM - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#include "lirs_cache.h" -#include "log.h" - -#include - -#define OPEN_SYNTAX_CHECK -#define REMOVE_LIR_WHEN_STACK_FULL // the replacement policy when the stack if full - -LirsCache::LirsStack::LirsStack( - const int maxLirNum, - const int maxStackSize) -: -mMaxLirEntryNum(maxLirNum), -mMaxStackSize(maxStackSize), -mCurrLirEntryNum(0), -mCurrStackSize(0), -mStackBottom(NULL), -mStackTop(NULL) -{ -} - -LirsCache::LirsStack::~LirsStack() -{ - LirsEntry_t *prev, *curr = mStackBottom; - while (curr) - { - prev = curr; - curr = curr->sStackNext; - - if (prev->sEntryState & HIR_BLOCK_SHARED) - prev->sEntryState &= ~HIR_BLOCK_SHARED; - else - delete prev; - } -} - -void -LirsCache::LirsStack::removeEntry( - LirsEntry_t *entry, - std::map &entryMap, - const bool releaseEntry) -{ - if (!entry || !(entry->sEntryState & HIR_BLOCK_ONSTACK)) - { - log_error("internal error, entryEmpty:%d.", !entry); - return; - } - - if (!entry->sStackPrev) - { - assert(entry == mStackBottom); - - mStackBottom = entry->sStackNext; - if (!mStackBottom) mStackTop = NULL; - else mStackBottom->sStackPrev = NULL; - } - else if (!entry->sStackNext) - { - assert(entry == mStackTop); - - mStackTop = entry->sStackPrev; - if (!mStackTop) mStackBottom = NULL; - else mStackTop->sStackNext = NULL; - } - else - { - assert(entry != mStackBottom && entry != mStackTop); - - entry->sStackPrev->sStackNext = entry->sStackNext; - entry->sStackNext->sStackPrev = entry->sStackPrev; - } - - char &state = entry->sEntryState; - bool canRelease = (releaseEntry && !(state & HIR_BLOCK_SHARED)); - if (state & LIR_BLOCK) mCurrLirEntryNum--; - state &= (~HIR_BLOCK_ONSTACK & ~HIR_BLOCK_SHARED & ~LIR_BLOCK); - mCurrStackSize--; - - if (canRelease) - { - log_info("remove entry, key:%s", entry->sKey.c_str()); - entryMap.erase(entry->sKey); - delete entry; - entry = NULL; - } - else - { - entry->sStackPrev = entry->sStackNext = NULL; - } - - return; -} - -// 1.when call this function, must be has enough space for the appending entry -// 2.the entry must be a non-exist entry in the stack -void -LirsCache::LirsStack::appendEntry(LirsEntry_t *entry) -{ - if (!entry) - { - log_error("append empty entry."); - assert(false); - return; - } - - char &state = entry->sEntryState; - if (state < 0 || (mCurrLirEntryNum >= mMaxLirEntryNum && (state & LIR_BLOCK))) - { - log_error("no enough space for the Lir entry"); - assert(false); - return; - } - - if (mCurrStackSize >= mMaxStackSize) - { - log_error("no enough space for the Hir entry"); - assert(false); - return; - } - - // has enough space, append it - if (!mStackTop) - { - // the first one - mStackTop = mStackBottom = entry; - entry->sStackPrev = NULL; - entry->sStackNext = NULL; - } - else - { - // append to the behind of the top entry - mStackTop->sStackNext = entry; - entry->sStackPrev = mStackTop; - mStackTop = entry; - mStackTop->sStackNext = NULL; - } - - if (state & LIR_BLOCK) mCurrLirEntryNum++; - mCurrStackSize++; - - state |= HIR_BLOCK_ONSTACK; - if (state & (HIR_BLOCK_ONQUEUE | HIR_RESIDENT_BLOCK)) state |= HIR_BLOCK_SHARED; - - return; -} - -// evicted all HIR blocks those located in the bottom of stack -void -LirsCache::LirsStack::stackPrune(std::map &entryMap) -{ - if (!mStackBottom) return; - - while (mStackBottom) - { - if (mStackBottom->sEntryState & LIR_BLOCK) break; - removeEntry(mStackBottom, entryMap); - } - - return; -} - -// release one hir block from the bottom of the stack -void -LirsCache::LirsStack::releaseOneHirEntry(std::map &entryMap) -{ - if (!mStackBottom) return; - - LirsEntry_t *curr = mStackBottom->sStackNext; - while (curr) - { - if (curr->sEntryState & LIR_BLOCK) curr = curr->sStackNext; - - // remove the entry - removeEntry(curr, entryMap, true); - break; - } - - return; -} - -/////////////////////////////////////////////////////////// -// Lirs queue relevant -/////////////////////////////////////////////////////////// -LirsCache::LirsQueue::LirsQueue(const int maxQueueSize) -: -mMaxQueueSize(maxQueueSize), -mCurrQueueSize(0), -mQueueHead(NULL), -mQueueTail(NULL) -{ -} - -LirsCache::LirsQueue::~LirsQueue() -{ - LirsEntry_t *prev, *curr = mQueueHead; - while (curr) - { - prev = curr; - curr = curr->sQueueNext; - - if (prev->sEntryState & HIR_BLOCK_SHARED) - prev->sEntryState &= ~HIR_BLOCK_SHARED; - else - delete prev; - } -} - -// evicted the resident HIR block from the queue -// use flag 'release' to forbidden the caller to release the entry -// if someone holding it current now -void -LirsCache::LirsQueue::removeEntry( - LirsEntry_t *entry, - std::map &entryMap, - const bool release) -{ - if (!entry) - { - log_error("can not remove an empty entry."); - return; - } - - char &state = entry->sEntryState; - if (!(state & HIR_RESIDENT_BLOCK)) - { - assert(false); - log_error("incorrect entry state."); - return; - } - - if (!entry->sQueuePrev) - { - mQueueHead = entry->sQueueNext; - if (!mQueueHead) mQueueTail = NULL; - else mQueueHead->sQueuePrev = NULL; - } - else if (!entry->sQueueNext) - { - mQueueTail = entry->sQueuePrev; - if (!mQueueTail) mQueueHead = NULL; - else mQueueTail->sQueueNext = NULL; - } - else - { - entry->sQueuePrev->sQueueNext = entry->sQueueNext; - entry->sQueueNext->sQueuePrev = entry->sQueuePrev; - } - - // double check - if (release && !(state & HIR_BLOCK_ONSTACK) && !(state & HIR_BLOCK_SHARED)) - { - log_info("remove entry, key:%s", entry->sKey.c_str()); - entryMap.erase(entry->sKey); - delete entry; - entry = NULL; - } - else - { - // clear flag - entry->sQueuePrev = entry->sQueueNext = NULL; - state &= (~HIR_BLOCK_ONQUEUE & ~HIR_BLOCK_SHARED & ~HIR_RESIDENT_BLOCK); - } - mCurrQueueSize--; - - return; -} - -// when call this function, queue should has enough remaining space for appending -void -LirsCache::LirsQueue::appendEntry(LirsEntry_t *entry) -{ - if (!entry || (entry->sEntryState & LIR_BLOCK)) - { - log_error("empty entry:%d.", entry == NULL); - assert(false); - return; - } - - char &state = entry->sEntryState; - if (state < 0 || mCurrQueueSize >= mMaxQueueSize) - { - log_error("incorrect queue data."); - return; - } - - // just append to the tail directly - if (!mQueueTail) - { - mQueueHead = mQueueTail = entry; - mQueueHead->sQueuePrev = NULL; - mQueueTail->sQueueNext = NULL; - } - else - { - mQueueTail->sQueueNext = entry; - entry->sQueuePrev = mQueueTail; - mQueueTail = entry; - mQueueTail->sQueueNext = NULL; - } - mCurrQueueSize++; - - state |= (HIR_BLOCK_ONQUEUE | HIR_RESIDENT_BLOCK); - state &= ~LIR_BLOCK; - if (state & HIR_BLOCK_ONSTACK) state |= HIR_BLOCK_SHARED; - - return; -} - -/////////////////////////////////////////////////////////// -// LIRS cache relevant -/////////////////////////////////////////////////////////// -LirsCache::LirsCache(const int cacheSize) -: -mCacheEntrySize(cacheSize) -{ - if (mCacheEntrySize < eMinCacheEntrySize || mCacheEntrySize > eMaxCacheEntrySize) - mCacheEntrySize = mCacheEntrySize < eMinCacheEntrySize ? eMinCacheEntrySize : mCacheEntrySize; - - int queueSize = mCacheEntrySize * eQueueSizeRate / 100; - if (queueSize < eMinQueueSize) queueSize = eMinQueueSize; - - int maxLirEntryNum = mCacheEntrySize - queueSize; - int maxStackSize = mCacheEntrySize + queueSize; // the extra queue size for holding non-resident HIR blocks - - // LIRS algorithm dynamically vary the stack cache size and it will not consume - // all the memory with eInfiniteCacheEntrySize cache size???? - // maxStackSize = 5 * maxLirEntryNum; - maxStackSize = eInfiniteCacheEntrySize; - mBlockStack = new LirsStack(maxLirEntryNum, maxStackSize); - mBlockQueue = new LirsQueue(queueSize); -} - -LirsCache::~LirsCache() -{ - if (mBlockStack) delete mBlockStack; - if (mBlockQueue) delete mBlockQueue; -} - -// find the key and adjust the lirs cache -LirsEntry_t* -LirsCache::findEntry(const std::string &key) -{ - MapItr_t itr = mEntryMap.find(key); - if (itr == mEntryMap.end()) return NULL; - - LirsEntry_t *entry = itr->second; - assert(entry != NULL); - // Note:In LIRS paper, it says that if accessing a non-resident HIR block - // in the stack(if the non-resident block in the map, it must be in the stack), - // we change its status to LIR and move it to the top of the stack,so it - // also a hit in the cache - // if (!entry || !(entry->sEntryState & HIR_RESIDENT_BLOCK)) return NULL; - - // hit Lir or Resident Hir block, adjust the cache - adjustLirsCache(entry); - syntaxCheck(); - return entry; -} - -// 1.if exist, update the value -// 2.append a new entry -bool -LirsCache::appendEntry( - const std::string &key, - const std::string &value) -{ - // find in the stack first - LirsEntry_t *entry = NULL; - MapItr_t itr = mEntryMap.find(key); - if (itr != mEntryMap.end()) - { - entry = itr->second; -#if (__cplusplus >= 201103L) - // c++0x, use rvalue reference, value can not be used any more - entry->sValue = std::move(value); -#else - entry->sValue = value; -#endif // __cplusplus >= 201103L - adjustLirsCache(entry); - syntaxCheck(); - - log_info("update entry, key:%s, value:%s, state:%d", key.c_str(),\ - value.c_str(), entry->sEntryState); - return true; - } - - // append a new entry - entry = new LirsEntry_t(); - if (!entry) - { - log_error("allocate memory failed."); - return false; - } - entry->initEntry(0, NULL, NULL, NULL, NULL, key, value); - char &state = entry->sEntryState; - - // add into the map - mEntryMap[key] = entry; - - // make sure have enough space for appending - bool isLirFull = mBlockStack->isLirEntryFull(); - bool isStackFull = mBlockStack->isStackFull(); - if (!isLirFull) - { -#ifndef REMOVE_LIR_WHEN_STACK_FULL - if (isStackFull) mBlockStack->releaseOneHirEntry(mEntryMap); -#else - if (isStackFull) removeStackBottom(); -#endif - - // add as a lir entry - state |= LIR_BLOCK; - mBlockStack->appendEntry(entry); - syntaxCheck(); - - log_info("append entry, key:%s, value:%s, state:%d",\ - key.c_str(), value.c_str(), entry->sEntryState); - return true; - } - - // add as a resident HIR block - bool isQueueFull = mBlockQueue->isQueueFull(); - if (isQueueFull || isStackFull) - { - if (isQueueFull) - { - // remove resident HIR block from queue - LirsEntry_t *head = mBlockQueue->getHeadOfQueue(); - mBlockQueue->removeEntry(head, mEntryMap); - } - - // check whether the stack is full or not - if (isStackFull) - { - // remove the lir block in the bottom - removeStackBottom(); - - // append entry as a lir block - state |= LIR_BLOCK; - mBlockStack->appendEntry(entry); - syntaxCheck(); - - log_info("append entry, key:%s, value:%s, state:%d",\ - key.c_str(), value.c_str(), entry->sEntryState); - return true; - } - } - - // append to both the stack and the queue as an resident block - // state |= (HIR_RESIDENT_BLOCK | HIR_BLOCK_SHARED); - mBlockStack->appendEntry(entry); - mBlockQueue->appendEntry(entry); - assert(state == 30); - syntaxCheck(); - - log_info("append entry, key:%s, value:%s, state:%d",\ - key.c_str(), value.c_str(), entry->sEntryState); - return true; -} - -bool -LirsCache::removeEntry(const std::string &key) -{ - MapItr_t itr = mEntryMap.find(key); - if (itr == mEntryMap.end()) return true; - - LirsEntry_t *entry = itr->second; - char state = entry->sEntryState; - - // remove from the stack - if (state & HIR_BLOCK_ONSTACK) - { - mBlockStack->removeEntry(entry, mEntryMap); - - // try to conduct a pruning - mBlockStack->stackPrune(mEntryMap); - } - - // remove from the queue - if (state & HIR_BLOCK_ONQUEUE) - { - mBlockQueue->removeEntry(entry, mEntryMap); - } - - return true; -} - -// remove the bottom LIR entry which has the maximum recency although it -// is a hot block -void -LirsCache::removeStackBottom() -{ - bool isQueueFull = mBlockQueue->isQueueFull(); - if (isQueueFull) - { - LirsEntry_t *head = mBlockQueue->getHeadOfQueue(); - mBlockQueue->removeEntry(head, mEntryMap, true); - } - - // remove bottom of the stack - LirsEntry_t *bottom = mBlockStack->getBottomOfStack(); - mBlockStack->removeEntry(bottom, mEntryMap, false); - mBlockQueue->appendEntry(bottom); - mBlockStack->stackPrune(mEntryMap); - - return; -} - -// entry must be exist in the cache, even if it's a non-resident block -void -LirsCache::adjustLirsCache(LirsEntry_t *entry) -{ - char &state = entry->sEntryState; - if (state & LIR_BLOCK) - { - // Upon accessing an LIR block X - // 1.move X to the top of stack - // 2.conduct a stack pruning if it is located in the bottom of the stack - mBlockStack->removeEntry(entry, mEntryMap, false); - - // maybe the removed entry is bottom, try to conduct a stack pruning - mBlockStack->stackPrune(mEntryMap); - - state |= LIR_BLOCK; - } - else - { - // Upon accessing an Hir block X - // 1.if X is an resident Hir block: - // 1.1) move it to the top of stack - // 1.2) if X is in the stack - // a) change its status to LIR, and removed it from the queue - // b) move the bottom entry in the stack to the end of queue - // c) conduct a stack pruning - // 1.3) if X is not in the stack - // a) leave its status in HIR and move it to the end of the queue - // - // 2.if X is an non-resident Hir block X: - // note:in this case, block X must be located in the stack, otherwise, wo regard - // it as a newly entry and will be append to the cache by API appendEntry - // 2.1) remove the HIR resident block that at the front of queue(if queue full) - // 2.2) append the block X on the top of stack - // 2.3) change its status to LIR - // 2.4) if the num of LIR block is bigger than maxLirEntrySize, remove - // the bottom one of the stack - if (state & HIR_RESIDENT_BLOCK) - { - // resident hir block - if (state & HIR_BLOCK_ONSTACK) - { - // evicted from queue - mBlockQueue->removeEntry(entry, mEntryMap, false); - - // move the bottom entry in the stack to the end of the queue if lir entry full - bool isLirEntryFull = mBlockStack->isLirEntryFull(); - if (isLirEntryFull) - { - LirsEntry_t *bottom = mBlockStack->getBottomOfStack(); - mBlockStack->removeEntry(bottom, mEntryMap, false); - mBlockQueue->appendEntry(bottom); - } - - // evicted myself from stack - mBlockStack->removeEntry(entry, mEntryMap, false); - mBlockStack->stackPrune(mEntryMap); - - state |= LIR_BLOCK; - } - else - { - // 1.leave its status in HIR and move this block to the end of the queue - mBlockQueue->removeEntry(entry, mEntryMap, false); - mBlockQueue->appendEntry(entry); - - // 2.append to the stack - bool isStackFull = mBlockStack->isStackFull(); - if (isStackFull) - { -#ifndef REMOVE_LIR_WHEN_STACK_FULL - // remove the first HIR entry from stack bottom - mBlockStack->releaseOneHirEntry(mEntryMap); -#else - // remove the Lir block in the bottom of the stack - removeStackBottom(); -#endif - } - } - } - else - { - // non-resident hir block, block must be in the stack, if not in the stack, - // it must be a new entry that we should call appendEntry function to add it - if (!(state & HIR_BLOCK_ONSTACK) || (state & HIR_BLOCK_ONQUEUE)) - { - log_error("internal error."); - assert(false); - return; - } - - bool isLirEntryFull = mBlockStack->isLirEntryFull(); - if (isLirEntryFull) - { - bool isQueueFull = mBlockQueue->isQueueFull(); - if (isQueueFull) - { - // remove the resident HIR block from the head of queue first - LirsEntry_t *head = mBlockQueue->getHeadOfQueue(); - mBlockQueue->removeEntry(head, mEntryMap, true); - } - - // move the entry in the bottom of the stack into the tail of the queue - LirsEntry_t *bottom = mBlockStack->getBottomOfStack(); - mBlockStack->removeEntry(bottom, mEntryMap, false); - mBlockQueue->appendEntry(bottom); - } - - // remove the entry from the stack first, then conduct stack prune - mBlockStack->removeEntry(entry, mEntryMap, false); - - mBlockStack->stackPrune(mEntryMap); - - state |= LIR_BLOCK; - } - } - - // append this entry to the top of the stack - mBlockStack->appendEntry(entry); - - return; -} - -// check LIRS cache -bool LirsCache::syntaxCheck() -{ -#ifndef OPEN_SYNTAX_CHECK - // do nothing -#else - int stackBlockNum = 0; - int stackLirBlockNum = 0; - int stackRHirBlockNum = 0; - int stackNRHirBlockNum = 0; - int queueSharedBlockNum = 0; - - // check stack - if (mBlockStack) - { - LirsEntry_t *bottom = mBlockStack->getBottomOfStack(); - LirsEntry_t *top = mBlockStack->getTopOfStack(); - - char state; - LirsEntry_t *prev = NULL, *curr = bottom; - while (curr) - { - state = curr->sEntryState; - if (state <= 0 || - state > (HIR_BLOCK_SHARED + HIR_BLOCK_ONQUEUE + HIR_BLOCK_ONSTACK + HIR_RESIDENT_BLOCK)) - { - log_error("incorrect entry state."); - assert(false); - return false; - } - - if (!(state & HIR_BLOCK_ONSTACK)) - { - log_error("incorrect LIR block state. state:%d", state); - assert(false); - return false; - } - - stackBlockNum++; - - if (state & LIR_BLOCK) - { - if ((state & HIR_RESIDENT_BLOCK) - || (state & HIR_BLOCK_ONQUEUE) - || (state & HIR_BLOCK_SHARED)) - { - log_error("incorrect LIR block. state:%d", state); - assert(false); - return false; - } - stackLirBlockNum++; - } - else if (state & HIR_RESIDENT_BLOCK) - { - if (!(state & HIR_BLOCK_ONQUEUE) - || !(state & HIR_BLOCK_SHARED)) - { - log_error("incorrect LIR block. state:%d", state); - assert(false); - return false; - } - stackRHirBlockNum++; - } - else - { - if ((state & HIR_BLOCK_ONQUEUE) - || (state & HIR_BLOCK_SHARED)) - { - log_error("incorrect LIR block. state:%d", state); - assert(false); - return false; - } - stackNRHirBlockNum++; - } - - prev = curr; - curr = curr->sStackNext; - if (curr && prev != curr->sStackPrev) - { - log_error("incorrect double link."); - assert(false); - return false; - } - } - - assert(prev == top); - } - - // check cache size - if (stackRHirBlockNum > mBlockQueue->getCurrQueueSize()) - { - log_error("check RHir block failed."); - assert(false); - return false; - } - - // check queue - if (mBlockQueue) - { - LirsEntry_t *head = mBlockQueue->getHeadOfQueue(); - LirsEntry_t *tail = mBlockQueue->getTailOfQueue(); - - char state; - LirsEntry_t *prev = NULL, *curr = head; - while (curr) - { - state = curr->sEntryState; - if (state <= 0 || - state > (HIR_BLOCK_SHARED + HIR_BLOCK_ONQUEUE + HIR_BLOCK_ONSTACK + HIR_RESIDENT_BLOCK)) - { - log_error("incorrect entry state."); - assert(false); - return false; - } - - if (!(state & HIR_BLOCK_ONQUEUE) || !(state & HIR_RESIDENT_BLOCK)) - { - log_error("incorrect Resident HIR block state. state:%d", state); - assert(false); - return false; - } - - if (state & LIR_BLOCK) - { - log_error("incorrect Resident HIR block state. state:%d", state); - assert(false); - return false; - } - - if (state & HIR_BLOCK_ONSTACK) - { - if (!(state & HIR_BLOCK_SHARED)) - { - log_error("incorrect Resident HIR block state. state:%d", state); - assert(false); - return false; - } - - queueSharedBlockNum++; - } - - prev = curr; - curr = curr->sQueueNext; - if (curr && prev != curr->sQueuePrev) - { - log_error("incorrect double link."); - assert(false); - return false; - } - } - - assert(prev == tail); - } - - if (stackRHirBlockNum != queueSharedBlockNum) - { - log_error("shared pointer occur error."); - assert(false); - return false; - } -#endif - - return true; -} diff --git a/src/search_local/index_read/lirs_cache.h b/src/search_local/index_read/lirs_cache.h deleted file mode 100644 index b1a7c7c..0000000 --- a/src/search_local/index_read/lirs_cache.h +++ /dev/null @@ -1,178 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: lirs_cache.h - * - * Description: lirs cache class definition. - * - * Version: 1.0 - * Created: 22/04/2019 10:02:05 PM - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#ifndef LIRS_CACHE_H__ -#define LIRS_CACHE_H__ - -#include -#include -#include -#include - -// LIRS use two data structure to hold all cache data, LIRS stack and queue. -// Data to be described as hot data and cold data, hot data names LIR(low IRR) -// and cold data is HIR(high IRR), all LIR data are located in the LIRS stack and -// the others located either in the stack or queue; The HIR data also be -// divided into resident and non-resident, all resident HIR data are linked -// to be a small size queue - -#define LIR_BLOCK 1 // Hot data -#define HIR_RESIDENT_BLOCK 2 // HIR is cold data -#define HIR_BLOCK_ONSTACK 4 -#define HIR_BLOCK_ONQUEUE 8 -#define HIR_BLOCK_SHARED 16 // shared Resident HIR entry reference between Stack and Queue -// 1.Unfixed data type(include either key or value): -// unsigned long long, float, double, string -// 2.Except 1, others is fixed, such as the following: -// char, short, int, the unsigned series that size is small than 8, and so on -// #define HIR_BLOCK_FIXED 32 - -typedef struct LirsEntry -{ - char sEntryState; - // 1.we assume that the value is big enough, so the space cost in using shared entry - // mechanism(two double link pointer) will be cheaper than clone the same entry - // 2.use shared ptr let us implement the LRU cache with only one Hashmap - struct LirsEntry *sStackPrev; - struct LirsEntry *sStackNext; - struct LirsEntry *sQueuePrev; - struct LirsEntry *sQueueNext; - std::string sKey; - std::string sValue; - - void initEntry( - const char state, - struct LirsEntry *sPrev, - struct LirsEntry *sNext, - struct LirsEntry *qPrev, - struct LirsEntry *qNext, - const std::string &key, - const std::string &value) - { - sEntryState = state; - sStackPrev = sPrev; - sStackNext = sNext; - sQueuePrev = qPrev; - sQueueNext = qNext; -#if (__cplusplus >= 201103L) - sKey = std::move(key); - sValue = std::move(value); -#else - sKey = key; - sValue = value; -#endif - } -}LirsEntry_t; - - -class LirsCache -{ -private: - enum CacheRelevant - { - eQueueSizeRate = 1, // 1% - eMinQueueSize = 2, - // the minimum of the queue size is must gt 2 in case of the - // latest additional HIR block is removed due to the removeStackBottom - // operation - eMinCacheEntrySize = 100, - eMaxCacheEntrySize = 500000, - eInfiniteCacheEntrySize = 0x7FFFFFF - }; - -private: - typedef std::map::iterator MapItr_t; - - class LirsStack - { - private: - int mMaxLirEntryNum; // Maximum LIR entry number - int mMaxStackSize; // maximum real stack capacity, contain LIR + resident HIR + non-resident blocks - int mCurrLirEntryNum; - int mCurrStackSize; - LirsEntry_t* mStackBottom; - LirsEntry_t* mStackTop; - - public: - LirsStack(const int maxLir, const int maxStackSize); - virtual ~LirsStack(); - - inline LirsEntry_t* getBottomOfStack() { return mStackBottom; } - inline LirsEntry_t* getTopOfStack() { return mStackTop; } - inline bool isLirEntryFull() { return mCurrLirEntryNum >= mMaxLirEntryNum; } - inline bool isStackFull() { return mCurrStackSize >= mMaxStackSize; } - void stackPrune(std::map &entryMap); - void releaseOneHirEntry(std::map &entryMap); - - void appendEntry(LirsEntry_t *entry); - void removeEntry( - LirsEntry_t *entry, - std::map &entryMap, - const bool releaseEntry = true); - }; - - class LirsQueue - { - private: - int mMaxQueueSize; // Maximum resident HIR entry number - int mCurrQueueSize; - LirsEntry_t *mQueueHead; - LirsEntry_t *mQueueTail; - - public: - LirsQueue(const int maxQueueSize); - virtual ~LirsQueue(); - - inline LirsEntry_t* getHeadOfQueue() { return mQueueHead; } - inline LirsEntry_t* getTailOfQueue() { return mQueueTail; } - inline bool isQueueFull() { return mCurrQueueSize >= mMaxQueueSize; } - inline int getCurrQueueSize() { return mCurrQueueSize; } - - void appendEntry(LirsEntry_t *entry); - void removeEntry( - LirsEntry_t *entry, - std::map &entryMap, - const bool releaseEntry = true); - }; - -public: - explicit LirsCache(const int cacheSize = eMaxCacheEntrySize); - virtual ~LirsCache(); - - LirsEntry_t* findEntry(const std::string &key); - - // user convert all basic data type to string - bool appendEntry(const std::string &key, const std::string &value); - bool removeEntry(const std::string &key); - -private: - void removeStackBottom(); - void adjustLirsCache(LirsEntry_t * entry); - bool syntaxCheck(); - -private: - int mCacheEntrySize; // LIR and resident HIR block nums - LirsStack* mBlockStack; // store all LIR blocks and some HIR blocks - LirsQueue* mBlockQueue; // store all resident HIR blocks - std::map mEntryMap; // store all cache data for efficient search - - friend class LirsStack; - friend class LirsQueue; -}; - -#endif // LIRS_CACHE_H__ diff --git a/src/search_local/index_read/search_env.cc b/src/search_local/index_read/search_env.cc deleted file mode 100644 index 7b099c8..0000000 --- a/src/search_local/index_read/search_env.cc +++ /dev/null @@ -1,161 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: search_env.h - * - * Description: search env class definition. - * - * Version: 1.0 - * Created: 16/03/2018 - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#include -#include -#include -#include -#include "log.h" -#include "search_env.h" -#include -#include "comm.h" - - -int SearchEnv::cpa_atoi(char *line, size_t n) { - int value; - - if (n == 0) { - return -RT_PARA_ERR; - } - - for (value = 0; n--; line++) { - if (*line < '0' || *line > '9') { - return -RT_PARA_ERR; - } - - value = value * 10 + (*line - '0'); - } - - if (value < 0) { - return -RT_PARA_ERR; - } - - return value; -} - -char *SearchEnv::UnresolveAddr(struct sockaddr *addr, int addrlen) -{ - static char unresolve[NI_MAXHOST + NI_MAXSERV]; - static char host[NI_MAXHOST], service[NI_MAXSERV]; - int status; - - status = getnameinfo(addr, addrlen, host, sizeof(host), service, - sizeof(service), NI_NUMERICHOST | NI_NUMERICSERV); - if (status < 0) - { - log_error("get sock name err, errno %d, errmsg %s", status, gai_strerror(status)); - snprintf(unresolve, sizeof(unresolve), "unknown"); - return unresolve; - } - - snprintf(unresolve, sizeof(unresolve), "%s:%s", host, service); - log_error("addr:%s", unresolve); - return unresolve; -} - -char *SearchEnv::UnresolveDesc(int sd) -{ - static char unresolve[NI_MAXHOST + NI_MAXSERV]; - struct sockaddr *addr; - socklen_t addrlen = sizeof(struct sockaddr); - int status; - addr = (struct sockaddr *)malloc(sizeof(struct sockaddr)); - if (addr == NULL) - { - log_error("no mem"); - snprintf(unresolve, sizeof(unresolve), "unknown"); - return unresolve; - } - status = getsockname(sd, addr, &addrlen); - if (status < 0) { - log_error("get sock name err, errno %d, errmsg %s", errno, strerror(errno)); - snprintf(unresolve, sizeof(unresolve), "unknown"); - return unresolve; - } - sockaddr_in sin; - memcpy(&sin, addr, sizeof(sin)); - return UnresolveAddr(addr, addrlen); -} - -int SearchEnv::WriteSocketEnv(int fd) -{ - int len = 0; - memcpy(os_environ, NC_ENV_FDS, strlen(NC_ENV_FDS)); - len = strlen(NC_ENV_FDS); - memcpy(os_environ + len, "=", strlen("=")); - len += strlen("="); - sprintf(os_environ + len, "%d;", fd); - log_debug("global env is %s", os_environ); - return 0; -} - -int SearchEnv::ReadSocketEnv(struct sockaddr *addr, int addrlen) -{ - char *listen_address = UnresolveAddr(addr, addrlen); - char *p, *q; - int sock; - char *inherited; - inherited = getenv(NC_ENV_FDS); - - char address[NI_MAXHOST + NI_MAXSERV]; - - if (inherited == NULL) { - return 0; - } - - - strncpy(address, listen_address, sizeof(address)); - log_debug("address %s", address); - log_debug("inherited %s", inherited); - for (p = inherited, q = inherited; *p; p++) { - if (*p == ';') { - sock = cpa_atoi(q, p - q); - if (strcmp(address, UnresolveDesc(sock)) == 0) { - log_debug("get inherited socket %d for '%s' from '%s'", sock, - address, inherited); - sock = dup(sock); - log_debug("dup inherited socket as %d", sock); - return sock; - } - q = p + 1; - } - } - - return 0; -} - -int SearchEnv::CleanSocketEnv() -{ - int sock = 0; - char *inherited; - char *p, *q; - - inherited = getenv(NC_ENV_FDS); - if (inherited == NULL) { - return 0; - } - - for (p = inherited, q = inherited; *p; p++) { - if (*p == ';') { - sock = cpa_atoi(q, p - q); - close(sock); - q = p + 1; - } - } - return 0; -} \ No newline at end of file diff --git a/src/search_local/index_read/search_env.h b/src/search_local/index_read/search_env.h deleted file mode 100644 index eab3c57..0000000 --- a/src/search_local/index_read/search_env.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: search_env.h - * - * Description: search env class definition. - * - * Version: 1.0 - * Created: 16/03/2018 - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#ifndef __SEARCH_ENV_H__ -#define __SEARCH_ENV_H__ -#include "singleton.h" - -class SearchEnv { -public: - SearchEnv() { - NC_ENV_FDS = "NC_ENV_FDS"; - } - static SearchEnv *Instance() { - return CSingleton::Instance(); - } - static void Destroy() { - CSingleton::Destroy(); - } - int WriteSocketEnv(int fd); - int ReadSocketEnv(struct sockaddr *addr, int addrlen); - int CleanSocketEnv(); - char *UnresolveDesc(int sd); - char *UnresolveAddr(struct sockaddr *addr, int addrlen); - char *GetEnv() { - return os_environ; - } -private: - int cpa_atoi(char *line, size_t n); - char os_environ[128]; - const char *NC_ENV_FDS; -}; -#endif diff --git a/src/search_local/index_read/search_list.cc b/src/search_local/index_read/search_list.cc deleted file mode 100644 index a2cc729..0000000 --- a/src/search_local/index_read/search_list.cc +++ /dev/null @@ -1,331 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: search_list.h - * - * Description: search list class definition. - * - * Version: 1.0 - * Created: 15/08/2019 - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#include -#include -#include - -#include "log.h" -#include "search_list.h" - -MemPool listNodePool; - -bool SearchList::InsertNode(ListNode *&list, const char *str) -{ - if (str == NULL || strlen(str) == 0) { - log_error("str is empty!"); - return false; - } - - if (strlen(str) >= MAX_DOCID_LENGTH) { - log_error("the length of str : %s is too long", str); - return false; - } - - if (list == NULL) { - list = (ListNode*)listNodePool.PoolAlloc(); - if (list == NULL) { - log_error("allocate listnode error,no more memory!"); - return false; - } - memset(list->key, 0, MAX_DOCID_LENGTH); - list->prev = list->next = list->forward = list->backward = NULL; - list->forward = (ListNode*)listNodePool.PoolAlloc(); - if (list->forward == NULL) { - log_error("allocate listnode error,no more memory!"); - listNodePool.PoolFree(list); - list = NULL; - return false; - } - memset(list->forward->key, 0, MAX_DOCID_LENGTH); - list->forward->prev = list->forward->next = list->forward->forward = list->forward->backward = NULL; - list->forward->backward = list; - strncpy(list->forward->key, str, strlen(str)); - return true; - } - - ListNode *node = NULL; - ListNode *insertNode = list; - while (insertNode->forward != NULL && strcmp(str, insertNode->forward->key) > 0) - insertNode = insertNode->forward; - - if (insertNode->forward != NULL && strcmp(str, insertNode->forward->key) == 0) { - return true; - } - - if (insertNode->forward == NULL) { - node = (ListNode*)listNodePool.PoolAlloc(); - if (node == NULL){ - log_error("allocate listnode error,no more memory!"); - return false; - } - - memset(node->key, 0, MAX_DOCID_LENGTH); - node->prev = node->next = node->forward = node->backward = NULL; - strncpy(node->key, str, strlen(str)); - insertNode->forward = node; - node->backward = insertNode; - return true; - } - - node = (ListNode*)listNodePool.PoolAlloc(); - if (node == NULL){ - log_error("allocate listnode error,no more memory!"); - return false; - } - memset(node->key, 0, MAX_DOCID_LENGTH); - node->prev = node->next = node->forward = node->backward = NULL; - node->forward = insertNode->forward; - node->backward = insertNode; - insertNode->forward->backward = node; - insertNode->forward = node; - strncpy(node->key, str, strlen(str)); - - return true; -} - -void SearchList::InsertListNode(ListNode *list) -{ - if (list == NULL || list->forward == NULL) { - return ; - } - if (size == 0) - { - list->prev = list; - list->next = list; - head = tail = list; - } else if (strcmp(list->forward->key, head->forward->key) <= 0) { - list->next = head; - list->prev = tail; - head->prev = list; - tail->next = list; - head = list; - } else if (strcmp(list->forward->key, tail->forward->key) >= 0) { - list->prev = tail; - list->next = head; - tail->next = list; - head->prev = list; - tail = list; - } else { - ListNode *tmp = head; - while (strcmp(list->forward->key, tmp->forward->key) > 0) - tmp = tmp->next; - list->prev = tmp->prev; - list->next = tmp; - tmp->prev->next = list; - tmp->prev = list; - } - size = size + 1; - - return ; -} - -void SearchList::GetIntersectionSets() -{ - if (size == 0 || size == 1) - return ; - bool res = false; - - ListNode *list = Intersection(res); //res为false代表内存不够，不能新增链表节点 - if (!res) { - DestroyNode(list); - DestroyList(); - list = NULL; - return ; - } - DestroyList(); - InsertListNode(list); - - return ; -} - -ListNode* SearchList::Intersection(bool& res) -{ - ListNode *newList = NULL; //把得到的交集放入newList中，进而销毁原链表，把newList插入链表，此时链表只有newList一个链 - char str[MAX_DOCID_LENGTH]; - - int flag = 0; - int count = 0; - while(1) { - if (count == 0){ - memset(str, 0, MAX_DOCID_LENGTH); - strcpy(str, tail->forward->key); - } else if(tail->forward && tail->forward->forward) { - memset(str, 0, MAX_DOCID_LENGTH); - strcpy(str, tail->forward->forward->key); - } else { - return newList; - } - - ListNode* list = head; - while (list) { - if (list->forward == NULL){ - return newList; - } - flag = ListKeyCompare(list, str); - if (flag == -1) { - log_debug("can't find value str : %s", str); - return newList; - } else if (flag == 1){ - res = InsertNode(newList, str); - if (!res) - return newList; - break; - } - - list = list->next; - } - - count = count + 1; - } - - return newList; -} - -/* - 返回1代表遍历结束，说明所有list均包含str字符串 - 返回-1代表该list中没有与str匹配的值，找不到交集，遍历结束 - 返回0代表该list中发现了与str匹配的值 -*/ -int SearchList::ListKeyCompare(ListNode* list, char *str) -{ - if (!(strcmp(list->forward->key, str) < 0)) { - return 1; - } - ListNode* tmp = list->forward; - while (tmp != NULL && strcmp (tmp->key, str) < 0) { - ListNode *del = tmp; - tmp = tmp->forward; - ListNodeDelete(del); - del = NULL; - } - if (tmp == NULL) { - return -1; - } - memset(str, 0, MAX_DOCID_LENGTH); - strcpy(str, tmp->key); // 把tmp中的key更新str，由遍历结果得知该tmp中的key是大于或者等于str的 - return 0; -} - -bool SearchList::IsContainKey(ListNode* keyList, const char* str) -{ - int res; - if (keyList == NULL || keyList->forward == NULL) - return false; - ListNode* tmp = keyList->forward; - while (tmp != NULL) { - res = strcmp(tmp->key, str); - if (res > 0) - return false; - else if (res == 0) - return true; - tmp = tmp->forward; - } - return false; -} - -void SearchList::GetDifferenceList(ListNode* keyList) -{ - if (size != 1 || keyList == NULL || head->forward == NULL) - return ; - ListNode *node = head->forward; - ListNode *temp = node->forward; - while (node != NULL) - { - if (IsContainKey(keyList, node->key)) - ListNodeDelete(node); - node = temp; - if (node == NULL) - break; - temp = node->forward; - } - - /*防止在删除节点的时候，head链表只剩一个头结点*/ - if (head && head->forward == NULL) { - listNodePool.PoolFree(head); - head = tail = NULL; - size = 0; - } - return ; -} - -void SearchList::GetListElement(vector& eles) -{ - if (size == 0){ - log_debug("search list is empty"); - return ; - } - - ListNode *node = head->forward; - while (node != NULL) { - eles.push_back(node->key); - node = node->forward; - } - return ; -} - -void SearchList::ListNodeDelete(ListNode *node) { - if (node == NULL) { - log_debug("node is NULL, can't delete"); - return ; - } - if (node->forward == NULL) { - node->backward->forward = NULL; - listNodePool.PoolFree(node); - node = NULL; - return ; - } - node->backward->forward = node->forward; - node->forward->backward = node->backward; - listNodePool.PoolFree(node); - node = NULL; - return ; -} - -void SearchList::DestroyList() -{ - if (head == NULL && tail == NULL) - return ; - ListNode *list = head; - ListNode *tmp; - while (list != tail) { - tmp = list->next; - DestroyNode(list); - list = tmp; - } - DestroyNode(list); - head = NULL; - tail = NULL; - size = 0; -} - -void SearchList::DestroyNode(ListNode *list) -{ - if (list == NULL) - return ; - ListNode *tmp = list; - ListNode *li; - while(tmp != NULL) - { - li = tmp->forward; - listNodePool.PoolFree(tmp); - tmp = li; - } - tmp = NULL; - li = NULL; -} \ No newline at end of file diff --git a/src/search_local/index_read/search_list.h b/src/search_local/index_read/search_list.h deleted file mode 100644 index 578d198..0000000 --- a/src/search_local/index_read/search_list.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * ===================================================================================== - * - * Filename: search_list.h - * - * Description: search list class definition. - * - * Version: 1.0 - * Created: 15/08/2019 - * Revision: none - * Compiler: gcc - * - * Author: zhulin, shzhulin3@jd.com - * Company: JD.com, Inc. - * - * ===================================================================================== - */ - -#ifndef SEARCH_LIST_H_H_ -#define SEARCH_LIST_H_H_ -#include "mem_pool.h" -#include "comm.h" -#include -#include -using namespace std; - -struct ListNode -{ - char key[MAX_DOCID_LENGTH]; - ListNode *prev; - ListNode *next; - ListNode *forward; - ListNode *backward; -}; - -class SearchList -{ -public: - SearchList():head(NULL),tail(NULL),size(0){} - ~SearchList(){ - DestroyList(); - } - - bool InsertNode(ListNode *&list, const char *str); - void InsertListNode(ListNode *list); - void GetIntersectionSets(); - ListNode* Intersection(bool& res); - int ListKeyCompare(ListNode* list, char *str); - void ListNodeDelete(ListNode *node); - int GetSize() {return size;} - ListNode* GetHeadList() {return head;} - bool IsContainKey(ListNode* keyList, const char* str); - void GetDifferenceList(ListNode* keyList); - void GetListElement(vector &eles); - - void DestroyList(); - void DestroyNode(ListNode *list); -private: - ListNode *head; - ListNode *tail; - int size; -}; - -#endif \ No newline at end of file