mirrors/isearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

delete TernaryTree

Browse Source
This commit is contained in:
shzhulin3 2021-04-22 14:02:56 +08:00
parent 710f8f8df8
commit f026781a61
9 changed files with 1 additions and 1612 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/search_local/index_read/index_read.cc
View File

@ -67,10 +67,7 @@ static CAgentProcess* agentProcess;
static CAgentListenPkg* agentListener;
static CSearchProcess* searchProcess;
static CPollThread* updateThread;
extern MemPool listNodePool;
extern MemPool skipNodePool;
pthread_mutex_t mutex;
SyncIndexTimer *globalSyncIndexTimer;
int stop = 0;
@ -116,7 +113,6 @@ static int ServicePreRun(int log_level, bool deam, string log_path)
log_debug("start service verison %s build at %s %s", SEARCH_VERSION_STR, __DATE__, __TIME__);
pthread_mutex_init(&mutex, NULL);
listNodePool.SetMemInfo("listnode", sizeof(ListNode));
skipNodePool.SetMemInfo("skipnode", sizeof(SkipListNode));
if (SIG_ERR == signal(SIGTERM, catch_signal))
@ -142,11 +138,6 @@ static int ServicePreRun(int log_level, bool deam, string log_path)
return -RT_INIT_ERR;
}
if (!TernaryTree::Instance()->Init(global_cfg.suggestPath)){
log_error("int suggest words error");
return -RT_INIT_ERR;
}
CConfig* DTCCacheConfig = new CConfig();
if (cache_file == NULL || DTCCacheConfig->ParseConfig(cache_file, "search_cache")) {
log_error("no cache config or config file [%s] is error", cache_file);
@ -177,7 +168,6 @@ void ServicePostRun(string str_pid_file) {
SearchConf::Instance()->Destroy();
DataManager::Instance()->Destroy();
SplitManager::Instance()->Destroy();
TernaryTree::Instance()->Destroy();
search_delete_pid(str_pid_file);
}

8
src/search_local/index_read/index_sync/sequence_search_index.cc
View File

@ -403,16 +403,10 @@ vector<string> split(const string& str, const string& delim) {
}
std::string getPath(const char *bind_addr){
string str = bind_addr;
vector<string> res = split(str, ":");
res = split(res[1], "/");
string s = "/tmp/domain_socket/rocks_direct_";
s.append(res[0]);
s.append(".sock");
string s = "/tmp/domain_socket/rocks_direct_20000.sock";
return s;
}
void SearchRocksDBIndex::buildTableFieldMap(CConfig* _DTCTableConfig){
int fieldCount = _DTCTableConfig->GetIntVal("TABLE_DEFINE", "FieldCount", 0);

1
src/search_local/index_read/index_tbl_op.h
View File

@ -23,7 +23,6 @@
#include "chash.h"
#include "search_conf.h"
#include "search_list.h"
#include "search_util.h"
#include "json/value.h"
#include <stdio.h>

813
src/search_local/index_read/lirs_cache.cc
View File

@ -1,813 +0,0 @@
/*
* =====================================================================================
*
* Filename: lirs_cache.h
*
* Description: lirs cache class definition.
*
* Version: 1.0
* Created: 22/04/2019 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "lirs_cache.h"
#include "log.h"
#include <assert.h>
#define OPEN_SYNTAX_CHECK
#define REMOVE_LIR_WHEN_STACK_FULL // the replacement policy when the stack if full
LirsCache::LirsStack::LirsStack(
const int maxLirNum,
const int maxStackSize)
:
mMaxLirEntryNum(maxLirNum),
mMaxStackSize(maxStackSize),
mCurrLirEntryNum(0),
mCurrStackSize(0),
mStackBottom(NULL),
mStackTop(NULL)
{
}
LirsCache::LirsStack::~LirsStack()
{
LirsEntry_t *prev, *curr = mStackBottom;
while (curr)
{
prev = curr;
curr = curr->sStackNext;
if (prev->sEntryState & HIR_BLOCK_SHARED)
prev->sEntryState &= ~HIR_BLOCK_SHARED;
else
delete prev;
}
}
void
LirsCache::LirsStack::removeEntry(
LirsEntry_t *entry,
std::map<std::string, LirsEntry_t*> &entryMap,
const bool releaseEntry)
{
if (!entry || !(entry->sEntryState & HIR_BLOCK_ONSTACK))
{
log_error("internal error, entryEmpty:%d.", !entry);
return;
}
if (!entry->sStackPrev)
{
assert(entry == mStackBottom);
mStackBottom = entry->sStackNext;
if (!mStackBottom) mStackTop = NULL;
else mStackBottom->sStackPrev = NULL;
}
else if (!entry->sStackNext)
{
assert(entry == mStackTop);
mStackTop = entry->sStackPrev;
if (!mStackTop) mStackBottom = NULL;
else mStackTop->sStackNext = NULL;
}
else
{
assert(entry != mStackBottom && entry != mStackTop);
entry->sStackPrev->sStackNext = entry->sStackNext;
entry->sStackNext->sStackPrev = entry->sStackPrev;
}
char &state = entry->sEntryState;
bool canRelease = (releaseEntry && !(state & HIR_BLOCK_SHARED));
if (state & LIR_BLOCK) mCurrLirEntryNum--;
state &= (~HIR_BLOCK_ONSTACK & ~HIR_BLOCK_SHARED & ~LIR_BLOCK);
mCurrStackSize--;
if (canRelease)
{
log_info("remove entry, key:%s", entry->sKey.c_str());
entryMap.erase(entry->sKey);
delete entry;
entry = NULL;
}
else
{
entry->sStackPrev = entry->sStackNext = NULL;
}
return;
}
// 1.when call this function, must be has enough space for the appending entry
// 2.the entry must be a non-exist entry in the stack
void
LirsCache::LirsStack::appendEntry(LirsEntry_t *entry)
{
if (!entry)
{
log_error("append empty entry.");
assert(false);
return;
}
char &state = entry->sEntryState;
if (state < 0 || (mCurrLirEntryNum >= mMaxLirEntryNum && (state & LIR_BLOCK)))
{
log_error("no enough space for the Lir entry");
assert(false);
return;
}
if (mCurrStackSize >= mMaxStackSize)
{
log_error("no enough space for the Hir entry");
assert(false);
return;
}
// has enough space, append it
if (!mStackTop)
{
// the first one
mStackTop = mStackBottom = entry;
entry->sStackPrev = NULL;
entry->sStackNext = NULL;
}
else
{
// append to the behind of the top entry
mStackTop->sStackNext = entry;
entry->sStackPrev = mStackTop;
mStackTop = entry;
mStackTop->sStackNext = NULL;
}
if (state & LIR_BLOCK) mCurrLirEntryNum++;
mCurrStackSize++;
state |= HIR_BLOCK_ONSTACK;
if (state & (HIR_BLOCK_ONQUEUE | HIR_RESIDENT_BLOCK)) state |= HIR_BLOCK_SHARED;
return;
}
// evicted all HIR blocks those located in the bottom of stack
void
LirsCache::LirsStack::stackPrune(std::map<std::string, LirsEntry_t*> &entryMap)
{
if (!mStackBottom) return;
while (mStackBottom)
{
if (mStackBottom->sEntryState & LIR_BLOCK) break;
removeEntry(mStackBottom, entryMap);
}
return;
}
// release one hir block from the bottom of the stack
void
LirsCache::LirsStack::releaseOneHirEntry(std::map<std::string, LirsEntry_t*> &entryMap)
{
if (!mStackBottom) return;
LirsEntry_t *curr = mStackBottom->sStackNext;
while (curr)
{
if (curr->sEntryState & LIR_BLOCK) curr = curr->sStackNext;
// remove the entry
removeEntry(curr, entryMap, true);
break;
}
return;
}
///////////////////////////////////////////////////////////
// Lirs queue relevant
///////////////////////////////////////////////////////////
LirsCache::LirsQueue::LirsQueue(const int maxQueueSize)
:
mMaxQueueSize(maxQueueSize),
mCurrQueueSize(0),
mQueueHead(NULL),
mQueueTail(NULL)
{
}
LirsCache::LirsQueue::~LirsQueue()
{
LirsEntry_t *prev, *curr = mQueueHead;
while (curr)
{
prev = curr;
curr = curr->sQueueNext;
if (prev->sEntryState & HIR_BLOCK_SHARED)
prev->sEntryState &= ~HIR_BLOCK_SHARED;
else
delete prev;
}
}
// evicted the resident HIR block from the queue
// use flag 'release' to forbidden the caller to release the entry
// if someone holding it current now
void
LirsCache::LirsQueue::removeEntry(
LirsEntry_t *entry,
std::map<std::string, LirsEntry_t*> &entryMap,
const bool release)
{
if (!entry)
{
log_error("can not remove an empty entry.");
return;
}
char &state = entry->sEntryState;
if (!(state & HIR_RESIDENT_BLOCK))
{
assert(false);
log_error("incorrect entry state.");
return;
}
if (!entry->sQueuePrev)
{
mQueueHead = entry->sQueueNext;
if (!mQueueHead) mQueueTail = NULL;
else mQueueHead->sQueuePrev = NULL;
}
else if (!entry->sQueueNext)
{
mQueueTail = entry->sQueuePrev;
if (!mQueueTail) mQueueHead = NULL;
else mQueueTail->sQueueNext = NULL;
}
else
{
entry->sQueuePrev->sQueueNext = entry->sQueueNext;
entry->sQueueNext->sQueuePrev = entry->sQueuePrev;
}
// double check
if (release && !(state & HIR_BLOCK_ONSTACK) && !(state & HIR_BLOCK_SHARED))
{
log_info("remove entry, key:%s", entry->sKey.c_str());
entryMap.erase(entry->sKey);
delete entry;
entry = NULL;
}
else
{
// clear flag
entry->sQueuePrev = entry->sQueueNext = NULL;
state &= (~HIR_BLOCK_ONQUEUE & ~HIR_BLOCK_SHARED & ~HIR_RESIDENT_BLOCK);
}
mCurrQueueSize--;
return;
}
// when call this function, queue should has enough remaining space for appending
void
LirsCache::LirsQueue::appendEntry(LirsEntry_t *entry)
{
if (!entry || (entry->sEntryState & LIR_BLOCK))
{
log_error("empty entry:%d.", entry == NULL);
assert(false);
return;
}
char &state = entry->sEntryState;
if (state < 0 || mCurrQueueSize >= mMaxQueueSize)
{
log_error("incorrect queue data.");
return;
}
// just append to the tail directly
if (!mQueueTail)
{
mQueueHead = mQueueTail = entry;
mQueueHead->sQueuePrev = NULL;
mQueueTail->sQueueNext = NULL;
}
else
{
mQueueTail->sQueueNext = entry;
entry->sQueuePrev = mQueueTail;
mQueueTail = entry;
mQueueTail->sQueueNext = NULL;
}
mCurrQueueSize++;
state |= (HIR_BLOCK_ONQUEUE | HIR_RESIDENT_BLOCK);
state &= ~LIR_BLOCK;
if (state & HIR_BLOCK_ONSTACK) state |= HIR_BLOCK_SHARED;
return;
}
///////////////////////////////////////////////////////////
// LIRS cache relevant
///////////////////////////////////////////////////////////
LirsCache::LirsCache(const int cacheSize)
:
mCacheEntrySize(cacheSize)
{
if (mCacheEntrySize < eMinCacheEntrySize || mCacheEntrySize > eMaxCacheEntrySize)
mCacheEntrySize = mCacheEntrySize < eMinCacheEntrySize ? eMinCacheEntrySize : mCacheEntrySize;
int queueSize = mCacheEntrySize * eQueueSizeRate / 100;
if (queueSize < eMinQueueSize) queueSize = eMinQueueSize;
int maxLirEntryNum = mCacheEntrySize - queueSize;
int maxStackSize = mCacheEntrySize + queueSize; // the extra queue size for holding non-resident HIR blocks
// LIRS algorithm dynamically vary the stack cache size and it will not consume
// all the memory with eInfiniteCacheEntrySize cache size????
// maxStackSize = 5 * maxLirEntryNum;
maxStackSize = eInfiniteCacheEntrySize;
mBlockStack = new LirsStack(maxLirEntryNum, maxStackSize);
mBlockQueue = new LirsQueue(queueSize);
}
LirsCache::~LirsCache()
{
if (mBlockStack) delete mBlockStack;
if (mBlockQueue) delete mBlockQueue;
}
// find the key and adjust the lirs cache
LirsEntry_t*
LirsCache::findEntry(const std::string &key)
{
MapItr_t itr = mEntryMap.find(key);
if (itr == mEntryMap.end()) return NULL;
LirsEntry_t *entry = itr->second;
assert(entry != NULL);
// Note:In LIRS paper, it says that if accessing a non-resident HIR block
// in the stack(if the non-resident block in the map, it must be in the stack),
// we change its status to LIR and move it to the top of the stack,so it
// also a hit in the cache
// if (!entry || !(entry->sEntryState & HIR_RESIDENT_BLOCK)) return NULL;
// hit Lir or Resident Hir block, adjust the cache
adjustLirsCache(entry);
syntaxCheck();
return entry;
}
// 1.if exist, update the value
// 2.append a new entry
bool
LirsCache::appendEntry(
const std::string &key,
const std::string &value)
{
// find in the stack first
LirsEntry_t *entry = NULL;
MapItr_t itr = mEntryMap.find(key);
if (itr != mEntryMap.end())
{
entry = itr->second;
#if (__cplusplus >= 201103L)
// c++0x, use rvalue reference, value can not be used any more
entry->sValue = std::move(value);
#else
entry->sValue = value;
#endif // __cplusplus >= 201103L
adjustLirsCache(entry);
syntaxCheck();
log_info("update entry, key:%s, value:%s, state:%d", key.c_str(),\
value.c_str(), entry->sEntryState);
return true;
}
// append a new entry
entry = new LirsEntry_t();
if (!entry)
{
log_error("allocate memory failed.");
return false;
}
entry->initEntry(0, NULL, NULL, NULL, NULL, key, value);
char &state = entry->sEntryState;
// add into the map
mEntryMap[key] = entry;
// make sure have enough space for appending
bool isLirFull = mBlockStack->isLirEntryFull();
bool isStackFull = mBlockStack->isStackFull();
if (!isLirFull)
{
#ifndef REMOVE_LIR_WHEN_STACK_FULL
if (isStackFull) mBlockStack->releaseOneHirEntry(mEntryMap);
#else
if (isStackFull) removeStackBottom();
#endif
// add as a lir entry
state |= LIR_BLOCK;
mBlockStack->appendEntry(entry);
syntaxCheck();
log_info("append entry, key:%s, value:%s, state:%d",\
key.c_str(), value.c_str(), entry->sEntryState);
return true;
}
// add as a resident HIR block
bool isQueueFull = mBlockQueue->isQueueFull();
if (isQueueFull || isStackFull)
{
if (isQueueFull)
{
// remove resident HIR block from queue
LirsEntry_t *head = mBlockQueue->getHeadOfQueue();
mBlockQueue->removeEntry(head, mEntryMap);
}
// check whether the stack is full or not
if (isStackFull)
{
// remove the lir block in the bottom
removeStackBottom();
// append entry as a lir block
state |= LIR_BLOCK;
mBlockStack->appendEntry(entry);
syntaxCheck();
log_info("append entry, key:%s, value:%s, state:%d",\
key.c_str(), value.c_str(), entry->sEntryState);
return true;
}
}
// append to both the stack and the queue as an resident block
// state |= (HIR_RESIDENT_BLOCK | HIR_BLOCK_SHARED);
mBlockStack->appendEntry(entry);
mBlockQueue->appendEntry(entry);
assert(state == 30);
syntaxCheck();
log_info("append entry, key:%s, value:%s, state:%d",\
key.c_str(), value.c_str(), entry->sEntryState);
return true;
}
bool
LirsCache::removeEntry(const std::string &key)
{
MapItr_t itr = mEntryMap.find(key);
if (itr == mEntryMap.end()) return true;
LirsEntry_t *entry = itr->second;
char state = entry->sEntryState;
// remove from the stack
if (state & HIR_BLOCK_ONSTACK)
{
mBlockStack->removeEntry(entry, mEntryMap);
// try to conduct a pruning
mBlockStack->stackPrune(mEntryMap);
}
// remove from the queue
if (state & HIR_BLOCK_ONQUEUE)
{
mBlockQueue->removeEntry(entry, mEntryMap);
}
return true;
}
// remove the bottom LIR entry which has the maximum recency although it
// is a hot block
void
LirsCache::removeStackBottom()
{
bool isQueueFull = mBlockQueue->isQueueFull();
if (isQueueFull)
{
LirsEntry_t *head = mBlockQueue->getHeadOfQueue();
mBlockQueue->removeEntry(head, mEntryMap, true);
}
// remove bottom of the stack
LirsEntry_t *bottom = mBlockStack->getBottomOfStack();
mBlockStack->removeEntry(bottom, mEntryMap, false);
mBlockQueue->appendEntry(bottom);
mBlockStack->stackPrune(mEntryMap);
return;
}
// entry must be exist in the cache, even if it's a non-resident block
void
LirsCache::adjustLirsCache(LirsEntry_t *entry)
{
char &state = entry->sEntryState;
if (state & LIR_BLOCK)
{
// Upon accessing an LIR block X
// 1.move X to the top of stack
// 2.conduct a stack pruning if it is located in the bottom of the stack
mBlockStack->removeEntry(entry, mEntryMap, false);
// maybe the removed entry is bottom, try to conduct a stack pruning
mBlockStack->stackPrune(mEntryMap);
state |= LIR_BLOCK;
}
else
{
// Upon accessing an Hir block X
// 1.if X is an resident Hir block:
// 1.1) move it to the top of stack
// 1.2) if X is in the stack
// a) change its status to LIR, and removed it from the queue
// b) move the bottom entry in the stack to the end of queue
// c) conduct a stack pruning
// 1.3) if X is not in the stack
// a) leave its status in HIR and move it to the end of the queue
//
// 2.if X is an non-resident Hir block X:
// note:in this case, block X must be located in the stack, otherwise, wo regard
// it as a newly entry and will be append to the cache by API appendEntry
// 2.1) remove the HIR resident block that at the front of queue(if queue full)
// 2.2) append the block X on the top of stack
// 2.3) change its status to LIR
// 2.4) if the num of LIR block is bigger than maxLirEntrySize, remove
// the bottom one of the stack
if (state & HIR_RESIDENT_BLOCK)
{
// resident hir block
if (state & HIR_BLOCK_ONSTACK)
{
// evicted from queue
mBlockQueue->removeEntry(entry, mEntryMap, false);
// move the bottom entry in the stack to the end of the queue if lir entry full
bool isLirEntryFull = mBlockStack->isLirEntryFull();
if (isLirEntryFull)
{
LirsEntry_t *bottom = mBlockStack->getBottomOfStack();
mBlockStack->removeEntry(bottom, mEntryMap, false);
mBlockQueue->appendEntry(bottom);
}
// evicted myself from stack
mBlockStack->removeEntry(entry, mEntryMap, false);
mBlockStack->stackPrune(mEntryMap);
state |= LIR_BLOCK;
}
else
{
// 1.leave its status in HIR and move this block to the end of the queue
mBlockQueue->removeEntry(entry, mEntryMap, false);
mBlockQueue->appendEntry(entry);
// 2.append to the stack
bool isStackFull = mBlockStack->isStackFull();
if (isStackFull)
{
#ifndef REMOVE_LIR_WHEN_STACK_FULL
// remove the first HIR entry from stack bottom
mBlockStack->releaseOneHirEntry(mEntryMap);
#else
// remove the Lir block in the bottom of the stack
removeStackBottom();
#endif
}
}
}
else
{
// non-resident hir block, block must be in the stack, if not in the stack,
// it must be a new entry that we should call appendEntry function to add it
if (!(state & HIR_BLOCK_ONSTACK) || (state & HIR_BLOCK_ONQUEUE))
{
log_error("internal error.");
assert(false);
return;
}
bool isLirEntryFull = mBlockStack->isLirEntryFull();
if (isLirEntryFull)
{
bool isQueueFull = mBlockQueue->isQueueFull();
if (isQueueFull)
{
// remove the resident HIR block from the head of queue first
LirsEntry_t *head = mBlockQueue->getHeadOfQueue();
mBlockQueue->removeEntry(head, mEntryMap, true);
}
// move the entry in the bottom of the stack into the tail of the queue
LirsEntry_t *bottom = mBlockStack->getBottomOfStack();
mBlockStack->removeEntry(bottom, mEntryMap, false);
mBlockQueue->appendEntry(bottom);
}
// remove the entry from the stack first, then conduct stack prune
mBlockStack->removeEntry(entry, mEntryMap, false);
mBlockStack->stackPrune(mEntryMap);
state |= LIR_BLOCK;
}
}
// append this entry to the top of the stack
mBlockStack->appendEntry(entry);
return;
}
// check LIRS cache
bool LirsCache::syntaxCheck()
{
#ifndef OPEN_SYNTAX_CHECK
// do nothing
#else
int stackBlockNum = 0;
int stackLirBlockNum = 0;
int stackRHirBlockNum = 0;
int stackNRHirBlockNum = 0;
int queueSharedBlockNum = 0;
// check stack
if (mBlockStack)
{
LirsEntry_t *bottom = mBlockStack->getBottomOfStack();
LirsEntry_t *top = mBlockStack->getTopOfStack();
char state;
LirsEntry_t *prev = NULL, *curr = bottom;
while (curr)
{
state = curr->sEntryState;
if (state <= 0 ||
state > (HIR_BLOCK_SHARED + HIR_BLOCK_ONQUEUE + HIR_BLOCK_ONSTACK + HIR_RESIDENT_BLOCK))
{
log_error("incorrect entry state.");
assert(false);
return false;
}
if (!(state & HIR_BLOCK_ONSTACK))
{
log_error("incorrect LIR block state. state:%d", state);
assert(false);
return false;
}
stackBlockNum++;
if (state & LIR_BLOCK)
{
if ((state & HIR_RESIDENT_BLOCK)
|| (state & HIR_BLOCK_ONQUEUE)
|| (state & HIR_BLOCK_SHARED))
{
log_error("incorrect LIR block. state:%d", state);
assert(false);
return false;
}
stackLirBlockNum++;
}
else if (state & HIR_RESIDENT_BLOCK)
{
if (!(state & HIR_BLOCK_ONQUEUE)
|| !(state & HIR_BLOCK_SHARED))
{
log_error("incorrect LIR block. state:%d", state);
assert(false);
return false;
}
stackRHirBlockNum++;
}
else
{
if ((state & HIR_BLOCK_ONQUEUE)
|| (state & HIR_BLOCK_SHARED))
{
log_error("incorrect LIR block. state:%d", state);
assert(false);
return false;
}
stackNRHirBlockNum++;
}
prev = curr;
curr = curr->sStackNext;
if (curr && prev != curr->sStackPrev)
{
log_error("incorrect double link.");
assert(false);
return false;
}
}
assert(prev == top);
}
// check cache size
if (stackRHirBlockNum > mBlockQueue->getCurrQueueSize())
{
log_error("check RHir block failed.");
assert(false);
return false;
}
// check queue
if (mBlockQueue)
{
LirsEntry_t *head = mBlockQueue->getHeadOfQueue();
LirsEntry_t *tail = mBlockQueue->getTailOfQueue();
char state;
LirsEntry_t *prev = NULL, *curr = head;
while (curr)
{
state = curr->sEntryState;
if (state <= 0 ||
state > (HIR_BLOCK_SHARED + HIR_BLOCK_ONQUEUE + HIR_BLOCK_ONSTACK + HIR_RESIDENT_BLOCK))
{
log_error("incorrect entry state.");
assert(false);
return false;
}
if (!(state & HIR_BLOCK_ONQUEUE) || !(state & HIR_RESIDENT_BLOCK))
{
log_error("incorrect Resident HIR block state. state:%d", state);
assert(false);
return false;
}
if (state & LIR_BLOCK)
{
log_error("incorrect Resident HIR block state. state:%d", state);
assert(false);
return false;
}
if (state & HIR_BLOCK_ONSTACK)
{
if (!(state & HIR_BLOCK_SHARED))
{
log_error("incorrect Resident HIR block state. state:%d", state);
assert(false);
return false;
}
queueSharedBlockNum++;
}
prev = curr;
curr = curr->sQueueNext;
if (curr && prev != curr->sQueuePrev)
{
log_error("incorrect double link.");
assert(false);
return false;
}
}
assert(prev == tail);
}
if (stackRHirBlockNum != queueSharedBlockNum)
{
log_error("shared pointer occur error.");
assert(false);
return false;
}
#endif
return true;
}

178
src/search_local/index_read/lirs_cache.h
View File

@ -1,178 +0,0 @@
/*
* =====================================================================================
*
* Filename: lirs_cache.h
*
* Description: lirs cache class definition.
*
* Version: 1.0
* Created: 22/04/2019 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef LIRS_CACHE_H__
#define LIRS_CACHE_H__
#include <stdint.h>
#include <map>
#include <string>
#include <utility>
// LIRS use two data structure to hold all cache data, LIRS stack and queue.
// Data to be described as hot data and cold data, hot data names LIR(low IRR)
// and cold data is HIR(high IRR), all LIR data are located in the LIRS stack and
// the others located either in the stack or queue; The HIR data also be
// divided into resident and non-resident, all resident HIR data are linked
// to be a small size queue
#define LIR_BLOCK 1 // Hot data
#define HIR_RESIDENT_BLOCK 2 // HIR is cold data
#define HIR_BLOCK_ONSTACK 4
#define HIR_BLOCK_ONQUEUE 8
#define HIR_BLOCK_SHARED 16 // shared Resident HIR entry reference between Stack and Queue
// 1.Unfixed data type(include either key or value):
// unsigned long long, float, double, string
// 2.Except 1, others is fixed, such as the following:
// char, short, int, the unsigned series that size is small than 8, and so on
// #define HIR_BLOCK_FIXED 32
typedef struct LirsEntry
{
char sEntryState;
// 1.we assume that the value is big enough, so the space cost in using shared entry
// mechanism(two double link pointer) will be cheaper than clone the same entry
// 2.use shared ptr let us implement the LRU cache with only one Hashmap
struct LirsEntry *sStackPrev;
struct LirsEntry *sStackNext;
struct LirsEntry *sQueuePrev;
struct LirsEntry *sQueueNext;
std::string sKey;
std::string sValue;
void initEntry(
const char state,
struct LirsEntry *sPrev,
struct LirsEntry *sNext,
struct LirsEntry *qPrev,
struct LirsEntry *qNext,
const std::string &key,
const std::string &value)
{
sEntryState = state;
sStackPrev = sPrev;
sStackNext = sNext;
sQueuePrev = qPrev;
sQueueNext = qNext;
#if (__cplusplus >= 201103L)
sKey = std::move(key);
sValue = std::move(value);
#else
sKey = key;
sValue = value;
#endif
}
}LirsEntry_t;
class LirsCache
{
private:
enum CacheRelevant
{
eQueueSizeRate = 1, // 1%
eMinQueueSize = 2,
// the minimum of the queue size is must gt 2 in case of the
// latest additional HIR block is removed due to the removeStackBottom
// operation
eMinCacheEntrySize = 100,
eMaxCacheEntrySize = 500000,
eInfiniteCacheEntrySize = 0x7FFFFFF
};
private:
typedef std::map<std::string, LirsEntry_t*>::iterator MapItr_t;
class LirsStack
{
private:
int mMaxLirEntryNum; // Maximum LIR entry number
int mMaxStackSize; // maximum real stack capacity, contain LIR + resident HIR + non-resident blocks
int mCurrLirEntryNum;
int mCurrStackSize;
LirsEntry_t* mStackBottom;
LirsEntry_t* mStackTop;
public:
LirsStack(const int maxLir, const int maxStackSize);
virtual ~LirsStack();
inline LirsEntry_t* getBottomOfStack() { return mStackBottom; }
inline LirsEntry_t* getTopOfStack() { return mStackTop; }
inline bool isLirEntryFull() { return mCurrLirEntryNum >= mMaxLirEntryNum; }
inline bool isStackFull() { return mCurrStackSize >= mMaxStackSize; }
void stackPrune(std::map<std::string, LirsEntry_t*> &entryMap);
void releaseOneHirEntry(std::map<std::string, LirsEntry_t*> &entryMap);
void appendEntry(LirsEntry_t *entry);
void removeEntry(
LirsEntry_t *entry,
std::map<std::string, LirsEntry_t*> &entryMap,
const bool releaseEntry = true);
};
class LirsQueue
{
private:
int mMaxQueueSize; // Maximum resident HIR entry number
int mCurrQueueSize;
LirsEntry_t *mQueueHead;
LirsEntry_t *mQueueTail;
public:
LirsQueue(const int maxQueueSize);
virtual ~LirsQueue();
inline LirsEntry_t* getHeadOfQueue() { return mQueueHead; }
inline LirsEntry_t* getTailOfQueue() { return mQueueTail; }
inline bool isQueueFull() { return mCurrQueueSize >= mMaxQueueSize; }
inline int getCurrQueueSize() { return mCurrQueueSize; }
void appendEntry(LirsEntry_t *entry);
void removeEntry(
LirsEntry_t *entry,
std::map<std::string, LirsEntry_t*> &entryMap,
const bool releaseEntry = true);
};
public:
explicit LirsCache(const int cacheSize = eMaxCacheEntrySize);
virtual ~LirsCache();
LirsEntry_t* findEntry(const std::string &key);
// user convert all basic data type to string
bool appendEntry(const std::string &key, const std::string &value);
bool removeEntry(const std::string &key);
private:
void removeStackBottom();
void adjustLirsCache(LirsEntry_t * entry);
bool syntaxCheck();
private:
int mCacheEntrySize; // LIR and resident HIR block nums
LirsStack* mBlockStack; // store all LIR blocks and some HIR blocks
LirsQueue* mBlockQueue; // store all resident HIR blocks
std::map<std::string, LirsEntry_t*> mEntryMap; // store all cache data for efficient search
friend class LirsStack;
friend class LirsQueue;
};
#endif // LIRS_CACHE_H__

161
src/search_local/index_read/search_env.cc
View File

@ -1,161 +0,0 @@
/*
* =====================================================================================
*
* Filename: search_env.h
*
* Description: search env class definition.
*
* Version: 1.0
* Created: 16/03/2018
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdlib.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include "log.h"
#include "search_env.h"
#include <arpa/inet.h>
#include "comm.h"
int SearchEnv::cpa_atoi(char *line, size_t n) {
int value;
if (n == 0) {
return -RT_PARA_ERR;
}
for (value = 0; n--; line++) {
if (*line < '0' || *line > '9') {
return -RT_PARA_ERR;
}
value = value * 10 + (*line - '0');
}
if (value < 0) {
return -RT_PARA_ERR;
}
return value;
}
char *SearchEnv::UnresolveAddr(struct sockaddr *addr, int addrlen)
{
static char unresolve[NI_MAXHOST + NI_MAXSERV];
static char host[NI_MAXHOST], service[NI_MAXSERV];
int status;
status = getnameinfo(addr, addrlen, host, sizeof(host), service,
sizeof(service), NI_NUMERICHOST | NI_NUMERICSERV);
if (status < 0)
{
log_error("get sock name err, errno %d, errmsg %s", status, gai_strerror(status));
snprintf(unresolve, sizeof(unresolve), "unknown");
return unresolve;
}
snprintf(unresolve, sizeof(unresolve), "%s:%s", host, service);
log_error("addr:%s", unresolve);
return unresolve;
}
char *SearchEnv::UnresolveDesc(int sd)
{
static char unresolve[NI_MAXHOST + NI_MAXSERV];
struct sockaddr *addr;
socklen_t addrlen = sizeof(struct sockaddr);
int status;
addr = (struct sockaddr *)malloc(sizeof(struct sockaddr));
if (addr == NULL)
{
log_error("no mem");
snprintf(unresolve, sizeof(unresolve), "unknown");
return unresolve;
}
status = getsockname(sd, addr, &addrlen);
if (status < 0) {
log_error("get sock name err, errno %d, errmsg %s", errno, strerror(errno));
snprintf(unresolve, sizeof(unresolve), "unknown");
return unresolve;
}
sockaddr_in sin;
memcpy(&sin, addr, sizeof(sin));
return UnresolveAddr(addr, addrlen);
}
int SearchEnv::WriteSocketEnv(int fd)
{
int len = 0;
memcpy(os_environ, NC_ENV_FDS, strlen(NC_ENV_FDS));
len = strlen(NC_ENV_FDS);
memcpy(os_environ + len, "=", strlen("="));
len += strlen("=");
sprintf(os_environ + len, "%d;", fd);
log_debug("global env is %s", os_environ);
return 0;
}
int SearchEnv::ReadSocketEnv(struct sockaddr *addr, int addrlen)
{
char *listen_address = UnresolveAddr(addr, addrlen);
char *p, *q;
int sock;
char *inherited;
inherited = getenv(NC_ENV_FDS);
char address[NI_MAXHOST + NI_MAXSERV];
if (inherited == NULL) {
return 0;
}
strncpy(address, listen_address, sizeof(address));
log_debug("address %s", address);
log_debug("inherited %s", inherited);
for (p = inherited, q = inherited; *p; p++) {
if (*p == ';') {
sock = cpa_atoi(q, p - q);
if (strcmp(address, UnresolveDesc(sock)) == 0) {
log_debug("get inherited socket %d for '%s' from '%s'", sock,
address, inherited);
sock = dup(sock);
log_debug("dup inherited socket as %d", sock);
return sock;
}
q = p + 1;
}
}
return 0;
}
int SearchEnv::CleanSocketEnv()
{
int sock = 0;
char *inherited;
char *p, *q;
inherited = getenv(NC_ENV_FDS);
if (inherited == NULL) {
return 0;
}
for (p = inherited, q = inherited; *p; p++) {
if (*p == ';') {
sock = cpa_atoi(q, p - q);
close(sock);
q = p + 1;
}
}
return 0;
}

47
src/search_local/index_read/search_env.h
View File

@ -1,47 +0,0 @@
/*
* =====================================================================================
*
* Filename: search_env.h
*
* Description: search env class definition.
*
* Version: 1.0
* Created: 16/03/2018
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __SEARCH_ENV_H__
#define __SEARCH_ENV_H__
#include "singleton.h"
class SearchEnv {
public:
SearchEnv() {
NC_ENV_FDS = "NC_ENV_FDS";
}
static SearchEnv *Instance() {
return CSingleton<SearchEnv>::Instance();
}
static void Destroy() {
CSingleton<SearchEnv>::Destroy();
}
int WriteSocketEnv(int fd);
int ReadSocketEnv(struct sockaddr *addr, int addrlen);
int CleanSocketEnv();
char *UnresolveDesc(int sd);
char *UnresolveAddr(struct sockaddr *addr, int addrlen);
char *GetEnv() {
return os_environ;
}
private:
int cpa_atoi(char *line, size_t n);
char os_environ[128];
const char *NC_ENV_FDS;
};
#endif

331
src/search_local/index_read/search_list.cc
View File

@ -1,331 +0,0 @@
/*
* =====================================================================================
*
* Filename: search_list.h
*
* Description: search list class definition.
*
* Version: 1.0
* Created: 15/08/2019
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <malloc.h>
#include <string.h>
#include <string>
#include "log.h"
#include "search_list.h"
MemPool listNodePool;
bool SearchList::InsertNode(ListNode *&list, const char *str)
{
if (str == NULL || strlen(str) == 0) {
log_error("str is empty!");
return false;
}
if (strlen(str) >= MAX_DOCID_LENGTH) {
log_error("the length of str : %s is too long", str);
return false;
}
if (list == NULL) {
list = (ListNode*)listNodePool.PoolAlloc();
if (list == NULL) {
log_error("allocate listnode error,no more memory!");
return false;
}
memset(list->key, 0, MAX_DOCID_LENGTH);
list->prev = list->next = list->forward = list->backward = NULL;
list->forward = (ListNode*)listNodePool.PoolAlloc();
if (list->forward == NULL) {
log_error("allocate listnode error,no more memory!");
listNodePool.PoolFree(list);
list = NULL;
return false;
}
memset(list->forward->key, 0, MAX_DOCID_LENGTH);
list->forward->prev = list->forward->next = list->forward->forward = list->forward->backward = NULL;
list->forward->backward = list;
strncpy(list->forward->key, str, strlen(str));
return true;
}
ListNode *node = NULL;
ListNode *insertNode = list;
while (insertNode->forward != NULL && strcmp(str, insertNode->forward->key) > 0)
insertNode = insertNode->forward;
if (insertNode->forward != NULL && strcmp(str, insertNode->forward->key) == 0) {
return true;
}
if (insertNode->forward == NULL) {
node = (ListNode*)listNodePool.PoolAlloc();
if (node == NULL){
log_error("allocate listnode error,no more memory!");
return false;
}
memset(node->key, 0, MAX_DOCID_LENGTH);
node->prev = node->next = node->forward = node->backward = NULL;
strncpy(node->key, str, strlen(str));
insertNode->forward = node;
node->backward = insertNode;
return true;
}
node = (ListNode*)listNodePool.PoolAlloc();
if (node == NULL){
log_error("allocate listnode error,no more memory!");
return false;
}
memset(node->key, 0, MAX_DOCID_LENGTH);
node->prev = node->next = node->forward = node->backward = NULL;
node->forward = insertNode->forward;
node->backward = insertNode;
insertNode->forward->backward = node;
insertNode->forward = node;
strncpy(node->key, str, strlen(str));
return true;
}
void SearchList::InsertListNode(ListNode *list)
{
if (list == NULL || list->forward == NULL) {
return ;
}
if (size == 0)
{
list->prev = list;
list->next = list;
head = tail = list;
} else if (strcmp(list->forward->key, head->forward->key) <= 0) {
list->next = head;
list->prev = tail;
head->prev = list;
tail->next = list;
head = list;
} else if (strcmp(list->forward->key, tail->forward->key) >= 0) {
list->prev = tail;
list->next = head;
tail->next = list;
head->prev = list;
tail = list;
} else {
ListNode *tmp = head;
while (strcmp(list->forward->key, tmp->forward->key) > 0)
tmp = tmp->next;
list->prev = tmp->prev;
list->next = tmp;
tmp->prev->next = list;
tmp->prev = list;
}
size = size + 1;
return ;
}
void SearchList::GetIntersectionSets()
{
if (size == 0 || size == 1)
return ;
bool res = false;
ListNode *list = Intersection(res); //res为false代表内存不够，不能新增链表节点
if (!res) {
DestroyNode(list);
DestroyList();
list = NULL;
return ;
}
DestroyList();
InsertListNode(list);
return ;
}
ListNode* SearchList::Intersection(bool& res)
{
ListNode *newList = NULL; //把得到的交集放入newList中，进而销毁原链表，把newList插入链表，此时链表只有newList一个链
char str[MAX_DOCID_LENGTH];
int flag = 0;
int count = 0;
while(1) {
if (count == 0){
memset(str, 0, MAX_DOCID_LENGTH);
strcpy(str, tail->forward->key);
} else if(tail->forward && tail->forward->forward) {
memset(str, 0, MAX_DOCID_LENGTH);
strcpy(str, tail->forward->forward->key);
} else {
return newList;
}
ListNode* list = head;
while (list) {
if (list->forward == NULL){
return newList;
}
flag = ListKeyCompare(list, str);
if (flag == -1) {
log_debug("can't find value str : %s", str);
return newList;
} else if (flag == 1){
res = InsertNode(newList, str);
if (!res)
return newList;
break;
}
list = list->next;
}
count = count + 1;
}
return newList;
}
/*
返回1代表遍历结束，说明所有list均包含str字符串
返回-1代表该list中没有与str匹配的值，找不到交集，遍历结束
返回0代表该list中发现了与str匹配的值
*/
int SearchList::ListKeyCompare(ListNode* list, char *str)
{
if (!(strcmp(list->forward->key, str) < 0)) {
return 1;
}
ListNode* tmp = list->forward;
while (tmp != NULL && strcmp (tmp->key, str) < 0) {
ListNode *del = tmp;
tmp = tmp->forward;
ListNodeDelete(del);
del = NULL;
}
if (tmp == NULL) {
return -1;
}
memset(str, 0, MAX_DOCID_LENGTH);
strcpy(str, tmp->key); // 把tmp中的key更新str，由遍历结果得知该tmp中的key是大于或者等于str的
return 0;
}
bool SearchList::IsContainKey(ListNode* keyList, const char* str)
{
int res;
if (keyList == NULL || keyList->forward == NULL)
return false;
ListNode* tmp = keyList->forward;
while (tmp != NULL) {
res = strcmp(tmp->key, str);
if (res > 0)
return false;
else if (res == 0)
return true;
tmp = tmp->forward;
}
return false;
}
void SearchList::GetDifferenceList(ListNode* keyList)
{
if (size != 1 || keyList == NULL || head->forward == NULL)
return ;
ListNode *node = head->forward;
ListNode *temp = node->forward;
while (node != NULL)
{
if (IsContainKey(keyList, node->key))
ListNodeDelete(node);
node = temp;
if (node == NULL)
break;
temp = node->forward;
}
/*防止在删除节点的时候，head链表只剩一个头结点*/
if (head && head->forward == NULL) {
listNodePool.PoolFree(head);
head = tail = NULL;
size = 0;
}
return ;
}
void SearchList::GetListElement(vector<string>& eles)
{
if (size == 0){
log_debug("search list is empty");
return ;
}
ListNode *node = head->forward;
while (node != NULL) {
eles.push_back(node->key);
node = node->forward;
}
return ;
}
void SearchList::ListNodeDelete(ListNode *node) {
if (node == NULL) {
log_debug("node is NULL, can't delete");
return ;
}
if (node->forward == NULL) {
node->backward->forward = NULL;
listNodePool.PoolFree(node);
node = NULL;
return ;
}
node->backward->forward = node->forward;
node->forward->backward = node->backward;
listNodePool.PoolFree(node);
node = NULL;
return ;
}
void SearchList::DestroyList()
{
if (head == NULL && tail == NULL)
return ;
ListNode *list = head;
ListNode *tmp;
while (list != tail) {
tmp = list->next;
DestroyNode(list);
list = tmp;
}
DestroyNode(list);
head = NULL;
tail = NULL;
size = 0;
}
void SearchList::DestroyNode(ListNode *list)
{
if (list == NULL)
return ;
ListNode *tmp = list;
ListNode *li;
while(tmp != NULL)
{
li = tmp->forward;
listNodePool.PoolFree(tmp);
tmp = li;
}
tmp = NULL;
li = NULL;
}

64
src/search_local/index_read/search_list.h
View File

@ -1,64 +0,0 @@
/*
* =====================================================================================
*
* Filename: search_list.h
*
* Description: search list class definition.
*
* Version: 1.0
* Created: 15/08/2019
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef SEARCH_LIST_H_H_
#define SEARCH_LIST_H_H_
#include "mem_pool.h"
#include "comm.h"
#include <vector>
#include <string>
using namespace std;
struct ListNode
{
char key[MAX_DOCID_LENGTH];
ListNode *prev;
ListNode *next;
ListNode *forward;
ListNode *backward;
};
class SearchList
{
public:
SearchList():head(NULL),tail(NULL),size(0){}
~SearchList(){
DestroyList();
}
bool InsertNode(ListNode *&list, const char *str);
void InsertListNode(ListNode *list);
void GetIntersectionSets();
ListNode* Intersection(bool& res);
int ListKeyCompare(ListNode* list, char *str);
void ListNodeDelete(ListNode *node);
int GetSize() {return size;}
ListNode* GetHeadList() {return head;}
bool IsContainKey(ListNode* keyList, const char* str);
void GetDifferenceList(ListNode* keyList);
void GetListElement(vector<string> &eles);
void DestroyList();
void DestroyNode(ListNode *list);
private:
ListNode *head;
ListNode *tail;
int size;
};
#endif