mirrors/isearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add resource file / add Dockerfile

Browse Source
This commit is contained in:
shzhulin3 2021-09-14 17:54:31 +08:00
parent a87f83ef69
commit 559a9a9b65
148 changed files with 1207723 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -17,6 +17,6 @@ endif()
add_subdirectory(src/comm)
add_subdirectory(src/comm/stat)
add_subdirectory(src/search_local/index_write)
add_subdirectory(src/search_local/index_read)
add_subdirectory(src/search_agent)
#add_subdirectory(src/search_local/index_write)
#add_subdirectory(src/search_local/index_read)
#add_subdirectory(src/search_agent)

36
README.md
View File

@ -156,27 +156,23 @@ fields：返回指定字段值，多个字段用逗号隔开
### 源码编译
运行build.sh脚本可以编译isearch需要的所有bin文件，编译环境说明如下：
建议通过isearch_env镜像进行源码编译，获取isearch_env镜像的方式为： `docker pull intelligentsearch/isearch_env:2.0`
由于检索存储层依赖rocksdb，所以编译时需要满足以下前置配置：
1）CentOS 7.x
2）gcc 4.8
3）Cmake版本需要大于等于3.6.2
4）安装gflags：
gflags是google开源的一套命令行参数解析工具，支持从环境变量和配置文件读取参数
安装命令：
```
git clone https://github.com/gflags/gflags.git
cd gflags
git checkout -b 2.2 v2.2.2
cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON -DGFLAGS_NAMESPACE=google -G "Unix Makefiles" .
make && make install
sudo ldconfig
sudo ln -s /usr/local/lib/libgflags.so.2.2 /lib64
```
安装后，需要将gflags的包含路径添加到PATH环境变量中
5）安装rocksdb依赖库：zlib，bzip2，lz4，snappy，zstandard
`sudo yum install -y snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan openssl-devel`
也可以自行编译isearch_env镜像，Dockerfile文件位于dockerfiles\env目录： `docker build -t intelligentsearch/isearch_env:2.0 .`
然后运行容器： `docker run -itd intelligentsearch/isearch_env:2.0`
进入容器： `docker exec -it 容器id /bin/bash`
执行如下命令进行编译和安装：
```
cd /usr/local
git clone https://gitee.com/jd-platform-opensource/isearch.git
mv isearch jdisearch
cd jdisearch
sh build.sh
sh install.sh
```
### 项目成员

9
build.sh
View File

@ -35,8 +35,8 @@ src_common="$srcdir/$common"
src_stat="$srcdir/$common/$stat"
src_index_write="$srcdir/$search_local/$index_write"
src_index_read="$srcdir/$search_local/$index_read"
src_search_agent="$srcdir/$search_agent"
src_index_storage="$srcdir/$search_local/$index_storage"
src_search_agent="$srcdir/$search_agent"
cd $src_common
cmake .
@ -48,6 +48,9 @@ cmake .
make
cd $localdir
cd $src_index_storage
make
cd $src_index_write
cmake .
make
@ -61,6 +64,4 @@ cd $localdir
cd $src_search_agent
cmake .
make
cd $src_index_storage
make
cd $localdir

7
clean.sh
View File

@ -10,13 +10,11 @@ index_write="index_write"
index_read="index_read"
index_storage="index_storage"
search_local="search_local"
search_agent="search_agent"
src_common="$srcdir/$common"
src_stat="$srcdir/$common/$stat"
src_index_write="$srcdir/$search_local/$index_write"
src_index_read="$srcdir/$search_local/$index_read"
src_search_agent="$srcdir/$search_agent"
src_index_storage="$srcdir/$search_local/$index_storage"
cd $src_common
@ -37,10 +35,5 @@ rm CMakeCache.txt Makefile cmake_install.cmake
rm -rf CMakeFiles/
rm -rf bin/
cd $src_search_agent
rm CMakeCache.txt Makefile cmake_install.cmake
rm -rf CMakeFiles/
rm -rf bin/
cd $src_index_storage
make clean

6
dockerfiles/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM intelligentsearch/isearch_env:2.0
COPY install.sh /root/install.sh
RUN /root/install.sh

32
dockerfiles/env/Dockerfile vendored Normal file
View File

@ -0,0 +1,32 @@
FROM centos:centos7.2.1511
RUN yum -y update
RUN yum install -y wget pcre pcre-devel gcc gcc-c++ make zlib-devel sudo openssh-server vim lrzsz openssl-devel &&\
yum install -y crypto-policies snappy-devel psmisc git epel-release jq && \
yum clean all && \
useradd --create-home --no-log-init --shell /bin/bash isearch && echo "isearch:isearch" | chpasswd && \
mkdir -p /env/app &&\
cd /env/app/ &&\
wget https://cmake.org/files/v3.20/cmake-3.20.0-rc3.tar.gz &&\
tar xf cmake-3.20.0-rc3.tar.gz &&\
cd /env/app/cmake-3.20.0-rc3 &&\
./bootstrap &&\
gmake &&\
gmake install &&\
cd /usr/bin &&\
ln -s cmake3 cmake &&\
cd /usr/local &&\
git clone https://github.com/facebook/rocksdb.git &&\
cd rocksdb &&\
git checkout -b 6.6.0 ad528fe5ca08dafff47d79c85abbf3e1fbb21568 &&\
make shared_lib &&\
ln -s /usr/local/rocksdb/librocksdb.so.6.6.0 /lib64/librocksdb.so.6 &&\
cd /usr/local &&\
git clone https://github.com/gflags/gflags.git &&\
cd gflags &&\
git checkout -b 2.2 v2.2.2 &&\
cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON -DGFLAGS_NAMESPACE=google -G "Unix Makefiles" . &&\
make && make install &&\
ldconfig &&\
ln -s /usr/local/lib/libgflags.so.2.2 /lib64

82
dockerfiles/install.sh Normal file
View File

@ -0,0 +1,82 @@
#!/bin/sh
cd /usr/local
git clone https://gitee.com/jd-platform-opensource/isearch.git
mv isearch jdisearch
cd jdisearch
sh build.sh
srcdir="/usr/local/jdisearch"
dstdir="/usr/local/isearch"
mkdir -p $dstdir/index_storage/inverted_index
cd $dstdir/index_storage/inverted_index
mkdir rocksdb_data bin conf
cp $srcdir/resource/index_storage/inverted_index/conf/cache.conf conf
cp $srcdir/resource/index_storage/inverted_index/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cp $srcdir/src/search_local/index_storage/rocksdb_helper/rocksdb_helper bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
mkdir -p $dstdir/index_storage/intelligent_index
cd $dstdir/index_storage/intelligent_index
mkdir rocksdb_data bin conf
cp $srcdir/resource/index_storage/intelligent_index/conf/cache.conf conf
cp $srcdir/resource/index_storage/intelligent_index/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cp $srcdir/src/search_local/index_storage/rocksdb_helper/rocksdb_helper bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
mkdir -p $dstdir/index_storage/original_data
cd $dstdir/index_storage/original_data
mkdir bin conf
cp $srcdir/resource/index_storage/original_data/conf/cache.conf conf
cp $srcdir/resource/index_storage/original_data/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
ln -s $dstdir/src/search_local/index_storage/api/c_api_cc/libdtc-gcc-4.8-r4646582.so /lib64/libdtc.so.1
cd $dstdir
mkdir index_write index_read search_agent
cd index_write
mkdir log bin stat conf
cp $srcdir/resource/index_write/conf/{index_gen.json,index_write.conf,localCluster.json} conf
cp $srcdir/resource/{app_field_define.json,character_map.txt,msr_training.utf8,phonetic_base.txt,phonetic_map.txt,stop_words.dict,words_base.txt} conf
cp $srcdir/src/search_local/index_write/bin/index_write bin
cd bin
./index_write
cd $dstdir/index_read
mkdir log bin stat conf data
cp $srcdir/resource/{app_field_define.txt,app_field_define.json,character_map.txt,en_intelligent_match.txt,en_stem.txt,en_words_base.txt} conf
cp $srcdir/resource/{intelligent_match.txt,msr_training.utf8,phonetic_base.txt,phonetic_map.txt,stop_words.dict,suggest_base.txt,words_base.txt} conf
cp $srcdir/resource/index_read/conf/{index_read.conf,cache.conf,table.conf} conf
cp $srcdir/resource/index_read/data/{analyze_data,relate_data,sensitive_data,synonym_data} data
cp $srcdir/src/search_local/index_read/bin/index_read bin
cd bin
./index_read
cd $dstdir/search_agent
mkdir log bin conf
cp $srcdir/resource/search_agent/conf/sa.conf conf
cp $srcdir/src/search_agent/bin/search_agent bin/
cd bin
./search_agent -d -c ../conf/sa.conf -v 3
yum install -y jq
cd /usr/local/isearch
mkdir tools
cp $srcdir/resource/tools/* tools
cd tools
sh load_data.sh
sh query.sh

22
docs/Quickstart.md
View File

@ -1,3 +1,25 @@
## 源码编译方式（推荐）
建议通过isearch_env镜像进行源码编译，获取isearch_env镜像的方式为： `docker pull intelligentsearch/isearch_env:2.0`
也可以自行编译isearch_env镜像，Dockerfile文件位于dockerfiles\env目录： `docker build -t intelligentsearch/isearch_env:2.0 .`
然后运行容器： `docker run -itd intelligentsearch/isearch_env:2.0`
进入容器： `docker exec -it 容器id /bin/bash`
执行如下命令进行编译和安装：
```
cd /usr/local
git clone https://gitee.com/jd-platform-opensource/isearch.git
mv isearch jdisearch
cd jdisearch
sh build.sh
sh install.sh
```
## 直接获取镜像（包含编译好的程序）
为了省去配置机器环境的麻烦，建议通过docker来运行demo容器，在终端执行如下命令：
```
docker run -d intelligentsearch/isearch:latest

86
install.sh Normal file
View File

@ -0,0 +1,86 @@
#!/bin/sh
: '
cd /usr/local
git clone https://gitee.com/jd-platform-opensource/isearch.git
#git clone https://coding.jd.com/content-store/intelligentsearch.git
mv isearch jdisearch
cd jdisearch
git checkout -b dev_zhulin origin/dev_zhulin
sh build.sh
'
srcdir="/usr/local/jdisearch"
dstdir="/usr/local/isearch"
mkdir -p $dstdir/index_storage/inverted_index
cd $dstdir/index_storage/inverted_index
mkdir rocksdb_data bin conf
cp $srcdir/resource/index_storage/inverted_index/conf/cache.conf conf
cp $srcdir/resource/index_storage/inverted_index/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cp $srcdir/src/search_local/index_storage/rocksdb_helper/rocksdb_helper bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
mkdir -p $dstdir/index_storage/intelligent_index
cd $dstdir/index_storage/intelligent_index
mkdir rocksdb_data bin conf
cp $srcdir/resource/index_storage/intelligent_index/conf/cache.conf conf
cp $srcdir/resource/index_storage/intelligent_index/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cp $srcdir/src/search_local/index_storage/rocksdb_helper/rocksdb_helper bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
mkdir -p $dstdir/index_storage/original_data
cd $dstdir/index_storage/original_data
mkdir bin conf
cp $srcdir/resource/index_storage/original_data/conf/cache.conf conf
cp $srcdir/resource/index_storage/original_data/conf/table.conf conf
cp $srcdir/resource/dtcd.sh bin
cp $srcdir/src/search_local/index_storage/cache/dtcd bin
cd bin
chmod +x dtcd.sh
./dtcd.sh start
ln -s $dstdir/src/search_local/index_storage/api/c_api_cc/libdtc-gcc-4.8-r4646582.so /lib64/libdtc.so.1
cd $dstdir
mkdir index_write index_read search_agent
cd index_write
mkdir log bin stat conf
cp $srcdir/resource/index_write/conf/{index_gen.json,index_write.conf,localCluster.json} conf
cp $srcdir/resource/{app_field_define.json,character_map.txt,msr_training.utf8,phonetic_base.txt,phonetic_map.txt,stop_words.dict,words_base.txt} conf
cp $srcdir/src/search_local/index_write/bin/index_write bin
cd bin
./index_write
cd $dstdir/index_read
mkdir log bin stat conf data
cp $srcdir/resource/{app_field_define.txt,app_field_define.json,character_map.txt,en_intelligent_match.txt,en_stem.txt,en_words_base.txt} conf
cp $srcdir/resource/{intelligent_match.txt,msr_training.utf8,phonetic_base.txt,phonetic_map.txt,stop_words.dict,suggest_base.txt,words_base.txt} conf
cp $srcdir/resource/index_read/conf/{index_read.conf,cache.conf,table.conf} conf
cp $srcdir/resource/index_read/data/{analyze_data,relate_data,sensitive_data,synonym_data} data
cp $srcdir/src/search_local/index_read/bin/index_read bin
cd bin
./index_read
cd $dstdir/search_agent
mkdir log bin conf
cp $srcdir/resource/search_agent/conf/sa.conf conf
cp $srcdir/src/search_agent/bin/search_agent bin/
cd bin
./search_agent -d -c ../conf/sa.conf -v 3
yum install -y jq
cd /usr/local/isearch
mkdir tools
cp $srcdir/resource/tools/* tools
cd tools
sh load_data.sh
sh query.sh

731
resource/app_field_define.json Normal file
View File

@ -0,0 +1,731 @@
{
"tableDefine": [{
"id": 1,
"appId": 10064,
"fieldName": "doc_id",
"fieldType": 2,
"fieldId": 1,
"IsPrimaryKey": 1,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 2,
"appId": 10064,
"fieldName": "source",
"fieldType": 1,
"fieldId": 2,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 3,
"appId": 10064,
"fieldName": "style",
"fieldType": 1,
"fieldId": 3,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 4,
"appId": 10064,
"fieldName": "sub_position",
"fieldType": 1,
"fieldId": 4,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 5,
"appId": 10064,
"fieldName": "private_status",
"fieldType": 1,
"fieldId": 5,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 6,
"appId": 10064,
"fieldName": "author_id",
"fieldType": 2,
"fieldId": 6,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 7,
"appId": 10064,
"fieldName": "status",
"fieldType": 1,
"fieldId": 7,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 8,
"appId": 10064,
"fieldName": "createtime",
"fieldType": 1,
"fieldId": 8,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 9,
"appId": 10064,
"fieldName": "title",
"fieldType": 2,
"fieldId": 10,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 11,
"appId": 10064,
"fieldName": "idx_author_title",
"fieldType": 11,
"fieldId": 11,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "6,10",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 86,
"appId": 10061,
"fieldName": "doc_id",
"fieldType": 2,
"fieldId": 0,
"IsPrimaryKey": 1,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 87,
"appId": 10061,
"fieldName": "poi_name",
"fieldType": 2,
"fieldId": 1,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 1,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 88,
"appId": 10061,
"fieldName": "gd_c1_type",
"fieldType": 2,
"fieldId": 2,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 89,
"appId": 10061,
"fieldName": "gd_c2_type",
"fieldType": 2,
"fieldId": 3,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 90,
"appId": 10061,
"fieldName": "gd_c3_type",
"fieldType": 2,
"fieldId": 4,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 91,
"appId": 10061,
"fieldName": "gd_c1_typecode",
"fieldType": 2,
"fieldId": 5,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 92,
"appId": 10061,
"fieldName": "gd_c2_typecode",
"fieldType": 2,
"fieldId": 6,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 93,
"appId": 10061,
"fieldName": "gd_c3_typecode",
"fieldType": 2,
"fieldId": 7,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 94,
"appId": 10061,
"fieldName": "std_region",
"fieldType": 2,
"fieldId": 8,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 95,
"appId": 10061,
"fieldName": "std_regionid",
"fieldType": 2,
"fieldId": 9,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 96,
"appId": 10061,
"fieldName": "std_province",
"fieldType": 2,
"fieldId": 10,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 97,
"appId": 10061,
"fieldName": "std_provinceid",
"fieldType": 2,
"fieldId": 11,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 98,
"appId": 10061,
"fieldName": "std_city",
"fieldType": 2,
"fieldId": 12,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 99,
"appId": 10061,
"fieldName": "std_cityid",
"fieldType": 2,
"fieldId": 13,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 100,
"appId": 10061,
"fieldName": "std_district",
"fieldType": 2,
"fieldId": 14,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 101,
"appId": 10061,
"fieldName": "std_districtid",
"fieldType": 2,
"fieldId": 15,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 102,
"appId": 10061,
"fieldName": "std_town",
"fieldType": 2,
"fieldId": 16,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 103,
"appId": 10061,
"fieldName": "std_townid",
"fieldType": 2,
"fieldId": 17,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 104,
"appId": 10061,
"fieldName": "jd_region",
"fieldType": 2,
"fieldId": 18,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 105,
"appId": 10061,
"fieldName": "jd_regionid",
"fieldType": 2,
"fieldId": 19,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 106,
"appId": 10061,
"fieldName": "jd_province",
"fieldType": 2,
"fieldId": 20,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 107,
"appId": 10061,
"fieldName": "jd_provinceid",
"fieldType": 2,
"fieldId": 21,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 108,
"appId": 10061,
"fieldName": "jd_city",
"fieldType": 2,
"fieldId": 22,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 109,
"appId": 10061,
"fieldName": "jd_cityid",
"fieldType": 2,
"fieldId": 23,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 112,
"appId": 10061,
"fieldName": "jd_district",
"fieldType": 2,
"fieldId": 24,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 113,
"appId": 10061,
"fieldName": "jd_districtid",
"fieldType": 2,
"fieldId": 25,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 114,
"appId": 10061,
"fieldName": "address",
"fieldType": 2,
"fieldId": 26,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 116,
"appId": 10061,
"fieldName": "location",
"fieldType": 5,
"fieldId": 27,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 117,
"appId": 10061,
"fieldName": "jd_town",
"fieldType": 2,
"fieldId": 28,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 118,
"appId": 10061,
"fieldName": "jd_townid",
"fieldType": 2,
"fieldId": 29,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 119,
"appId": 10061,
"fieldName": "idx_gis_poi",
"fieldType": 11,
"fieldId": 30,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "27,1",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 1,
"appId": 10010,
"fieldName": "doc_id",
"fieldType": 2,
"fieldId": 0,
"IsPrimaryKey": 1,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 2,
"appId": 10010,
"fieldName": "birthPlace",
"fieldType": 3,
"fieldId": 1,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 3,
"appId": 10010,
"fieldName": "homeAddress",
"fieldType": 3,
"fieldId": 2,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 1,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 4,
"appId": 10010,
"fieldName": "dreamPlace",
"fieldType": 2,
"fieldId": 3,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 3,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 5,
"appId": 10010,
"fieldName": "name",
"fieldType": 2,
"fieldId": 4,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 4,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 6,
"appId": 10010,
"fieldName": "gender",
"fieldType": 2,
"fieldId": 14,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 7,
"appId": 10010,
"fieldName": "year",
"fieldType": 1,
"fieldId": 5,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 5,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 8,
"appId": 10010,
"fieldName": "height",
"fieldType": 9,
"fieldId": 6,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 9,
"appId": 10010,
"fieldName": "brithday",
"fieldType": 10,
"fieldId": 7,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 5,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 10,
"appId": 10010,
"fieldName": "ip",
"fieldType": 4,
"fieldId": 8,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 11,
"appId": 10010,
"fieldName": "currentLocation",
"fieldType": 5,
"fieldId": 9,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 12,
"appId": 10010,
"fieldName": "preLocation",
"fieldType": 5,
"fieldId": 10,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 13,
"appId": 10010,
"fieldName": "postLocation",
"fieldType": 5,
"fieldId": 11,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 14,
"appId": 10010,
"fieldName": "currentShape",
"fieldType": 14,
"fieldId": 12,
"IsPrimaryKey": 0,
"indexTag": 1,
"snapshotTag": 1,
"segmentTag": 0,
"segmentFeature": 2,
"unionField": "",
"createTime": "2021/4/13 15:49:09"
}, {
"id": 15,
"appId": 10010,
"fieldName": "unionTest",
"fieldType": 11,
"fieldId": 13,
"IsPrimaryKey": 0,
"indexTag": 0,
"snapshotTag": 0,
"segmentTag": 0,
"segmentFeature": 0,
"unionField": "5,1,2",
"createTime": "2021/4/13 15:49:09"
}]
}

66
resource/app_field_define.txt Normal file
View File

@ -0,0 +1,66 @@
1 10001 doc_id 1 2 0 0 0 4 2018-12-21 15:08:28 0
5 10001 content 0 3 1 1 1 1 2018-12-21 15:10:28 0
4 10001 title 0 2 1 1 0 2 2018-12-21 15:10:28 0
3 10001 author 0 2 1 1 3 6 2018-12-21 15:10:28 1
2 10001 weight 0 1 1 1 5 5 2018-12-21 15:09:04 0
7 10001 longitude 0 5 1 1 0 0 2018-12-21 15:09:04 0
8 10001 gis 0 1 1 0 0 8 2018-12-21 15:09:04 0
9 10001 distance 0 8 1 0 0 0 2018-12-21 15:09:04 0
10 10001 telephone 0 2 1 1 4 7 2019-11-22 10:16:56 1
1 10064 doc_id 1 2 0 0 0 1 2021-03-26 16:22:47 0
2 10064 source 0 1 0 1 0 2 2021-03-26 16:22:47 0
3 10064 style 0 1 0 1 0 3 2021-03-26 16:22:47 0
4 10064 sub_position 0 1 0 1 0 4 2021-03-26 16:22:47 0
5 10064 private_status 0 1 0 1 0 5 2021-03-26 16:22:47 0
6 10064 author_id 0 2 1 1 0 6 2021-03-26 16:22:47 2
7 10064 status 0 1 0 1 0 7 2021-03-26 16:22:47 0
8 10064 createtime 0 1 1 1 0 8 2021-03-26 16:22:47 0
9 10064 title 0 2 1 1 1 10 2021-03-26 16:22:47 0
10 10064 idx_author_sub_pos 0 11 0 0 0 9 2021-03-26 16:22:47 0 6,4
11 10064 idx_author_title 0 11 0 0 0 11 2021-03-26 16:22:47 0 6,10
86 10061 doc_id 1 1 0 0 0 0 2020-07-14 19:24:01 0
87 10061 poi_name 0 2 1 1 1 1 2020-07-14 19:24:36 1
88 10061 gd_c1_type 0 2 0 1 1 2 2020-07-14 19:24:36 0
89 10061 gd_c2_type 0 2 0 1 1 3 2020-07-14 19:24:36 0
90 10061 gd_c3_type 0 2 0 1 1 4 2020-07-14 19:24:36 0
91 10061 gd_c1_typecode 0 2 0 1 1 5 2020-07-14 19:24:36 0
92 10061 gd_c2_typecode 0 2 0 1 1 6 2020-07-14 19:24:36 0
93 10061 gd_c3_typecode 0 2 0 1 1 7 2020-07-14 19:24:36 0
96 10061 std_region 0 2 0 1 1 8 2020-07-14 19:24:36 0
97 10061 std_regionid 0 2 0 1 1 9 2020-07-14 19:24:36 0
98 10061 std_province 0 2 0 1 1 10 2020-07-14 19:24:37 0
99 10061 std_provinceid 0 2 0 1 1 11 2020-07-14 19:24:37 0
100 10061 std_city 0 2 0 1 1 12 2020-07-14 19:24:37 0
101 10061 std_cityid 0 2 0 1 1 13 2020-07-14 19:24:37 0
102 10061 std_district 0 2 0 1 1 14 2020-07-14 19:24:37 0
103 10061 std_districtid 0 2 0 1 1 15 2020-07-14 19:24:37 0
104 10061 std_town 0 2 0 1 1 16 2020-07-14 19:24:37 0
105 10061 std_townid 0 2 0 1 1 17 2020-07-14 19:24:37 0
106 10061 jd_region 0 2 0 1 1 18 2020-07-14 19:24:37 0
107 10061 jd_regionid 0 2 0 1 1 19 2020-07-14 19:24:37 0
108 10061 jd_province 0 2 0 1 1 20 2020-07-14 19:24:37 0
109 10061 jd_provinceid 0 2 0 1 1 21 2020-07-14 19:24:37 0
110 10061 jd_city 0 2 0 1 1 22 2020-07-14 19:24:37 0
111 10061 jd_cityid 0 2 0 1 1 23 2020-07-14 19:24:37 0
112 10061 jd_district 0 2 0 1 1 24 2020-07-14 19:24:37 0
113 10061 jd_districtid 0 2 0 1 1 25 2020-07-14 19:24:37 0
114 10061 address 0 2 1 1 1 26 2020-07-14 19:24:37 0
116 10061 location 0 5 1 1 0 27 2020-07-14 19:24:38 2
117 10061 jd_town 0 1 1 1 1 28 2020-07-14 19:24:38 0
118 10061 jd_townid 0 1 1 1 1 29 2020-07-14 19:24:38 0
119 10061 idx_gis_poi 0 11 0 0 0 30 2020-07-14 19:24:38 0 27,1
119 10010 doc_id 1 2 0 0 0 0 2021/4/13 15:49 0
120 10010 birthPlace 0 3 1 1 1 1 2021/4/13 15:49 2
121 10010 homeAddress 0 3 1 1 1 2 2021/4/13 15:49 2
122 10010 dreamPlace 0 2 1 1 3 3 2021/4/14 15:49 2
123 10010 gender 0 2 0 1 0 14 2021/4/15 15:49 2
124 10010 name 0 2 1 1 4 4 2021/4/15 15:49 2
125 10010 year 0 1 1 1 5 5 2021/4/16 15:49 2
126 10010 height 0 9 1 1 0 6 2021/4/17 15:49 2
127 10010 brithday 0 10 1 1 5 7 2021/4/18 15:49 2
128 10010 ip 0 4 1 1 0 8 2021/4/19 15:49 2
129 10010 currentLocation 0 5 1 1 0 9 2021/4/20 15:49 2
130 10010 preLocation 0 5 1 1 0 10 2021/4/21 15:49 2
131 10010 postLocation 0 5 1 1 0 11 2021/4/20 15:49 2
132 10010 currentShape 0 14 1 1 0 12 2021/4/21 15:49 2
133 10010 unionTest 0 11 0 0 0 13 2021/4/21 15:49 0 5,1,2

6823
resource/character_map.txt Normal file
View File

File diff suppressed because it is too large Load Diff

26
resource/dtcd.sh Normal file
View File

@ -0,0 +1,26 @@
#!/bin/sh
ulimit -c unlimited
session_token=`pwd | awk -F'/' '{print $5}'`
session_port=`pwd | awk -F'/' '{print $6}'`
DTC_BIN="dtcd_${session_token}_${session_port}"
rm -f "$DTC_BIN"
ln -s dtcd "$DTC_BIN"
DisableDataSource=`cat ../conf/cache.conf | grep DisableDataSource | awk -F"=" '{print $2}'`
db_machine_num=`cat ../conf/table.conf | grep MachineNum | awk -F"=" '{print $2}'`
if [ "$1" = "stop" ] ; then
killall $DTC_BIN
elif [ "$1" = "restart" ]; then
killall $DTC_BIN
sleep 3
./$DTC_BIN >> /dev/null 2>&1
elif [ "$1" = "start" ]; then
./$DTC_BIN >> /dev/null 2>&1
sleep 1
else
echo "usage: $0 stop | start |restart"
fi

132371
resource/en_intelligent_match.txt Normal file
View File

File diff suppressed because it is too large Load Diff

85546
resource/en_stem.txt Normal file
View File

File diff suppressed because it is too large Load Diff

132371
resource/en_words_base.txt Normal file
View File

File diff suppressed because it is too large Load Diff

28
resource/index_read/conf/cache.conf Normal file
View File

@ -0,0 +1,28 @@
[search_cache]
LogLevel=error
BindAddr=*:20000/tcp
IdleTimeout=30
HelperTimeout=30
CacheShmKey=30311
CacheMemorySize=128
CacheAverageDataSize=50
MaxFlushSpeed=1
MinDirtyNode=40
MaxDirtyNode=60
MinDirtyTime=3600
MaxDirtyTime=43200
DelayUpdate=0
DisableDataSource=0
OpenningFDAttrID=41784
EnableCoreDump=1
RemoteLogAddr=127.0.0.1
RemoteLogPort=9876
UseSingleThread=0
DisableLRUUpdate=2
MaxIncomingPollers@inc0=10000
MaxFdCount=10240
MaxBarrierCount=100000
MaxKeyCount=10000
LimitNodeRows=400
TableName=keyword_index_data
SearchIndexType=SearchRocksDB

66
resource/index_read/conf/index_read.conf Normal file
View File

@ -0,0 +1,66 @@
{
"service_name":"index_read",
"pid_file" : "index_read.pid",
"log" : "../log/",
"log_level" : 7,
"daemon": true,
"time_interval": 3600,
"update_interval": 100000,
"cache_max_slot": 100000,
"index_cache_max_slot": 1000,
"cache_expire_time": 3600,
"vague_switch": 1,
"jdq_switch": 0,
"listen_addr": "127.0.0.1",
"listen_port": 12003,
"dtc_config" :
{
"table_name": "keyword_index_data",
"accesskey": "00002153a746aac69b2ca06f79f8154e3a03ad04",
"timeout": 1000,
"keytype": 4,
"route":
[
{
"ip": "127.0.0.1",
"bid": 2153,
"port": 20000,
"weight": 1,
"status": 1
}
]
},
"hanpin_index_config" :
{
"table_name" : "hanpin_index_data",
"accesskey" : "000021140f992fd33e6019ce2c66cd8614925953",
"timeout" : 1000,
"keytype" : 4,
"route":
[
{
"ip": "127.0.0.1",
"bid": 2114,
"port": 20005,
"weight": 1,
"status": 1
}
]
},
"phonetic_file" : "../conf/phonetic_map.txt",
"character_file" : "../conf/character_map.txt",
"intelligent_file" : "../conf/intelligent_match.txt",
"en_intelligent_file" : "../conf/en_intelligent_match.txt",
"words_file" : "../conf/words_base.txt",
"en_words_file" : "../conf/en_words_base.txt",
"phonetic_base_file" : "../conf/phonetic_base.txt",
"suggest_file" : "../conf/suggest_base.txt",
"training_file" : "../conf/msr_training.utf8",
"relate_file" : "../data/relate_data",
"synonym_file" : "../data/synonym_data",
"analyze_file" : "../data/analyze_data",
"sensitive_file" : "../data/sensitive_data",
"app_filed_file" : "../conf/app_field_define.txt",
"app_info" : [{"app_id":10001, "cache_switch":0, "en_query_switch":1}],
"split_mode": "Cache"
}

108
resource/index_read/conf/table.conf Normal file
View File

@ -0,0 +1,108 @@
[DB_DEFINE]
DbName=d_db
DbNum=(1,1)
dbMax=1
MachineNum=1
Deploy=0
EnableKeyHash=0
KeyHashSo=../lib/key-hash.so
KeyHashFunction=StringHash(1,128)
DatasourceType=2
[Machine1]
Procs=3
WriteProcs=3
CommitProcs=3
DbIdx=[0]
DbAddr=127.0.0.1
DbUser=root
DbPass=root
MyCnf=../conf/my.conf
[TABLE_DEFINE]
TableName=keyword_index_data
FieldCount=12
KeyFieldCount=1
IndexFieldCount=0
TableNum=(1,1)
[FIELD1]
FieldName=key
FieldType=4
FieldSize=100
UniqField=1
[FIELD2]
FieldName=doc_id
FieldType=4
FieldSize=20
UniqField=1
[FIELD3]
FieldName=doc_version
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD4]
FieldName=trans_version
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD5]
FieldName=created_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD6]
FieldName=field
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD7]
FieldName=word_freq
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD8]
FieldName=weight
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD9]
FieldName=location
FieldType=4
FieldSize=1000
UniqField=0
[FIELD10]
FieldName=start_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD11]
FieldName=end_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD12]
FieldName=extend
FieldType=4
FieldSize=65535
UniqField=0
DefaultValue=""

3
resource/index_read/data/analyze_data Normal file
View File

@ -0,0 +1,3 @@
京东会员 8
京东体验展 1
京东商城 50

3
resource/index_read/data/relate_data Normal file
View File

@ -0,0 +1,3 @@
化妆品 保养品
笔记本 computer
推车 婴儿车

3
resource/index_read/data/sensitive_data Normal file
View File

@ -0,0 +1,3 @@
兴奋剂
打人
违禁

3
resource/index_read/data/synonym_data Normal file
View File

@ -0,0 +1,3 @@
中华田园犬 小狗 狗 金毛
互联网 移动互联网
京东白条 白条

25
resource/index_storage/intelligent_index/conf/cache.conf Normal file
View File

@ -0,0 +1,25 @@
LogLevel=error
BindAddr=*:20005/tcp
IdleTimeout=30000
HelperTimeout=30000
HelperReplPort=40412
CacheShmKey=20005
CacheMemorySize=100
CacheAverageDataSize=50
MaxFlushSpeed=1
MinDirtyNode=40
MaxDirtyNode=60
MinDirtyTime=3600
MaxDirtyTime=43200
DelayUpdate=0
DisableDataSource=0
OpenningFDAttrID=41784
EnableCoreDump=1
RemoteLogAddr=127.0.0.1
RemoteLogPort=9876
UseSingleThread=0
DisableLRUUpdate=2
MaxIncomingPollers@inc0=10000
MaxFdCount=10240
MaxBarrierCount=100000
MaxKeyCount=10000

261
resource/index_storage/intelligent_index/conf/table.conf Normal file
View File

@ -0,0 +1,261 @@
[DB_DEFINE]
DbName=word_lib
DbNum=(1,1)
dbMax=1
MachineNum=1
Deploy=0
EnableKeyHash=0
KeyHashSo=../lib/key-hash.so
KeyHashFunction=StringHash(1,128)
DatasourceType=2
[Machine1]
Procs=10
WriteProcs=10
CommitProcs=10
DbIdx=[0-0]
DbAddr=/tmp/mysql.sock
DbUser=root
DbPass=root
MyCnf=../conf/my.conf
[TABLE_DEFINE]
TableName=hanpin_index_data
FieldCount=36
KeyFieldCount=1
IndexFieldCount=0
TableNum=(1,1)
[FIELD1]
FieldName=key
FieldType=4
FieldSize=20
UniqField=1
[FIELD2]
FieldName=doc_id
FieldType=4
FieldSize=20
UniqField=1
[FIELD3]
FieldName=doc_version
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=0
[FIELD4]
FieldName=word
FieldType=4
FieldSize=64
UniqField=1
[FIELD5]
FieldName=charact_id_01
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD6]
FieldName=charact_id_02
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD7]
FieldName=charact_id_03
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD8]
FieldName=charact_id_04
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD9]
FieldName=charact_id_05
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD10]
FieldName=charact_id_06
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD11]
FieldName=charact_id_07
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD12]
FieldName=charact_id_08
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD13]
FieldName=phonetic_id_01
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD14]
FieldName=phonetic_id_02
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD15]
FieldName=phonetic_id_03
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD16]
FieldName=phonetic_id_04
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD17]
FieldName=phonetic_id_05
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD18]
FieldName=phonetic_id_06
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD19]
FieldName=phonetic_id_07
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD20]
FieldName=phonetic_id_08
FieldType=1
FieldSize=2
UniqField=0
DefaultValue=0
[FIELD21]
FieldName=initial_char_01
FieldType=4
FieldSize=5
UniqField=0
[FIELD22]
FieldName=initial_char_02
FieldType=4
FieldSize=5
UniqField=0
[FIELD23]
FieldName=initial_char_03
FieldType=4
FieldSize=5
UniqField=0
[FIELD24]
FieldName=initial_char_04
FieldType=4
FieldSize=5
UniqField=0
[FIELD25]
FieldName=initial_char_05
FieldType=4
FieldSize=5
UniqField=0
[FIELD26]
FieldName=initial_char_06
FieldType=4
FieldSize=5
UniqField=0
[FIELD27]
FieldName=initial_char_07
FieldType=4
FieldSize=5
UniqField=0
[FIELD28]
FieldName=initial_char_08
FieldType=4
FieldSize=5
UniqField=0
[FIELD29]
FieldName=initial_char_09
FieldType=4
FieldSize=5
UniqField=0
[FIELD30]
FieldName=initial_char_10
FieldType=4
FieldSize=5
UniqField=0
[FIELD31]
FieldName=initial_char_11
FieldType=4
FieldSize=5
UniqField=0
[FIELD32]
FieldName=initial_char_12
FieldType=4
FieldSize=5
UniqField=0
[FIELD33]
FieldName=initial_char_13
FieldType=4
FieldSize=5
UniqField=0
[FIELD34]
FieldName=initial_char_14
FieldType=4
FieldSize=5
UniqField=0
[FIELD35]
FieldName=initial_char_15
FieldType=4
FieldSize=5
UniqField=0
[FIELD36]
FieldName=initial_char_16
FieldType=4
FieldSize=5
UniqField=0

26
resource/index_storage/inverted_index/bin/dtcd.sh Normal file
View File

@ -0,0 +1,26 @@
#!/bin/sh
ulimit -c unlimited
session_token=`pwd | awk -F'/' '{print $5}'`
session_port=`pwd | awk -F'/' '{print $6}'`
DTC_BIN="dtcd_${session_token}_${session_port}"
rm -f "$DTC_BIN"
ln -s dtcd "$DTC_BIN"
DisableDataSource=`cat ../conf/cache.conf | grep DisableDataSource | awk -F"=" '{print $2}'`
db_machine_num=`cat ../conf/table.conf | grep MachineNum | awk -F"=" '{print $2}'`
if [ "$1" = "stop" ] ; then
killall $DTC_BIN
elif [ "$1" = "restart" ]; then
killall $DTC_BIN
sleep 3
./$DTC_BIN >> /dev/null 2>&1
elif [ "$1" = "start" ]; then
./$DTC_BIN >> /dev/null 2>&1
sleep 1
else
echo "usage: $0 stop | start |restart"
fi

26
resource/index_storage/inverted_index/conf/cache.conf Normal file
View File

@ -0,0 +1,26 @@
LogLevel=error
BindAddr=*:20000/tcp
BindAddr1=@search.socket
IdleTimeout=30000
HelperTimeout=30000
HelperReplPort=40411
CacheShmKey=20000
CacheMemorySize=100
CacheAverageDataSize=50
MaxFlushSpeed=1
MinDirtyNode=40
MaxDirtyNode=60
MinDirtyTime=3600
MaxDirtyTime=43200
DelayUpdate=0
DisableDataSource=0
OpenningFDAttrID=41784
EnableCoreDump=1
RemoteLogAddr=172.28.133.131
RemoteLogPort=9876
UseSingleThread=0
DisableLRUUpdate=2
MaxIncomingPollers@inc0=10000
MaxFdCount=10240
MaxBarrierCount=100000
MaxKeyCount=10000

108
resource/index_storage/inverted_index/conf/table.conf Normal file
View File

@ -0,0 +1,108 @@
[DB_DEFINE]
DbName=d_db
DbNum=(1,1)
dbMax=1
MachineNum=1
Deploy=0
EnableKeyHash=0
KeyHashSo=../lib/key-hash.so
KeyHashFunction=StringHash(1,128)
DatasourceType=2
[Machine1]
Procs=3
WriteProcs=3
CommitProcs=3
DbIdx=[0]
DbAddr=127.0.0.1
DbUser=root
DbPass=root
MyCnf=../conf/my.conf
[TABLE_DEFINE]
TableName=keyword_index_data
FieldCount=12
KeyFieldCount=1
IndexFieldCount=0
TableNum=(1,1)
[FIELD1]
FieldName=key
FieldType=4
FieldSize=100
UniqField=1
[FIELD2]
FieldName=doc_id
FieldType=4
FieldSize=20
UniqField=1
[FIELD3]
FieldName=doc_version
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD4]
FieldName=trans_version
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD5]
FieldName=created_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD6]
FieldName=field
FieldType=1
FieldSize=4
UniqField=1
DefaultValue=1
[FIELD7]
FieldName=word_freq
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD8]
FieldName=weight
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD9]
FieldName=location
FieldType=4
FieldSize=1000
UniqField=0
[FIELD10]
FieldName=start_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD11]
FieldName=end_time
FieldType=1
FieldSize=4
UniqField=0
DefaultValue=0
[FIELD12]
FieldName=extend
FieldType=4
FieldSize=65535
UniqField=0
DefaultValue=""

24
resource/index_storage/original_data/conf/cache.conf Normal file
View File

@ -0,0 +1,24 @@
LogLevel=error
BindAddr=*:20070/tcp
IdleTimeout=30
HelperTimeout=30
CacheShmKey=20070
CacheMemorySize=100
CacheAverageDataSize=50
MaxFlushSpeed=1
MinDirtyNode=40
MaxDirtyNode=60
MinDirtyTime=3600
MaxDirtyTime=43200
DelayUpdate=0
DisableDataSource=1
OpenningFDAttrID=41784
EnableCoreDump=1
RemoteLogAddr=172.28.133.131
RemoteLogPort=9876
UseSingleThread=1
DisableLRUUpdate=2
MaxIncomingPollers@inc0=10000
MaxFdCount=10240
MaxBarrierCount=100000
MaxKeyCount=10000

44
resource/index_storage/original_data/conf/table.conf Normal file
View File

@ -0,0 +1,44 @@
[DB_DEFINE]
DbName=d_db
DbNum=(1,1)
dbMax=1
MachineNum=1
Deploy=0
EnableKeyHash=0
KeyHashSo=../lib/key-hash.so
KeyHashFunction=StringHash(1,128)
[Machine1]
Procs=10
WriteProcs=10
CommitProcs=10
DbIdx=[0]
DbAddr=127.0.0.1
DbUser=root
DbPass=root
MyCnf=../conf/my.conf
[TABLE_DEFINE]
TableName=index_original_data
FieldCount=3
KeyFieldCount=1
TableNum=(1,1)
[FIELD1]
FieldName=request_id
FieldType=2
FieldSize=4
UniqField=0
[FIELD2]
FieldName=content
FieldType=4
FieldSize=8192
UniqField=0
[FIELD3]
FieldName=create_time
FieldType=2
FieldSize=4
UniqField=0

9
resource/index_write/conf/index_gen.json Normal file
View File

@ -0,0 +1,9 @@
{
"CONFIG" :
{
"LogLevel":7,
"UpdateInterval": 1,
"CaDir" : "/usr/local/dtcadmin/ca/conf/",
"CaPid":1606976164
}
}

70
resource/index_write/conf/index_write.conf Normal file
View File

@ -0,0 +1,70 @@
{
"program_name" : "index_write v1.0",
"pid_file" : "index_write.pid",
"log" : "../log/",
"log_level" : 7,
"daemon": true,
"listen_addr": "*:11017/tcp",
"timeout": 6000,
"stop_words_path":"../conf/stop_words.dict",
"training_path":"../conf/msr_training.utf8",
"words_base_path":"../conf/words_base.dict",
"words_file":"../conf/words_base.txt",
"character_path":"../conf/character_map.txt",
"phonetic_path":"../conf/phonetic_map.txt",
"phonetic_base_file" : "../conf/phonetic_base.txt",
"app_filed_file" : "../conf/app_field_define.txt",
"service_type":"index_gen",
"dtc_index_config" :
{
"table_name": "keyword_index_data",
"accesskey": "00002153a746aac69b2ca06f79f8154e3a03ad04",
"timeout": 4000,
"keytype": 4,
"route":
[
{
"ip": "127.0.0.1",
"bid": 2153,
"port": 20000,
"weight": 1,
"status": 1
}
]
},
"dtc_intelligent_config" :
{
"table_name": "hanpin_index_data",
"accesskey": "000021140f992fd33e6019ce2c66cd8614925953",
"timeout": 4000,
"keytype": 4,
"route":
[
{
"ip": "127.0.0.1",
"bid": 2114,
"port": 20005,
"weight": 1,
"status": 1
}
]
},
"dtc_original_config" :
{
"table_name": "index_original_data",
"accesskey": "00002263b51c22d9b7f1a68dd023ff625cbabda0",
"timeout": 4000,
"keytype": 2,
"route":
[
{
"ip": "127.0.0.1",
"bid": 2263,
"port": 20070,
"weight": 1,
"status": 1
}
]
},
"split_mode":"Post"
}

39
resource/index_write/conf/localCluster.json Normal file
View File

@ -0,0 +1,39 @@
{
"MODULE" :
{
"SERVERSHARDING" :
[
{
"ShardingName" : "keyword_index_data1743",
"INSTANCE" :
[
{
"ip" : "127.0.0.1",
"port" : "12003",
"weight" : 1
},
{
"ip" : "127.0.0.1",
"port" : "11017",
"keywordtablePort" : "20000",
"rocksDbReplPort" : "40411",
"originaltablePort" : "20070",
"weight" : 1,
"role" : "index_gen",
"disasterRole" : "master"
},
{
"ip" : "127.0.0.2",
"port" : "11017",
"keywordtablePort" : "20000",
"rocksDbReplPort" : "40411",
"originaltablePort" : "20070",
"weight" : 1,
"role" : "index_gen",
"disasterRole" : "replicate"
}
]
}
]
}
}

331932
resource/intelligent_match.txt Normal file
View File

File diff suppressed because it is too large Load Diff

17115
resource/msr_training.utf8 Normal file
View File

File diff suppressed because it is too large Load Diff

7332
resource/phonetic_base.txt Normal file
View File

File diff suppressed because it is too large Load Diff

410
resource/phonetic_map.txt Normal file
View File

@ -0,0 +1,410 @@
172 a
49 ai
44 an
395 ang
11 ao
136 ba
325 bai
2 ban
298 bang
192 bao
41 bei
189 ben
287 beng
42 bi
102 bian
244 biao
344 bie
267 bin
280 bing
101 bo
40 bu
31 ca
327 cai
148 can
323 cang
350 cao
85 ce
149 cen
282 ceng
32 cha
292 chai
122 chan
80 chang
171 chao
188 che
264 chen
37 cheng
140 chi
261 chong
299 chou
68 chu
347 chuai
291 chuan
307 chuang
81 chui
370 chun
373 chuo
293 ci
306 cong
372 cou
118 cu
382 cuan
289 cui
256 cun
57 cuo
204 da
43 dai
123 dan
271 dang
133 dao
318 de
334 dei
290 deng
187 di
357 dia
71 dian
144 diao
63 die
137 ding
295 diu
14 dong
374 dou
193 du
377 duan
195 dui
161 dun
88 duo
69 e
396 ei
405 en
279 er
59 fa
128 fan
177 fang
158 fei
98 fen
227 feng
330 fo
160 fou
214 fu
216 ga
82 gai
183 gan
285 gang
358 gao
170 ge
388 gei
316 gen
50 geng
1 gong
60 gou
130 gu
108 gua
312 guai
269 guan
308 guang
230 gui
381 gun
35 guo
368 ha
254 hai
162 han
163 hang
3 hao
28 he
349 hei
332 hen
315 heng
367 hng
260 hong
54 hou
213 hu
103 hua
179 huai
226 huan
309 huang
125 hui
245 hun
6 huo
47 ji
8 jia
92 jian
225 jiang
26 jiao
18 jie
22 jin
23 jing
383 jiong
70 jiu
131 ju
112 juan
27 jue
176 jun
105 ka
75 kai
178 kan
314 kang
277 kao
138 ke
64 ken
164 keng
257 kong
129 kou
198 ku
73 kua
190 kuai
266 kuan
217 kuang
219 kui
181 kun
324 kuo
78 la
288 lai
246 lan
319 lang
24 lao
143 le
79 lei
205 leng
21 li
336 lia
220 lian
206 liang
274 liao
12 lie
157 lin
132 ling
346 liu
361 lo
83 long
96 lou
84 lu
115 luan
394 lue:
322 lun
362 luo
97 lv
360 m
173 ma
10 mai
52 man
340 mang
110 mao
311 me
232 mei
321 men
252 meng
270 mi
237 mian
235 miao
310 mie
284 min
155 ming
389 miu
208 mo
335 mou
186 mu
355 n
116 na
223 nai
117 nan
376 nang
16 nao
359 ne
365 nei
248 nen
402 neng
356 ng
185 ni
39 nian
241 niang
251 niao
255 nie
408 nin
259 ning
228 niu
326 nong
406 nou
17 nu
407 nuan
392 nue:
240 nuo
222 nv
353 o
94 ou
304 pa
333 pai
152 pan
313 pang
317 pao
46 pei
352 pen
197 peng
30 pi
91 pian
250 piao
301 pie
236 pin
184 ping
146 po
375 pou
51 pu
166 qi
106 qia
114 qian
210 qiang
4 qiao
233 qie
159 qin
113 qing
378 qiong
294 qiu
19 qu
7 quan
111 que
390 qun
371 ran
25 rang
243 rao
351 re
211 ren
320 reng
410 ri
265 rong
386 rou
34 ru
400 ruan
380 rui
404 run
99 ruo
124 sa
199 sai
127 san
296 sang
48 sao
200 se
409 sen
343 seng
65 sha
384 shai
38 shan
67 shang
20 shao
56 she
397 shei
150 shen
121 sheng
29 shi
147 shou
151 shu
345 shua
305 shuai
403 shuan
145 shuang
379 shui
165 shun
229 shuo
74 si
273 song
126 sou
203 su
385 suan
278 sui
258 sun
242 suo
202 ta
139 tai
142 tan
194 tang
134 tao
337 te
338 tei
393 teng
5 ti
201 tian
328 tiao
302 tie
86 ting
77 tong
95 tou
156 tu
329 tuan
339 tui
154 tun
182 tuo
239 wa
218 wai
238 wan
234 wang
90 wei
169 wen
354 weng
109 wo
9 wu
168 xi
66 xia
76 xian
61 xiang
13 xiao
100 xie
275 xin
87 xing
342 xiong
268 xiu
153 xu
262 xuan
253 xue
53 xun
72 ya
62 yan
215 yang
175 yao
141 ye
33 yi
58 yin
247 ying
364 yo
209 yong
107 you
36 yu
55 yuan
283 yue
191 yun
341 za
263 zai
363 zan
224 zang
369 zao
300 ze
399 zei
398 zen
207 zeng
174 zha
272 zhai
104 zhan
249 zhang
135 zhao
366 zhe
401 zhei
331 zhen
286 zheng
45 zhi
297 zhong
231 zhou
15 zhu
391 zhua
348 zhuai
303 zhuan
212 zhuang
180 zhui
281 zhun
120 zhuo
167 zi
93 zong
221 zou
119 zu
387 zuan
196 zui
276 zun
89 zuo

24
resource/search_agent/conf/sa.conf Normal file
View File

@ -0,0 +1,24 @@
{
"SERVERSHARDING" :
[
{
"ShardingName" : "keyword_index_data1743",
"INSTANCE" :
[
{
"ip" : "127.0.0.1",
"port" : "12003",
"weight" : 1
},
{
"ip" : "127.0.0.1",
"port" : "11017",
"weight" : 1,
"role" : "index_gen"
}
]
}
]
}

1894
resource/stop_words.dict Normal file
View File

File diff suppressed because it is too large Load Diff

243746
resource/suggest_base.txt Normal file
View File

File diff suppressed because it is too large Load Diff

7
resource/tools/load_data.sh Normal file
View File

@ -0,0 +1,7 @@
while read -r line
do
doc_id=`echo "$line"| jq '.table_content.fields.doc_id'`
curl -X POST http://127.0.0.1/insert -H 'content-type: application/json' -H 'doc_id:'"$doc_id" -d "$line"
echo 'doc_id:'"$doc_id"
echo $line
done < send.json

5
resource/tools/query.sh Normal file
View File

@ -0,0 +1,5 @@
while read -r line
do
echo $line
curl -X POST http://127.0.0.1/search -H 'content-type: application/json' -d "$line"
done < search.json

12
resource/tools/search.json Normal file
View File

@ -0,0 +1,12 @@
{"appid":10010,"query":{"match":{"birthPlace":"上海市"}}}
{"appid":10010,"query":{"match":{"dreamPlace":"王国"}}}
{"appid":10010,"query":{"match":{"name":"Jackie"}}}
{"appid":10010,"query":{"term":{"year":20}}}
{"appid":10010,"query":{"range":{"height":{"gte":175 ,"lte": 180}}}}
{"appid":10010,"query":{"geo_distance":{"currentLocation":"39.452, -76.589","distance":"0.5"}}}
{"appid":10010,"query":{"geo_polygon":{"currentShape":{"points":"POLYGON((121.437271 31.339747, 121.438022 31.337291, 121.435297 31.336814, 121.434524 31.339252, 121.437271 31.339747))"}}}}
{"appid":10010,"query":{"bool":{"must":{"match":{"birthPlace":"中华人民共和国"},"range":{"brithday":{"gt":19900654,"lte":19931124}}},"must_not":{"match":{"homeAddress":"上海市"}}}}}
{"appid":10010,"query":{"bool":{"must":{"match":{"birthPlace":"上海市"},"geo_distance":{"currentLocation":"39.452, -76.589","distance":"1.123"}}}}}
{"appid":10010,"query":{"bool":{"must":{"match":{"birthPlace":"上海市"},"term":{"gender":"男"}}}}}
{"appid":10010,"query":{"match":{"dreamPlace":"阿姆斯"}},"fields":"birthPlace,homeAddress,dreamPlace,name","page_index":1,"page_size":3}
{"appid":10010,"query":{"range":{"height":{"gte":174 ,"lte": 180}}},"fields":"birthPlace,height,name","sort_type":"5","sort_field":"height"}

4
resource/tools/send.json Normal file
View File

@ -0,0 +1,4 @@
{"appid":10010,"table_content":{"cmd":"add","fields":{"doc_id":"1","birthPlace":"中华人民共和国北京市","homeAddress":"中华人民共和国上海市","dreamPlace":"阿姆斯特丹","name":"Jackie","gender":"女","year":10,"height":175.456,"brithday":19930917,"ip":"192.168.0.77","currentLocation":"40.715, -74.011","preLocation":{"latitude":"-73.983","longitude":"40.719"},"postLocation":["-70.000","35.236"],"currentShape":"POLYGON((121.437271 31.339747, 121.438022 31.337291, 121.435297 31.336814, 121.434524 31.339252, 121.437271 31.339747))"}}}
{"appid":10010,"table_content":{"cmd":"add","fields":{"doc_id":"2","birthPlace":"中华人民共和国湖北省武汉市","homeAddress":"中华人民共和国上海市长宁区","dreamPlace":"比利时王国","name":"Joy","gender":"男","year":20,"height":174.325,"brithday":19910720,"ip":"192.168.0.35","currentLocation":"39.452, -76.589","preLocation":{"latitude":"-70.154","longitude":"35.247"},"postLocation":["-75.456","40.111"],"currentShape":"POLYGON((121.437271 31.339747, 121.438022 31.337291, 121.435297 31.336814, 121.434524 31.339252, 121.437271 31.339747))"}}}
{"appid":10010,"table_content":{"cmd":"add","fields":{"doc_id":"3","birthPlace":"中华人民共和国江苏省苏州市","homeAddress":"中华人民共和国上海市闵行区","dreamPlace":"东京郊外调布市","name":"Tom","gender":"男","year":30,"height":180.785,"brithday":19900654,"ip":"192.168.0.98","currentLocation":"34.452, -65.589","preLocation":{"latitude":"-68.355","longitude":"45.121"},"postLocation":["-71.456","27.986"],"currentShape":"POLYGON((121.437271 31.339747, 121.438022 31.337291, 121.435297 31.336814, 121.434524 31.339252, 121.437271 31.339747))"}}}
{"appid":10010,"table_content":{"cmd":"add","fields":{"doc_id":"4","birthPlace":"中华人民共和国上海市","homeAddress":"中华人民共和国上海市宝山区","dreamPlace":"梵蒂冈高地","name":"Amy","gender":"女","year":40,"height":176.258,"brithday":19931124,"ip":"192.168.0.18","currentLocation":"39.452, -76.589","preLocation":{"latitude":"-70.154","longitude":"35.247"},"postLocation":["-75.456","40.111"],"currentShape":"POLYGON((121.437271 31.339747, 121.438022 31.337291, 121.435297 31.336814, 121.434524 31.339252, 121.437271 31.339747))"}}}

233462
resource/words_base.txt Normal file
View File

File diff suppressed because it is too large Load Diff

1
src/comm/CMakeLists.txt
View File

@ -1,5 +1,6 @@
cmake_minimum_required(VERSION 2.6)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O2 -lrt -Wall")
AUX_SOURCE_DIRECTORY(. main)
AUX_SOURCE_DIRECTORY(./segment segment)
add_library(common ${main} ${segment})

2
src/comm/segment/custom_segment.cc
View File

@ -48,7 +48,7 @@ void CustomSegment::ConcreteSplit(const string& str, uint32_t appid, vector<stri
char res[100] = {'\0'};
word_split_func_(str.c_str(), res, 100);
string tmp = "";
for(int i = 0; i < strlen(res); i++){
for(int i = 0; i < (int)strlen(res); i++){
if(res[i] != ' '){
tmp += res[i];
} else {

3
src/comm/segment/hmm_manager.cc
View File

@ -47,6 +47,9 @@ bool HmmManager::Init(string train_path, const set<string>& punct_set) {
line_list.push_back(*iter);
}
}
if(line_list.size() == 0){
continue;
}
train_cnt_ += line_list.size();
for (int i = -1; i < (int)line_list.size(); i++) {
string word1;

3
src/comm/split_tool.cc
View File

@ -84,6 +84,9 @@ bool FBSegment::Init2(string train_path) {
line_list.push_back(*iter);
}
}
if(line_list.size() == 0){
continue;
}
train_cnt += line_list.size();
for (int i = -1; i < (int)line_list.size(); i++) {
string word1;

7
src/comm/stat/CMakeLists.txt
View File

@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 2.6)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O2 -lrt -Wall -pthread")
AUX_SOURCE_DIRECTORY(. main)
@ -12,14 +12,14 @@ ${CMAKE_CURRENT_BINARY_DIR}/../../3rdlib/jsoncpp/lib
${CMAKE_CURRENT_BINARY_DIR}/../../3rdlib/attr_api
)
ADD_EXECUTABLE(stattool ${main} ${http_service})
#ADD_EXECUTABLE(stattool ${main} ${http_service})
target_include_directories(stat PUBLIC
../../comm
../../3rdlib/attr_api
../../3rdlib/jsoncpp/include
)
#[[
target_include_directories(stattool PUBLIC
../../comm
../../3rdlib/attr_api
@ -27,3 +27,4 @@ target_include_directories(stattool PUBLIC
)
target_link_libraries(stattool libattr_api_64.a libcommon.a jsoncpp dl rt)
]]

2
src/search_agent/CMakeLists.txt
View File

@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.6)
PROJECT(search_agent)
EXECUTE_PROCESS(COMMAND git log -1 --pretty=format:%h . OUTPUT_VARIABLE version)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -lrt -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -lrt -Wall")
AUX_SOURCE_DIRECTORY(. main)

3
src/search_local/index_read/CMakeLists.txt
View File

@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.6)
PROJECT(index_read)
EXECUTE_PROCESS(COMMAND git log -1 --pretty=format:%h . OUTPUT_VARIABLE version)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -g -lrt -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -g -O2 -lrt -Wall")
AUX_SOURCE_DIRECTORY(. main)
AUX_SOURCE_DIRECTORY(./index_sync index_sync)
@ -17,6 +17,7 @@ LINK_DIRECTORIES(
${PROJECT_SOURCE_DIR}/../../comm
${PROJECT_SOURCE_DIR}/../../3rdlib/jsoncpp/lib
${PROJECT_SOURCE_DIR}/../../comm/stat
${PROJECT_SOURCE_DIR}/../index_storage/api/c_api_cc
)
ADD_EXECUTABLE(index_read ${main} ${index_sync} ${utils} ${correction} ${query} ${process} ${sort_operator})

11
src/search_local/index_read/data_manager.cc
View File

@ -433,15 +433,6 @@ bool DataManager::IsSensitiveWord(string word) {
return result;
}
bool DataManager::IsContainKey(vector<FieldInfo>& keys, string key) {
vector<FieldInfo>::iterator iter;
for (iter = keys.begin(); iter != keys.end(); iter++) {
if ((*iter).word == key)
return true;
}
return false;
}
void DataManager::GetSynonymByKey(const string& key, vector<FieldInfo>& keys)
{
if (synonym_map.find(key) == synonym_map.end())
@ -450,7 +441,7 @@ void DataManager::GetSynonymByKey(const string& key, vector<FieldInfo>& keys)
multimap<uint32_t, string>::iterator it;
it = synonym_id_map.find(id);
for (int i = 0, len = synonym_id_map.count(id);i < len; ++i,++it) {
if (IsContainKey(keys, it->second));
if (it->second == key)
continue ;
FieldInfo fieldInfo;
fieldInfo.word = it->second;

1
src/search_local/index_read/data_manager.h
View File

@ -81,7 +81,6 @@ public:
bool IsChineseWord(uint32_t appid, string str);
string GetPhonetic(string charact);
bool IsSensitiveWord(string word);
bool IsContainKey(vector<FieldInfo>& keys, string key);
bool IsPhonetic(string phonetic) {
bool result = false;
if (phonetic_set.find(phonetic) != phonetic_set.end())

1
src/search_local/index_read/doc_manager.cc
View File

@ -20,6 +20,7 @@
#include "doc_manager.h"
#include "log.h"
#include "search_util.h"
#include "index_tbl_op.h"
#include "db_manager.h"
#include "process/geo_distance_query_process.h"
#include <math.h>

2
src/search_local/index_read/index_sync/uds_client.cc
View File

@ -154,7 +154,7 @@ void parseResponse(char *data, int dataLen, std::vector<std::vector<std::string>
std::string value(data, value_len);
data += value_len;
dataLen -= value_len;
log_debug("field value:%s", value.c_str());
//log_debug("field value:%s", value.c_str());
row_fields[idx].push_back(value);
}
}

139
src/search_local/index_read/index_tbl_op.cc
View File

@ -998,4 +998,143 @@ vector<IndexInfo> vec_difference(vector<IndexInfo> &a, vector<IndexInfo> &b){
return diff_vec;
}
int ShiftIntelligentInfo(IntelligentInfo &info, int len)
{
uint16_t *p = &info.charact_id[0];
if (len >= 8) {
return -1;
}
if (len < 1) {
return -2;
}
memmove(&p[len], &p[0], sizeof(uint16_t) * (8 - len));
for (int i = 0; i < len; i++) {
p[i] = 0;
}
p = &info.phonetic_id[0];
memmove(&p[len], &p[0], sizeof(uint16_t) * (8 - len));
for (int i = 0; i < len; i++) {
p[i] = 0;
}
GetInitialVec(info, len);
return 0;
}
bool GetSuggestDoc(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
bool bRet;
int index = 0;
uint32_t field = fieldInfo.field;
uint32_t segment_feature = fieldInfo.segment_feature;
bRet = g_hanpinIndexInstance.GetSuggestDoc(appid, index, len, field, info, doc_id_set);
if (bRet == false)
goto resError;
if (segment_feature == 0) {
return true;
}
for (uint32_t i = 1; i <= 8 - len; i++) {
index = index + 1;
IntelligentInfo t2 = info;
if (ShiftIntelligentInfo(t2, i) < 0) {
continue;
}
bRet = g_hanpinIndexInstance.GetSuggestDoc(appid, index, len, field, t2, doc_id_set);
if (bRet == false)
goto resError;
}
return true;
resError:
log_error("GetSuggestDoc invalid.");
return false;
}
bool GetSuggestDocWithoutCharacter(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
bool bRet;
int index = 0;
uint32_t field = fieldInfo.field;
uint32_t segment_feature = fieldInfo.segment_feature;
bRet = g_hanpinIndexInstance.GetSuggestDocWithoutCharacter(appid, index, len, field, info, doc_id_set);
if (bRet == false)
goto resError;
if (segment_feature == 0) {
return true;
}
for (uint32_t i = 1; i <= 16 - len; i++) {
index = index + 1;
IntelligentInfo t2 = info;
if (ShiftIntelligentInfoWithoutCharacter(t2, i) < 0) {
continue;
}
bRet = g_hanpinIndexInstance.GetSuggestDocWithoutCharacter(appid, index, len, field, t2, doc_id_set);
if (bRet == false)
goto resError;
}
return true;
resError:
log_error("GetSuggestDoc invalid.");
return false;
}
int GetDocByShiftWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
log_debug("GetDocByShiftWord start");
bool bRet = true;
set<vector<Content> > result;
GetMultipleWords(fieldInfo.word, result);
int len = result.size();
if (len <= 0) {
log_error("get shift word error.");
return -RT_GET_SUGGEST_ERR;
}
set<vector<Content> >::iterator iter;
for (iter = result.begin(); iter != result.end(); iter++)
{
IntelligentInfo info;
vector<Content> tmp = *iter;
ConvertIntelligent(tmp, info, bRet);
int length = tmp.size();
if (bRet) {
bRet = GetSuggestDoc(fieldInfo, length, info, doc_id_set, appid);
if (!bRet) {
log_error("GetSuggestDocInfo error.");
return -RT_DTC_ERR;
}
}
}
return 0;
}
int GetDocByShiftEnWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
log_debug("GetDocByShiftEnWord start");
bool bRet = true;
int length = 0;
IntelligentInfo enInfo;
ConvertCharIntelligent(fieldInfo.word, enInfo, length);
bRet = GetSuggestDocWithoutCharacter(fieldInfo, length, enInfo, doc_id_set, appid);
if (!bRet) {
log_error("GetEnSuggestDocInfo error.");
return -RT_DTC_ERR;
}
return 0;
}

5
src/search_local/index_read/index_tbl_op.h
View File

@ -77,6 +77,11 @@ private:
vector<IndexInfo> vec_intersection(vector<IndexInfo> &a, vector<IndexInfo> &b);
vector<IndexInfo> vec_union(vector<IndexInfo> &a, vector<IndexInfo> &b);
vector<IndexInfo> vec_difference(vector<IndexInfo> &a, vector<IndexInfo> &b);
int ShiftIntelligentInfo(IntelligentInfo &info, int len);
bool GetSuggestDoc(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid);
bool GetSuggestDocWithoutCharacter(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set);
int GetDocByShiftWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid);
int GetDocByShiftEnWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid);
extern CIndexTableManager g_IndexInstance;
extern CIndexTableManager g_hanpinIndexInstance;

2
src/search_local/index_read/process/range_query_process.cc
View File

@ -34,7 +34,7 @@ int RangeQueryProcess::ParseContent(int logic_type)
if(field_value.isObject()){
Json::Value start;
Json::Value end;
RANGTYPE ui_range_type;
RANGTYPE ui_range_type = RANGE_INIT;
if(field_value.isMember(GTE)){
start = field_value[GTE];
if(field_value.isMember(LTE)){

29
src/search_local/index_read/request_context.cc
View File

@ -77,14 +77,22 @@ int RequestContext::ParseJson(const char *sz_json, int json_len, Json::Value &re
page_index_ = ParseJsonReturnInt("page_index" , recv_packet);
}
else {
page_index_ = 1 ;
if (recv_packet.isMember("from")){
page_index_ = ParseJsonReturnInt("from" , recv_packet);
} else {
page_index_ = 1 ;
}
}
if (recv_packet.isMember("page_size")){
page_size_ = ParseJsonReturnInt("page_size" , recv_packet);
}
else {
page_size_ = 10;
if (recv_packet.isMember("size")){
page_index_ = ParseJsonReturnInt("size" , recv_packet);
} else {
page_size_ = 10;
}
}
if(recv_packet.isMember("sort_type")){
@ -98,7 +106,22 @@ int RequestContext::ParseJson(const char *sz_json, int json_len, Json::Value &re
sort_field_ = recv_packet["sort_field"].asString();
}
else {
sort_field_ = "";
if (recv_packet.isMember("sort") && recv_packet["sort"].isArray()){
if(recv_packet["sort"].size() > 0){
Json::Value sort_info = recv_packet["sort"][0];
Json::Value::Members member = sort_info.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
sort_field_ = *iter;
Json::Value order_info = sort_info[*iter];
if(order_info.isMember("order") && order_info["order"].isString()){
sort_type_ = (order_info["order"].asString() == "desc" ? SORT_FIELD_DESC : SORT_FIELD_ASC);
}
} else {
sort_field_ = "";
}
} else {
sort_field_ = "";
}
}
if (recv_packet.isMember("return_all")){

141
src/search_local/index_read/search_util.cc
View File

@ -1056,147 +1056,6 @@ uint32_t GetIpNum(string ip)
return ntohl(s);
}
int ShiftIntelligentInfo(IntelligentInfo &info, int len)
{
uint16_t *p = &info.charact_id[0];
if (len >= 8) {
return -1;
}
if (len < 1) {
return -2;
}
memmove(&p[len], &p[0], sizeof(uint16_t) * (8 - len));
for (int i = 0; i < len; i++) {
p[i] = 0;
}
p = &info.phonetic_id[0];
memmove(&p[len], &p[0], sizeof(uint16_t) * (8 - len));
for (int i = 0; i < len; i++) {
p[i] = 0;
}
GetInitialVec(info, len);
return 0;
}
bool GetSuggestDoc(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
bool bRet;
int index = 0;
uint32_t field = fieldInfo.field;
uint32_t segment_feature = fieldInfo.segment_feature;
bRet = g_hanpinIndexInstance.GetSuggestDoc(appid, index, len, field, info, doc_id_set);
if (bRet == false)
goto resError;
if (segment_feature == 0) {
return true;
}
for (uint32_t i = 1; i <= 8 - len; i++) {
index = index + 1;
IntelligentInfo t2 = info;
if (ShiftIntelligentInfo(t2, i) < 0) {
continue;
}
bRet = g_hanpinIndexInstance.GetSuggestDoc(appid, index, len, field, t2, doc_id_set);
if (bRet == false)
goto resError;
}
return true;
resError:
log_error("GetSuggestDoc invalid.");
return false;
}
bool GetSuggestDocWithoutCharacter(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
bool bRet;
int index = 0;
uint32_t field = fieldInfo.field;
uint32_t segment_feature = fieldInfo.segment_feature;
bRet = g_hanpinIndexInstance.GetSuggestDocWithoutCharacter(appid, index, len, field, info, doc_id_set);
if (bRet == false)
goto resError;
if (segment_feature == 0) {
return true;
}
for (uint32_t i = 1; i <= 16 - len; i++) {
index = index + 1;
IntelligentInfo t2 = info;
if (ShiftIntelligentInfoWithoutCharacter(t2, i) < 0) {
continue;
}
bRet = g_hanpinIndexInstance.GetSuggestDocWithoutCharacter(appid, index, len, field, t2, doc_id_set);
if (bRet == false)
goto resError;
}
return true;
resError:
log_error("GetSuggestDoc invalid.");
return false;
}
int GetDocByShiftWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
log_debug("GetDocByShiftWord start");
bool bRet = true;
set<vector<Content> > result;
GetMultipleWords(fieldInfo.word, result);
int len = result.size();
if (len <= 0) {
log_error("get shift word error.");
return -RT_GET_SUGGEST_ERR;
}
set<vector<Content> >::iterator iter;
for (iter = result.begin(); iter != result.end(); iter++)
{
IntelligentInfo info;
vector<Content> tmp = *iter;
ConvertIntelligent(tmp, info, bRet);
int length = tmp.size();
if (bRet) {
bRet = GetSuggestDoc(fieldInfo, length, info, doc_id_set, appid);
if (!bRet) {
log_error("GetSuggestDocInfo error.");
return -RT_DTC_ERR;
}
}
}
return 0;
}
int GetDocByShiftEnWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid)
{
log_debug("GetDocByShiftEnWord start");
bool bRet = true;
int length = 0;
IntelligentInfo enInfo;
ConvertCharIntelligent(fieldInfo.word, enInfo, length);
bRet = GetSuggestDocWithoutCharacter(fieldInfo, length, enInfo, doc_id_set, appid);
if (!bRet) {
log_error("GetEnSuggestDocInfo error.");
return -RT_DTC_ERR;
}
return 0;
}
uint64_t GetSysTimeMicros(){
timeval tv;
memset(&tv, 0, sizeof(tv));

6
src/search_local/index_read/search_util.h
View File

@ -27,7 +27,6 @@
#include "utf8_str.h"
#include "comm.h"
#include "search_conf.h"
#include "index_tbl_op.h"
#include "result_context.h"
using namespace std;
struct GeoPointContext;
@ -65,11 +64,6 @@ void ConvertIntelligent(const vector<Content> &result, IntelligentInfo &info, bo
bool GetGisCode(string lng, string lat, string ip, double distance, vector<string>& gisCode);
bool GetGisCode(const vector<string>& lng_arr, const vector<string>& lat_arr, vector<string>& gisCode);
uint32_t GetIpNum(string ip);
int ShiftIntelligentInfo(IntelligentInfo &info, int len);
bool GetSuggestDoc(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set, uint32_t appid);
bool GetSuggestDocWithoutCharacter(FieldInfo& fieldInfo, uint32_t len, const IntelligentInfo &info, vector<IndexInfo> &doc_id_set);
int GetDocByShiftWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid);
int GetDocByShiftEnWord(FieldInfo fieldInfo, vector<IndexInfo> &doc_id_set, uint32_t appid);
uint64_t GetSysTimeMicros();
string trim(string& str);
string delPrefix(string& str);

6
src/search_local/index_read/unittest/CMakeLists.txt
View File

@ -1,11 +1,11 @@
cmake_minimum_required(VERSION 2.6)
PROJECT(correction_unittest)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++11 -lrt -Wall -O2")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fprofile-arcs -ftest-coverage -lrt -Wall -O2")
AUX_SOURCE_DIRECTORY(. main)
set(OTHER_FILE "../correction/correction.cc" "../utils/get_aois_action.cc"
"../index_tbl_op.cc" "../search_conf.cc" "../split_manager.cc" "../data_manager.cc" "../search_util.cc"
"../search_conf.cc" "../split_manager.cc" "../data_manager.cc" "../search_util.cc"
"../db_manager.cc" "../request_context.cc" "../result_context.cc")
LINK_DIRECTORIES(
@ -29,7 +29,7 @@ target_include_directories(correction_unittest PUBLIC
../../../search_local/index_storage/api/c_api_cc
)
target_link_libraries(correction_unittest common gtest_main gtest jsoncpp stat dtc dl)
target_link_libraries(correction_unittest common gtest_main gtest jsoncpp stat dl -lpthread -lm)
SET_TARGET_PROPERTIES(correction_unittest PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./bin")

61
src/search_local/index_read/unittest/correction_unittest.cc
View File

@ -50,6 +50,9 @@ TEST_F(CorrectionTest, CHINESE_) {
correction_->JudgeWord(10001, "京栋", is_correct, probably_word);
EXPECT_FALSE(is_correct);
EXPECT_EQ("京东", probably_word);
correction_->JudgeWord(10001, "荒鸟", is_correct, probably_word);
EXPECT_FALSE(is_correct);
EXPECT_EQ("荒岛", probably_word);
}
TEST_F(CorrectionTest, HYBRID_) {
@ -83,4 +86,62 @@ TEST_F(CorrectionTest, ENGLISH_) {
correction_->JudgeWord(10001, "appla", is_correct, probably_word);
EXPECT_FALSE(is_correct);
EXPECT_EQ("applaud", probably_word);
}
TEST_F(CorrectionTest, DataManager_Sensitive_) {
bool ret = false;
ret = DataManager::Instance()->IsSensitiveWord("兴奋剂");
EXPECT_TRUE(ret);
ret = DataManager::Instance()->IsSensitiveWord("京东");
EXPECT_FALSE(ret);
}
TEST_F(CorrectionTest, DataManager_Relate_) {
bool ret = false;
vector<string> word_vec;
ret = DataManager::Instance()->GetRelateWord("apple", word_vec, 1);
EXPECT_TRUE(ret);
EXPECT_EQ(1, word_vec.size());
word_vec.clear();
ret = DataManager::Instance()->GetRelateWord("banana", word_vec, 1);
EXPECT_FALSE(ret);
vector<vector<string> > relate_vvec;
vector<string> relate_vec;
relate_vec.push_back("pear");
relate_vec.push_back("梨");
relate_vvec.push_back(relate_vec);
int i_ret = DataManager::Instance()->UpdateRelateWord(relate_vvec);
EXPECT_EQ(0, i_ret);
}
TEST_F(CorrectionTest, DataManager_Synonym_) {
vector<FieldInfo> keys;
DataManager::Instance()->GetSynonymByKey("京东白条", keys);
EXPECT_EQ(1, keys.size());
keys.clear();
DataManager::Instance()->GetSynonymByKey("金条", keys);
EXPECT_EQ(0, keys.size());
}
TEST_F(CorrectionTest, DataManager_Word_) {
uint32_t id;
int ret = DataManager::Instance()->GetWordId("京东", 0, id);
EXPECT_EQ(0, ret);
EXPECT_EQ(7754420, id);
}
TEST_F(CorrectionTest, DataManager_Hot_) {
bool ret = false;
vector<string> word_vec;
ret = DataManager::Instance()->GetHotWord(10001, word_vec, 5);
EXPECT_TRUE(ret);
EXPECT_EQ(3, word_vec.size());
map<uint32_t, map<string, uint32_t> > new_hot_map;
map<string, uint32_t> word_map;
word_map["apple"] = 1;
word_map["pear"] = 3;
word_map["banana"] = 2;
new_hot_map[10001] = word_map;
int i_ret = DataManager::Instance()->UpdateHotWord(new_hot_map);
EXPECT_EQ(0, i_ret);
}

4
src/search_local/index_read/unittest/data/analyze_data Normal file
View File

@ -0,0 +1,4 @@
10001 apple 1
10001 pear 2
10001 banana 3
10002 apple 2

2
src/search_local/index_read/unittest/data/relate_data Normal file
View File

@ -0,0 +1,2 @@
apple 苹果
京东 京东商城

1
src/search_local/index_read/unittest/data/sensitive_data Normal file
View File

@ -0,0 +1 @@
兴奋剂

1
src/search_local/index_read/unittest/data/synonym_data Normal file
View File

@ -0,0 +1 @@
京东白条 白条

2
src/search_local/index_storage/rocksdb_helper/Makefile
View File

@ -6,7 +6,7 @@ CC = g++
.SUFFIXES: .cc .h
FLAGS = -g -std=c++11 -DROCKSDB_COMPILER=1 -DOMN_PLATFORM_UNIX $(Optimize) -Wl,--no-undefined -Xlinker -zmuldefs
FLAGS = -g -O2 -std=c++11 -DROCKSDB_COMPILER=1 -DOMN_PLATFORM_UNIX $(Optimize) -Wl,--no-undefined -Xlinker -zmuldefs
#FLAGS = -O3 -DOMN_PLATFORM_UNIX $(Optimize)
INCDIRS = -I.. -I../common -I../watchdog -I../cache -I../stat -I../../../3rdlib/rocksdb/include -I../../../3rdlib/rocksdb/logging -I../../../3rdlib/attr_api -I../../../comm

11
src/search_local/index_storage/rocksdb_helper/db_process_rocks.cc
View File

@ -748,7 +748,7 @@ bool RocksdbProcess::is_matched(
ret = memcmp(lv, rv, minLen);
else
ret = memcmp_ignore_case(lv, rv, minLen);
log_error("iret:%d , len:%d ,rLen:%d", ret , lLen , rLen);
//log_error("iret:%d , len:%d ,rLen:%d", ret , lLen , rLen);
return ret < 0 || (ret == 0 && lLen <= rLen);
case 4:
if (caseSensitive)
@ -998,7 +998,9 @@ void RocksdbProcess::build_direct_row(
}
int dataLen = fieldValue.length();
row.append(std::string((char *)&dataLen, 4)).append(fieldValue);
//row.append(std::string((char *)&dataLen, 4)).append(fieldValue);
row += std::string((char *)&dataLen, 4);
row += fieldValue;
}
((RangeQueryRows_t*)respCxt->sDirectRespValue.uRangeQueryRows)->sRowValues.push_front(row);
@ -2489,6 +2491,7 @@ int RocksdbProcess::process_direct_query(
#ifdef PRINT_STAT
mETime = GET_TIMESTAMP();
insert_stat(OprType::eDirectQuery, mETime - mSTime);
log_debug("cost time: %lld", (long long int)(mETime - mSTime));
#endif
return (0);
@ -3099,6 +3102,10 @@ int RocksdbProcess::split_values(
char *head = const_cast<char *>(compoundValue.data());
for (int idx = 0; idx < mExtraValueFieldNums; idx++)
{
if(idx == mExtraValueFieldNums-1){ // extend field no need to parse
values.push_back("");
break;
}
ret = get_value_by_id(head, mFieldIndexMapping[mCompoundKeyFieldNums + idx], value);
assert(ret == 0);
values.push_back(std::move(value));

7
src/search_local/index_storage/rocksdb_helper/key_format.cc
View File

@ -514,13 +514,14 @@ void key_format::DecodeBytes(const std::string &src, std::string &dst)
{
dst = "";
}
std::stringstream oss_dst;
//std::stringstream oss_dst;
for (size_t i = 0; i < src.length(); i += SEGMENT_SIZE)
{
char padding_bytes = ENCODER_MARKER - src[i + 7];
oss_dst << src.substr(i, SEGMENT_SIZE - 1 - padding_bytes);
//oss_dst << src.substr(i, SEGMENT_SIZE - 1 - padding_bytes);
dst += src.substr(i, SEGMENT_SIZE - 1 - padding_bytes);
}
dst = oss_dst.str();
//dst = oss_dst.str();
}
void key_format::DecodeBytes(const std::string &src, uint64_t &dst)

14
src/search_local/index_write/CMakeLists.txt
View File

@ -1,11 +1,12 @@
option(jdtestOpen "Build gtest's units programs." OFF)
option(INDEX_WRITE_OUTPUT "Build index_write programs." ON)
cmake_minimum_required(VERSION 2.6)
PROJECT(index_write_unit)
EXECUTE_PROCESS(COMMAND git log -1 --pretty=format:%h . OUTPUT_VARIABLE version)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -g -lrt -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -fprofile-arcs -ftest-coverage -g -lrt -Wall")
AUX_SOURCE_DIRECTORY(. main)
@ -14,9 +15,9 @@ ${PROJECT_SOURCE_DIR}/../../comm
${PROJECT_SOURCE_DIR}/../../3rdlib/jsoncpp/lib
${PROJECT_SOURCE_DIR}/../../3rdlib/google_test/lib
${PROJECT_SOURCE_DIR}/../../comm/stat
${PROJECT_SOURCE_DIR}/../index_storage/api/c_api_cc
)
ADD_EXECUTABLE(index_write ${main})
INCLUDE_DIRECTORIES(
${PROJECT_SOURCE_DIR}/../../3rdlib/jsoncpp/include
@ -26,10 +27,13 @@ ${PROJECT_SOURCE_DIR}/../../comm/stat
${PROJECT_SOURCE_DIR}/../index_storage/api/c_api_cc
)
add_definitions(-DGIT_VERSION="${version}" -DMAIN)
target_link_libraries(index_write libcommon.a libdtc.so jsoncpp stat ssl)
if(INDEX_WRITE_OUTPUT)
ADD_EXECUTABLE(index_write ${main})
add_definitions(-DGIT_VERSION="${version}" -DMAIN)
target_link_libraries(index_write pthread libcommon.a libdtc.so jsoncpp stat ssl)
SET_TARGET_PROPERTIES(index_write PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./bin")
endif()
if(jdtestOpen)
cmake_minimum_required(VERSION 2.8.8)
@ -52,7 +56,7 @@ if(jdtestOpen)
AUX_SOURCE_DIRECTORY(./jdtest jdtestFiles)
ADD_EXECUTABLE(gtest_index_write ${main} ${jdtestFiles})
TARGET_LINK_LIBRARIES(gtest_index_write gmock gtest pthread libcommon.a libdtc.so jsoncpp stat ssl)
TARGET_LINK_LIBRARIES(gtest_index_write gmock gtest pthread libcommon.a libdtc.so jsoncpp stat ssl)
SET_TARGET_PROPERTIES(gtest_index_write PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./bin")
install(TARGETS gtest_index_write RUNTIME DESTINATION bin)
endif()

11
src/tools/python_httprunner/README.md Normal file
View File

@ -0,0 +1,11 @@
### http接口测试工具
该python脚本来源于公司内部的http测试工具：https://coding.jd.com/fangjiankang/python_httprunner.git
需安装的python依赖：pip install requests xlrd xlwt colorama
使用文档：https://cf.jd.com/pages/viewpage.action?pageId=478195770
开始执行测试的命令为：python start.py
测试结果在report目录

22
src/tools/python_httprunner/checker.py Normal file
View File

@ -0,0 +1,22 @@
def check_options(case_options):
"""
检查post请求参数的合法性
:param case_options: 用例内容
:return:
"""
if 'url' not in case_options.keys():
print("url 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return False
if 'body' not in case_options.keys():
print("body 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return False
if 'headers' not in case_options.keys():
print("headers 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return False
if 'assert_type' not in case_options.keys():
print("assert_type 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return False
if 'assert_value' not in case_options.keys():
print("assert_value 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return False
return True

5
src/tools/python_httprunner/config.py Normal file
View File

@ -0,0 +1,5 @@
testcase_file_path = './myCase.xls' # 测试用例源文件地址
report_path = "./report" # 测试报告生成父级路径
xls_report = True # 是否生成测试报告，填写 True 或 False
output_model = "detail" # 定义命令行输出接口执行时输出内容的详细程度，传值要求：simple（简要） 或 detail（详细）

148
src/tools/python_httprunner/executor.py Normal file
View File

@ -0,0 +1,148 @@
import net_request
import checker
import time
import json
import config
import colorama
from colorama import init, Fore, Back, Style
init(autoreset=True)
def run(testcase_list):
print('\n\033[1;33m========= 执行接口测试 ==========\033[0m')
result_list = []
i = 1
for testcase in testcase_list:
print('[测试接口%s]' % i)
result_list.append(run_case(testcase))
i += 1
return result_list
def run_case(case_options):
"""
运行测试用例
:param case_options: 测试用例执行所需要的相关参数字典
:return:返回空值
"""
# 如果从用例参数中不能获取request_type，则用例跳出，不做执行
if 'request_type' not in case_options.keys():
print("request_type 参数未设置,跳过此调用例执行，参数为：%s" % case_options)
return
request_type = case_options['request_type'] # 获得请求类型
response_result = '' # 请求返回后的存储变量
begin_time = time.time() # 记录请求前的时间
# 执行post类型的请求
if request_type.lower() == 'post':
# 对用例参数的完整请进行检测
if not checker.check_options(case_options):
return
# 发送post请求
response_result = net_request.post(case_options['url'], case_options['headers'], case_options['body'])
# 执行get类型的请求
if request_type.lower() == 'get':
if not checker.check_options(case_options):
return
# 发送get请求
response_result = net_request.get(case_options['url'], case_options['headers'], case_options['body'])
end_time = time.time() # 记录请求完成后的时间
# 对执行结果进行判断，检查是否用例的通过情况
check_result = analyse_result(response_result, case_options['assert_value'], case_options['assert_type'])
# 将执行信息输出到控制台
cost_time = round(end_time - begin_time, 3)
output_execute_info(case_options, response_result, check_result, cost_time)
# 将执行结果进行组装并返回给调用方
return {'case_options': case_options, 'response_result': response_result, 'check_result': check_result,
'cost_time': cost_time}
def analyse_result(real_result, assert_value, assert_type):
"""
分析请求返回的测试结果
:param real_result: 返回请求的结果，json串
:param assert_value: 期望结果字串，来自请求的case_options字典
:param assert_type: 断言的判断方式，来自请求的case_options字典（提供：包含、相等）
:return:返回判断的结果，通过未为True，失败为False
"""
# 处理包含的逻辑，如果请求返回结果包含断言值，则判断结果为True
if '包含' == assert_type:
if json.dumps(real_result, ensure_ascii=False).__contains__(str(assert_value)):
return True
# 处理相等的逻辑，如果请求返回结果与断言值相同，则判断结果为True
if '相等' == assert_type:
if str(assert_value) == json.dumps(real_result, ensure_ascii=False):
return True
return False
def output_execute_info(case_options, response_result, check_result, time_consuming):
"""
在命令行输出测试执行报告信息(支持简单模式和详细模式输出)
:param case_options: 原测试用例信息
:param response_result: 请求返回结果
:param check_result: 测试结果，True、False
:param time_consuming: 测试用例执行耗时
:return:
"""
if config.output_model.lower() == 'simple':
simple_output(case_options, response_result, check_result, time_consuming)
elif config.output_model.lower() == 'detail':
detail_output(case_options, response_result, check_result, time_consuming)
else:
print("请到config.py文件中配置输出模式（output_model）！")
return
def detail_output(case_options, response_result, check_result, time_consuming):
"""
在命令行输出测试执行报告信息(详细模式)
:param case_options: 原测试用例信息
:param response_result: 请求返回结果
:param check_result: 测试结果，True、False
:param time_consuming: 测试用例执行耗时
:return:
"""
print("请求接口：", case_options['url'])
print("接口名称：", case_options['interface_name'])
print("请求类型：", case_options['request_type'].upper())
print("请求参数：", case_options['body'])
if check_result: # 按测试结果对返回内容进行着色输出
print('返回结果： \033[1;32;40m' + json.dumps(response_result, ensure_ascii=False) + '\033[0m')
print('断言内容： \033[1;32;40m' + case_options['assert_value'] + '\033[0m')
else:
print('返回结果： \033[1;31;40m' + json.dumps(response_result, ensure_ascii=False) + '\033[0m')
print('断言内容： \033[1;31;40m' + case_options['assert_value'] + '\033[0m')
print('断言方式： %s' % case_options['assert_type'])
print("执行耗时：", time_consuming, '秒')
test_result = '\033[1;32;40m通过\033[0m' if check_result is True else '\033[1;31;40m失败\033[0m' # 按测试结果染色命令行输出
print('测试结果：', test_result) # 无高亮
print("\n")
return
def simple_output(case_options, response_result, check_result, time_consuming):
"""
在命令行输出测试执行报告信息(简单模式)
:param case_options: 原测试用例信息
:param response_result: 请求返回结果
:param check_result: 测试结果，True、False
:return:
"""
print("请求接口：", case_options['url'])
if check_result: # 按测试结果对返回内容进行着色输出
print('返回结果： \033[1;32;40m' + json.dumps(response_result, ensure_ascii=False)[0:120] + '......' + '\033[0m')
print('断言内容： \033[1;32;40m' + case_options['assert_value'] + '\033[0m')
else:
print('返回结果： \033[1;31;40m' + json.dumps(response_result, ensure_ascii=False)[0:120] + '......' + '\033[0m')
print('断言内容： \033[1;31;40m' + case_options['assert_value'] + '\033[0m')
print("执行耗时：", time_consuming, '秒')
test_result = '\033[1;32;40m通过\033[0m' if check_result is True else '\033[1;31;40m失败\033[0m' # 按测试结果染色命令行输出
print('测试结果：', test_result) # 无高亮
print("\n")
return

BIN
src/tools/python_httprunner/myCase.xls Normal file
View File

Binary file not shown.

33
src/tools/python_httprunner/net_request.py Normal file
View File

@ -0,0 +1,33 @@
import requests
import traceback
def post(url, headers=None, body=None):
"""
发送post请求
:param url:请求地址
:param headers:头部信息
:param body:请求入参
:return:
"""
response_post = requests.post(url, json=body, headers=headers)
res = response_post.text
if res.__contains__('<title>登录页</title>'):
return {"错误": "cookie失效"}
return response_post.json()
def get(url, headers, body):
"""
发送get请求
:param headers: 头部信息
:param url:请求地址
:param body:请求入参
:return:
"""
response_get = requests.get(url, params=body, headers=headers)
res = response_get.text
if res.__contains__('<title>登录页</title>'):
return {"错误": "cookie失效"}
return response_get.json()

62
src/tools/python_httprunner/read_testcase.py Normal file
View File

@ -0,0 +1,62 @@
import xlrd
import os
def read(testcase_file_path):
"""
读取接口测试用例源文件，将读取的有效用例写入字典
:param testcase_file_path: 读取用例文件的路径地址
:return:
"""
if not os.path.exists(testcase_file_path):
print("%s 文件不存在，无法读取到文件内容" % testcase_file_path)
return
# 在命令行打印提示信息
print('\n\033[1;33m====== 读取测试用例列表文件 ======\033[0m')
print("文件地址：%s" % testcase_file_path)
# 读取xls文件内容
xls = xlrd.open_workbook(testcase_file_path)
sh = xls.sheet_by_index(0)
# 获得公共的头部文本信息
common_header_text = sh.row_values(1)[2]
# 头部无效行数定义，文件头3行是描述文本和列表头部栏位，不作为测试用例内容
head_row_num = 3
# 头部无效行索引定义，索引从0开始，文件0~2索引是描述文本和列表头部栏位，不作为测试用例内容
head_row_index = head_row_num - 1
# 预判断有效测试用例行数（即处理头部定义外可以视为测试用例的行，但不一定准确，比如出现行乱填的情况，需要在程序中进行排除）
total_row_num = sh.nrows # 文件总行数
need_read_num = total_row_num - head_row_num # 需要读取的总行数
i = 1
testcase_list = []
print("读取到被测接口信息如下：")
while i <= need_read_num:
row_index = head_row_index + i # 获得需要读取的行索引
row_testcase_info = sh.row_values(row_index) # 获取到一行测试用例文本信息
is_execute = row_testcase_info[0]
if is_execute == '是': # 只有"是否执行"单元格为 是 时才将用例加入被执行序列
print(str(row_testcase_info)[0:120] + '......')
# 将每条测试用例组合为一个字典进行存储
testcase = {'interface_name': row_testcase_info[1], 'url': row_testcase_info[2],
'assert_type': row_testcase_info[6],
'assert_value': row_testcase_info[7]}
request_type = row_testcase_info[3]
testcase['request_type'] = request_type
body = row_testcase_info[5]
body.strip()
if len(body) > 0: # 如果body没有填值，则默认随便给一个任意值
testcase['body'] = eval(row_testcase_info[5])
else:
testcase['body'] = {"test": "test"}
headers = row_testcase_info[4] # 获得行中的headers文本
if headers == "公共头部": # 如果文本填写内文为 '公共头部'，则取文件第二行的"公共头部信息"文本值
headers = common_header_text
testcase['headers'] = eval(headers)
testcase_list.append(testcase)
i += 1
print('用例总数：\033[1;32;40m%s条\033[0m' % len(testcase_list))
return testcase_list

21
src/tools/python_httprunner/reporter.py Normal file
View File

@ -0,0 +1,21 @@
import colorama
from colorama import init, Fore, Back, Style
init(autoreset=True)
def output_report(result_list):
print('\033[1;33m========== 统计测试结果 ==========\033[0m')
print("执行总数： %s" % len(result_list))
success_count = 0
fail_count = 0
cost_time = 0
for result in result_list:
cost_time += result['cost_time']
if result['check_result']:
success_count += 1
else:
fail_count += 1
print('成功/失败：\033[1;32;40m%s\033[0m / \033[1;31;40m%s\033[0m' % (success_count, fail_count))
print("执行总时长：%s 秒\n" % round(cost_time, 3))
return

17
src/tools/python_httprunner/start.py Normal file
View File

@ -0,0 +1,17 @@
import executor
import read_testcase
import write_xls_report
import config
import reporter
# 1、读取测试用例列表文件
testcase_list = read_testcase.read(config.testcase_file_path)
# 2、执行用例列表中的用例
result_list = executor.run(testcase_list)
# 3、统计测试结果并在终端进行显示
reporter.output_report(result_list)
# 4、将测试结果写入xls文件中
write_xls_report.write(result_list)

98
src/tools/python_httprunner/write_xls_report.py Normal file
View File

@ -0,0 +1,98 @@
import xlwt
import config
import time
import os
import json
import colorama
from colorama import init, Fore, Back, Style
init(autoreset=True)
def write(result_list):
"""
将执行测试结果的总汇内容写入xls文件中
:param result_list:
:return:
"""
# 读取配置文件，若开启xls报告，则进行xsl报告生成
if not config.xls_report:
return
if not os.path.exists(config.report_path): # 从配置文件中获取父目录路径，若目录不存在则进行创建
os.mkdir(config.report_path)
# 拼接生成的报告文件路径
report_path = os.path.join(config.report_path, time.strftime("%Y%m%d%H%M%S", time.localtime()) + '_report.xls')
print('\033[1;33m====== 生成xls格式的测试报告 ======\033[0m')
# 创建一个workbook 设置编码
workbook = xlwt.Workbook(encoding='utf-8')
# 创建一个worksheet
worksheet = workbook.add_sheet('测试结果')
# 设置列表头部样式
alignment = xlwt.Alignment()
alignment.horz = xlwt.Alignment.HORZ_CENTER
alignment.vert = xlwt.Alignment.VERT_CENTER
header_style = xlwt.XFStyle()
header_style.alignment = alignment
col1 = worksheet.col(0)
col1.width = 256 * 8
col2 = worksheet.col(1)
col2.width = 256 * 35
col3 = worksheet.col(2)
col3.width = 256 * 80
col4 = worksheet.col(3)
col4.width = 256 * 10
col5 = worksheet.col(4)
col5.width = 256 * 40
col6 = worksheet.col(5)
col6.width = 256 * 12
col7 = worksheet.col(6)
col7.width = 256 * 10
# 设置表头字体大小和颜色
font = xlwt.Font()
font.height = 20 * 12
font.colour_index = 17
header_style.font = font
# 设置头部第一行的行高
tall_style = xlwt.easyxf('font:height 480')
row1 = worksheet.row(0)
row1.set_style(tall_style)
# 定义表格头部栏文本
worksheet.write(0, 0, '序号', header_style)
worksheet.write(0, 1, '接口名称', header_style)
worksheet.write(0, 2, '被测接口', header_style)
worksheet.write(0, 3, '请求方式', header_style)
worksheet.write(0, 4, '请求参数', header_style)
worksheet.write(0, 5, '耗时(秒)', header_style)
worksheet.write(0, 6, '测试结果', header_style)
worksheet.write(0, 7, '失败备注', header_style)
# 结果单元格样式
col_style = xlwt.XFStyle()
col_style.alignment = alignment
style_success = xlwt.easyxf('pattern: pattern solid, fore_colour green')
style_success.alignment = alignment
style_fail = xlwt.easyxf('pattern: pattern solid, fore_colour red')
style_fail.alignment = alignment
index = 1 # 设置序号
for result in result_list: # 循环遍历出测试结果并按顺序写入xls文件中
worksheet.write(index, 0, index, col_style)
worksheet.write(index, 1, result['case_options']['interface_name'])
worksheet.write(index, 2, result['case_options']['url'])
worksheet.write(index, 3, str(result['case_options']['request_type']).upper(), col_style)
if result['case_options'].get("body", ) is not None: # 存在body参数时则输出到文件
worksheet.write(index, 4, json.dumps(result['case_options'].get('body', )))
worksheet.write(index, 5, result['cost_time'], col_style)
res = "通过" if result['check_result'] else '失败' # 通过检测结果True和False来重新设置结果文本问 通过和失败
if result['check_result']:
worksheet.write(index, 6, res, style_success) # 成功文本背景染色为绿色
else:
worksheet.write(index, 6, res, style_fail) # 成功文本背景染色为红色
worksheet.write(index, 7, '断言内容：' + str(result['case_options']['assert_value']) + "\n\r实际返回：" + str(
result['response_result'])[0:30000]) # 失败用例，将断言内容和实际结果进行输出,实际结果输出长度在30000以内
index += 1
workbook.save(report_path)
print("报告成功创建：" + str(report_path))
return

55
src/tools/search_monitor/conf/localCluster.json Normal file
View File

@ -0,0 +1,55 @@
{
"MODULE" :
{
"SERVERSHARDING" :
[
{
"ShardingName" : "keyword_index_data1743",
"INSTANCE" :
[
{
"ip" : "11.80.17.227",
"port" : "12003",
"weight" : 1
},
{
"ip" : "11.50.77.97",
"port" : "12003",
"weight" : 1
},
{
"ip" : "11.80.17.227",
"port" : "11017",
"weight" : 1,
"role" : "index_gen",
"disasterRole" : "master"
},
{
"ip" : "11.50.77.97",
"port" : "11017",
"weight" : 1,
"role" : "index_gen",
"disasterRole" : "replicate"
}
]
},
{
"ShardingName" : "keyword_index_data1744",
"INSTANCE" :
[
{
"ip" : "11.50.77.97",
"port" : "12003",
"weight" : 1
},
{
"ip" : "11.50.77.97",
"port" : "11017",
"weight" : 1 ,
"role" : "index_gen"
}
]
}
]
}
}

11
src/tools/search_monitor/conf/localCluster.xml Normal file
View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<MODULE>
<SERVERSHARDING ShardingName="keyword_index_data1010">
<INSTANCE ip="11.80.17.227" port="12003" weight="1"/>
<INSTANCE ip="11.80.17.227" port="11017" weight="1" role="index_gen"/>
</SERVERSHARDING>
<SERVERSHARDING ShardingName="keyword_index_data1011">
<INSTANCE ip="11.50.77.97" port="12003" weight="1"/>
<INSTANCE ip="11.50.77.97" port="11017" weight="1" role="index_gen"/>
</SERVERSHARDING>
</MODULE>

48
src/tools/search_monitor/conf/monitorCluster.conf Normal file
View File

@ -0,0 +1,48 @@
{
"logLevel":7,
"logFilePath":"../log",
"invokeTimeout":500,
"physicalId":3,
"reportAlarmUrl":"http://monitor.m.jd.com/tools/new_alarm_api/send_alarm",
"alarmReceivers":["chenyujie28@jd.com"],
"getPhysicalInfoUrl" : "http://g1.jsf.jd.local/com.jd.cap.data.api.service.XCMDBService/cap_read/findByQuery/8912/",
"listenAddr":
{
"ip":"11.80.17.227",
"port":17877
},
"clusterHosts":
[
{
"ip":"11.80.17.227",
"port":17878
},
{
"ip":"11.80.17.227",
"port":17879
}
],
"adminHosts":
[
{
"ip":"192.168.144.126",
"port":9988
}
],
"dtcInfo":
{
"detectPeriod":100,
"detectStep":300,
"detectTimeoutSet":
{
"sameZoneTimeout":1000,
"domesticZoneTimeout":2000,
"abroadZoneTimeout":5000
}
},
"Config":
{
"CaDir":"/usr/local/dtcadmin/ca/conf/",
"CaPid":1606976164
}
}

348
src/tools/search_monitor/src/DBInstance.cpp Normal file
View File

@ -0,0 +1,348 @@
/////////////////////////////////////////////////////////////////
//
// Handle access to mysql database
// created by qiuyu on Feb 21, 2019
////////////////////////////////////////////////////////////////
#include "DBInstance.h"
#include "DtcMonitorConfigMgr.h"
#include "log.h"
#include <errmsg.h>
#include <string.h>
DBInstance::DBInstance()
{
}
DBInstance::~DBInstance()
{
closeDB();
}
void DBInstance::freeResult()
{
mysql_free_result(mDBConnection.sMyRes);
mDBConnection.sMyRes = NULL;
mDBConnection.sMyRow = NULL;
}
// initialize the db connection in this instance
bool DBInstance::initDB(
const std::string& dbHostName,
const int port,
const std::string& dbName,
const std::string& dbUser,
const std::string& dbPassword,
const int iConnectTimeOut,
const int iReadTimeOut,
const int iWriteTimeOut)
{
monitor_log_error("Database initilized info. host:%s, port:%d, dbName:%s, dbUser:%s,\
dbPassword:%s", dbHostName.c_str(), port, dbName.c_str(), dbUser.c_str(), dbPassword.c_str());
if (dbHostName.empty() || port <= 0 || dbName.empty() || dbUser.empty() || dbPassword.empty())
{
monitor_log_error("invalid database initilized info.");
return false;
}
mDBConnection.sMyRes = NULL;
mDBConnection.hasConnected = false;
mDBConnection.inTransaction = false;
mDBConnection.sDBInfo.sDBHostName = dbHostName;
mDBConnection.sDBInfo.sDBPort = port;
mDBConnection.sDBInfo.sDBUserName = dbUser;
mDBConnection.sDBInfo.sPassword = dbPassword;
mDBConnection.sDBInfo.sDBName = dbName;
// init mysql
mysql_init(&mDBConnection.sMysqlInstance);
// hard code
mysql_options(&mDBConnection.sMysqlInstance, MYSQL_SET_CHARSET_NAME, "utf8");
// reconnect if connection lost from Mysql for alive timeout
bool reconnect = true;
mysql_options(&mDBConnection.sMysqlInstance, MYSQL_OPT_RECONNECT, &reconnect);
mysql_options(&mDBConnection.sMysqlInstance, MYSQL_OPT_CONNECT_TIMEOUT,(const char *)&iConnectTimeOut);
mysql_options(&mDBConnection.sMysqlInstance, MYSQL_OPT_READ_TIMEOUT,(const char *)&iReadTimeOut);
mysql_options(&mDBConnection.sMysqlInstance, MYSQL_OPT_WRITE_TIMEOUT,(const char *)&iWriteTimeOut);
return true;
}
void DBInstance::closeDB()
{
monitor_log_error("closing database connection");
freeResult();
// mDBConnection.sMyRes = NULL;
// mDBConnection.sMyRow = NULL;
mysql_close(&(mDBConnection.sMysqlInstance));
mDBConnection.hasConnected = false;
mDBConnection.inTransaction = false;
return;
}
bool DBInstance::connectDB()
{
if (mDBConnection.hasConnected)
{
monitor_log_error("has connected to mysql yet, check it.");
return true;
// closeDB();
}
monitor_log_error("connecting to db, dbHost:%s, user:%s, dbName:%s",
mDBConnection.sDBInfo.sDBHostName.c_str(), mDBConnection.sDBInfo.sDBUserName.c_str(),
mDBConnection.sDBInfo.sDBName.c_str());
MYSQL *pMysql = &(mDBConnection.sMysqlInstance);
if (!mysql_real_connect(pMysql, mDBConnection.sDBInfo.sDBHostName.c_str(),
mDBConnection.sDBInfo.sDBUserName.c_str(), mDBConnection.sDBInfo.sPassword.c_str(),
mDBConnection.sDBInfo.sDBName.c_str(),
mDBConnection.sDBInfo.sDBPort, NULL, 0))
{
monitor_log_error("connect to db failed:%s", mysql_error(pMysql));
return false;
}
mDBConnection.hasConnected = true;
monitor_log_error("connected to db success");
return true;
}
bool DBInstance::startTransaction()
{
if(mDBConnection.inTransaction)
{
monitor_log_error("has unfinished transaction in this process");
return false;
}
const char *sql="BEGIN;";
int ret = execSQL(sql);
if (ret < 0)
{
monitor_log_error("start transaction error");
}
else
{
mDBConnection.inTransaction = true; //a transaction start in this connection
}
return ret;
}
bool DBInstance::rollBackTransaction()
{
if (!mDBConnection.inTransaction)
{
monitor_log_error("no started transaction");
return false;
}
const char *sql="ROLLBACK;";
int ret = execSQL(sql);
if (ret < 0)
{
monitor_log_error("rollback transaction failed");
}
mDBConnection.inTransaction = false; //a transaction has been rollbacked
return true;
}
bool DBInstance::commitTransaction()
{
if (!mDBConnection.inTransaction)
{
monitor_log_error("no transaction for committed");
return false;
}
const char *sql="COMMIT;";
int ret = execSQL(sql);
if (ret < 0)
{
monitor_log_error("committed transaction failed");
}
mDBConnection.inTransaction = false; //a transaction rollback in mysql connection
return true;
}
int DBInstance::execSQL(const char *sql)
{
if (sql == NULL)
{
monitor_log_error("sql is NULL");
return -1;
}
monitor_log_error("execute sql. sql:%s", sql);
static int reConnect = 0;
RETRY:
if (!mDBConnection.hasConnected)
{
monitor_log_error("mysql db has not connected");
if (!mDBConnection.inTransaction)
{
monitor_log_error("not in transaction, try to reconnect");
if (!connectDB())
{
monitor_log_error("connect to mysql db failed");
reConnect = 0;
return -1;
}
}
else
{
monitor_log_error("has an unfinished transaction, not connect automatic");
reConnect = 0;
return -1;
}
}
MYSQL* pMysql = &(mDBConnection.sMysqlInstance);
if (mDBConnection.sDBInfo.sDBName.empty())
{
monitor_log_error("database name can not be empty");
reConnect = 0;
closeDB();
return -1;
}
if (mysql_select_db(pMysql, mDBConnection.sDBInfo.sDBName.c_str()) != 0)
{
if (mysql_errno(pMysql) == CR_SERVER_GONE_ERROR && reConnect < 10)
{
// server have closed the connection for alive time elapsed, reconnect it
monitor_log_error("retry connect, round:%d", reConnect);
// close the unclosed socket first
closeDB();
reConnect++;
goto RETRY;
}
monitor_log_error("switch to db failed:%s", mysql_error(pMysql));
reConnect = 0;
closeDB();
return -1;
}
reConnect = 0;
// mysql_query return 0 when excute success
if (mysql_query(pMysql, sql))
{
monitor_log_error("query failed:%s", mysql_error(pMysql));
if (mysql_errno(pMysql) == CR_SERVER_GONE_ERROR || mysql_errno(pMysql) == CR_SERVER_LOST)
closeDB();
return -mysql_errno(pMysql);
}
bool isSelect = false;
if (!strncasecmp(skipWhiteSpace(sql), "select", 6)) isSelect = true;
// function `mysql_store_result` store the result to local and return the pointer point to
// the it
//
// it's meaningless to call mysql_store_result when the query type is not select
if (isSelect && !(mDBConnection.sMyRes = mysql_store_result(pMysql)))
{
monitor_log_error("mysql return a NULL result set:%s", mysql_error(pMysql));
return -2;
}
return 0;
}
// return the pointer point to the value
char* DBInstance::getFieldValue(const int fieldIndex)
{
if (NULL == mDBConnection.sMyRes || NULL == mDBConnection.sMyRow)
{
monitor_log_error("empty dataset. errno:%s", mysql_error(&mDBConnection.sMysqlInstance));
return NULL;
}
int numFields = getNumFields();
if (fieldIndex < 0 || fieldIndex >= numFields)
{
monitor_log_error("field index out of boundary. totalFields:%d,\
fieldIndex:%d", numFields, fieldIndex);
return NULL;
}
return mDBConnection.sMyRow[fieldIndex];
}
// get the next row result which has been stored in the sMyRes with function
// mysql_store_result()
// Return:1. reach to the end or fetch failed, return NULL
// 2. otherwise return the result
int DBInstance::fetchRow()
{
if (!mDBConnection.hasConnected)
return -1;
if ((mDBConnection.sMyRow = mysql_fetch_row(mDBConnection.sMyRes)) != 0) return 0;
return -1;
}
// return number of rows those affected in the previous operation
// work for insert, delete, update sql command
// return:
// 1. > 0 : number of rows been touched
// 2. = 0 : previous operation do nothing
// 3. -1 : error
int DBInstance::getAffectedRows()
{
return mysql_affected_rows(&(mDBConnection.sMysqlInstance));
}
// return number of rows in the dataset
// this command is only work for select sql
// Note: if the table is empty, mysql_num_rows will return 1, not 0
int DBInstance::getNumRows()
{
if (!mDBConnection.sMyRes) {
return 0;
}
MYSQL_RES* pRes = mDBConnection.sMyRes;
return mysql_num_rows(pRes);
}
// get the field number in one record
int DBInstance::getNumFields()
{
if (!mDBConnection.sMyRes || NULL == mDBConnection.sMyRow) return 0;
MYSQL_RES* pRes = mDBConnection.sMyRes;
return mysql_num_fields(pRes);
}
/*
* get auto increment id
*/
my_ulonglong DBInstance::getInsertID()
{
return mysql_insert_id(&(mDBConnection.sMysqlInstance));
}
char* DBInstance::skipWhiteSpace(const char* sStr)
{
char *p = (char *)sStr;
while(*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') p++;
return p;
}

73
src/tools/search_monitor/src/DBInstance.h Normal file
View File

@ -0,0 +1,73 @@
#ifndef __DB_INSTANCE_H__
#define __DB_INSTANCE_H__
#include <string>
#define list_add my_list_add
// #include "mysql.h"
#include <mysql.h>
#undef list_add
typedef struct
{
std::string sDBHostName; /*DB host*/
unsigned int sDBPort;/*DB port*/
std::string sDBUserName;/*DB user*/
std::string sPassword; /*DB password*/
std::string sDBName;/*DB name*/
} DBInfo_t;
typedef struct
{
MYSQL sMysqlInstance;
MYSQL_RES* sMyRes;
MYSQL_ROW sMyRow;
// MYSQL_RES* sRes[MAX_MYSQL_RES];
// MYSQL_ROW sRows[MAX_MYSQL_RES];
DBInfo_t sDBInfo;
// int sConnectTimeOut;
// int sReadTimeOut;
// int WriteTimeOut;
bool hasConnected;
bool inTransaction;
} DBConnection_t;
class DBInstance
{
private:
DBConnection_t mDBConnection;
public:
DBInstance();
~DBInstance();
public:
bool initDB(
const std::string& dbHostName,
const int port,
const std::string& dbName,
const std::string& dbUser,
const std::string& dbPassword,
const int iConnectTimeOut,
const int iReadTimeOut,
const int iWriteTimeOut);
int execSQL(const char *sql);
int getAffectedRows();
void closeDB();
bool connectDB();
int fetchRow();
void freeResult();
my_ulonglong getInsertID();
int getNumFields();
int getNumRows();
bool startTransaction();
bool rollBackTransaction();
bool commitTransaction();
char* getFieldValue(const int fieldIndex);
private:
char* skipWhiteSpace(const char* sStr);
};
#endif // __DB_INSTANCE_H__

269
src/tools/search_monitor/src/DetectHandlerBase.cpp Normal file
View File

@ -0,0 +1,269 @@
///////////////////////////////////////////////////
//
// the foundation class for agent and DTC detector
// create by qiuyu on Dec 10, 2018
//
//////////////////////////////////////////////////
#include "DetectHandlerBase.h"
#include "DBInstance.h"
#include <fstream>
#ifdef TEST_MONITOR
// static const std::string sgDBHostName = "10.181.174.20";
// static const int sgDBPort = 3306;
// static const std::string sgDBUserName = "dtcadmin";
// static const std::string sgUserPassword = "dtcadmin@jd2015";
static const std::string sgDBHostName = "11.80.17.227";
static const int sgDBPort = 3306;
static const std::string sgDBUserName = "root";
static const std::string sgUserPassword = "root";
#else
static const std::string sgDBHostName = "my12115m.mysql.jddb.com";
static const int sgDBPort = 3358;
static const std::string sgDBUserName = "dtcadmin";
static const std::string sgUserPassword = "nbGFzcy9qb2luOztHRVQ=";
#endif
static const std::string sgDBName = "search_monitor_cluster";
extern const std::string gTableName = "distributed_monitor_cluster_lock";
static const int sgConnectTimeout = 10; // second
static const int sgReadTimeout = 5; // second
static const int sgWriteTimeout = 5; // second
// use sLockExpiredTime to delete the unexpected dead-lock,
// all timestampe base on MYSQL
extern const int sgLockExpiredTime = 10 * 60; // 10 min
DBInstance* DetectHandlerBase::mDBInstance = NULL;
std::string DetectHandlerBase::mGetPhysicalInfoUrl = "";
std::string DetectHandlerBase::mSelfAddr = "";
LirsCache* DetectHandlerBase::mPhysicalRegionCode = NULL;
int64_t DetectHandlerBase::mCacheExpiredWhen = 0;
std::bitset<DetectHandlerBase::eMaxZoneCode> DetectHandlerBase::mRegionCodeSet;
DetectHandlerBase::~DetectHandlerBase()
{
if (mDBInstance) delete mDBInstance;
if (mPhysicalRegionCode) delete mPhysicalRegionCode;
}
void DetectHandlerBase::addTimerEvent()
{
monitor_log_error("could never come here!!");
return;
}
void DetectHandlerBase::TimerNotify(void)
{
monitor_log_error("could never come here!!");
return;
}
// init static member data
bool DetectHandlerBase::initHandler()
{
if (!mDBInstance)
{
mDBInstance = new DBInstance();
if (!mDBInstance)
{
monitor_log_error("create database instance failed.");
return false;
}
bool rslt = mDBInstance->initDB(sgDBHostName, sgDBPort, sgDBName, sgDBUserName,\
sgUserPassword, sgConnectTimeout, sgReadTimeout, sgWriteTimeout);
if (!rslt) return false;
rslt = mDBInstance->connectDB();
if (!rslt) return false;
}
if (mGetPhysicalInfoUrl.empty()) mGetPhysicalInfoUrl = DtcMonitorConfigMgr::getInstance()->getPhysicalInfoUrl();
if (mGetPhysicalInfoUrl.empty())
{
monitor_log_error("invalid url.");
return false;
}
// No race condition
if (!mPhysicalRegionCode) mPhysicalRegionCode = new LirsCache(10000);
if (!mPhysicalRegionCode)
{
monitor_log_error("allocate cache failed.");
return false;
}
// register region code
if (mRegionCodeSet.none())
{
// need to register
registerRegionCode();
}
if (mSelfAddr.empty())
{
const std::pair<std::string, int>& pair= DtcMonitorConfigMgr::getInstance()->getListenAddr();
mSelfAddr = pair.first + ":" + DetectHandlerBase::toString<int>(pair.second);
}
if (mSelfAddr.empty())
{
monitor_log_error("invalid instance address!");
return false;
}
return true;
}
void DetectHandlerBase::reportAlarm(const std::string& errMessage)
{
CurlHttp curlHttp;
BuffV buff;
curlHttp.SetHttpParams("%s", errMessage.c_str());
curlHttp.SetTimeout(mDriverTimeout);
int http_ret = curlHttp.HttpRequest(mReportUrl.c_str(), &buff, false);
if(0 != http_ret)
{
monitor_log_error("report alarm http error! curlHttp.HttpRequest error ret:[%d]", http_ret);
return;
}
std::string response = ";";
response= buff.Ptr();
monitor_log_error("response from report server! ret:[%d], response:[%s]", http_ret, response.c_str());
return;
}
int DetectHandlerBase::getInstanceTimeout(
const DetectHandlerBase::DetectType type,
const std::string& detectorAddr,
const std::string& detectedNodeAddr)
{
DtcMonitorConfigMgr::TimeoutSet_t tmSet;
switch (type)
{
/* case DetectHandlerBase::eAgentDetect:
tmSet = DtcMonitorConfigMgr::getInstance()->getAgentTimeoutSet();
break; */
case DetectHandlerBase::eDtcDetect:
tmSet = DtcMonitorConfigMgr::getInstance()->getDtcTimeoutSet();
break;
default:
return -1;
}
return tmSet.sDomesticZoneTimeout;
}
void DetectHandlerBase::getNodeInfo(
const std::string& phyAddr,
std::string& zoneCode)
{
// in current now, this API is not allowed to be called
zoneCode = "";
return;
CurlHttp curlHttp;
BuffV buff;
std::stringstream body;
body << "{\"ip\":\"" << phyAddr << "\"},1,100";
// body << "{\"ip\":\"" << "10.187.155.131" << "\"},1,100";
monitor_log_info("body contents:%s", body.str().c_str());
curlHttp.SetHttpParams("%s", body.str().c_str());
curlHttp.SetTimeout(10); // 10s hard code
zoneCode = "";
int http_ret = curlHttp.HttpRequest(mGetPhysicalInfoUrl.c_str(), &buff, false);
if(0 != http_ret)
{
monitor_log_error("get physical info failed! addr:%s, errorCode:%d", phyAddr.c_str(), http_ret);
return;
}
std::string response = ";";
response= buff.Ptr();
monitor_log_info("physical info:%s", response.c_str());
Json::Reader infoReader;
Json::Value infoValue;
if(!infoReader.parse(response, infoValue))
{
monitor_log_error("parse physical info failed, addr:%s, response:%s", phyAddr.c_str(), response.c_str());
return;
}
if (!(infoValue.isMember("result") && infoValue["result"].isArray() && infoValue["result"].size() == 1))
{
monitor_log_error("invalid physical info, addr:%s, response:%s", phyAddr.c_str(), response.c_str());
return;
}
const Json::Value& instanceInfo = infoValue["result"][0];
if (!(instanceInfo.isMember("dataCenterId") && instanceInfo["dataCenterId"].isInt()))
{
monitor_log_error("invalid instance info, addr:%s, response:%s", phyAddr.c_str(), instanceInfo.toStyledString().c_str());
return;
}
// a bit duplicate conversion
long long int centerCode = instanceInfo["dataCenterId"].asInt();
if (centerCode <= 0 || centerCode >= (1LL << REGION_CODE_SHIFT))
{
monitor_log_error("illegal center code, addr:%s, centerCode:%lld", phyAddr.c_str(), centerCode);
return;
}
// random time to prevent cache avalanche
int64_t expiredTime = GET_TIMESTAMP();
expiredTime += (sCacheExpiredInterval + (sCacheExpiredInterval > 24 ? rand() % (sCacheExpiredInterval/24) : 0));
zoneCode = DetectHandlerBase::toString<long long>((expiredTime << REGION_CODE_SHIFT) + centerCode);
return;
}
// set aboard to 1 and the others to 0
void DetectHandlerBase::registerRegionCode()
{
// reset all bit to 0
mRegionCodeSet.reset();
mRegionCodeSet.set(eYNZoneCode);
mRegionCodeSet.set(eHKTHZoneCode);
monitor_log_error("Region code bitset size:%ld", mRegionCodeSet.size());
return;
}
bool DetectHandlerBase::isNodeExpired(const std::string& nodeInfo)
{
long long int va = atoll(nodeInfo.c_str());
int64_t time = GET_TIMESTAMP();
return (va >> REGION_CODE_SHIFT) <= time;
}
void DetectHandlerBase::stringSplit(
const std::string& src,
const std::string& delim,
std::vector<std::string>& dst)
{
dst.clear();
std::string::size_type start_pos = 0;
std::string::size_type end_pos = 0;
end_pos = src.find(delim);
while(std::string::npos != end_pos)
{
dst.push_back(src.substr(start_pos, end_pos - start_pos));
start_pos = end_pos + delim.size();
end_pos = src.find(delim, start_pos);
}
if(src.length() != start_pos)
{
dst.push_back(src.substr(start_pos));
}
}

199
src/tools/search_monitor/src/DetectHandlerBase.h Normal file
View File

@ -0,0 +1,199 @@
///////////////////////////////////////////////////
//
// the foundation class for agent and DTC detector
// create by qiuyu on Nov 26, 2018
//
//////////////////////////////////////////////////
#ifndef __DETECT_HANDLER_BASE_H__
#define __DETECT_HANDLER_BASE_H__
#include "DtcMonitorConfigMgr.h"
#include "DetectUtil.h"
// #include "InvokeMgr.h"
#include "curl_http.h"
#include "poll_thread.h"
#include "log.h"
#include "LirsCache.h"
#include <vector>
#include <map>
#include <set>
#include <sstream>
#include <algorithm>
#include <bitset>
#define REGION_CODE_SHIFT 10
#define REGION_CODE_MASK (~(-1LL << REGION_CODE_SHIFT))
class DBInstance;
class DetectHandlerBase : public CTimerObject
{
private:
static const int sExpiredTimeoffset = 60 * 1000 * 1000; // 60s
static const int64_t sCacheExpiredInterval = 1LL * 24 * 3600 * 1000 * 1000; // one day
protected:
static const int sEventTimerOffset = 300; // ms [-300, 300]
public:
enum DetectType
{
eAgentDetect = 1,
eDtcDetect
};
enum ReportType
{
eAgentDetectFailed = 0,
eOfflineAgentFailed = 1,
eOfflineAgentSuccess = 2,
eDtcDetectFailed,
eSwitchDtcFailed,
eSwitchDtcSuccess,
eAccessDBFailed,
eAccessCCFailed,
eSwitchSlaveDtc,
eReplicateDtcFailed,
eNoneUsefulDtc,
eParseJsonResponseFailed = 7000,
eResponseMissDefaultField,
eInvalidReportType = -1
};
enum ResponseCode
{
eROperateSuccess = 200,
eRSwitchSlaveDtc = 201
};
enum ZoneCode
{
eMinZoneCode = 0,
// demestic region
eLF1ZoneCode = 3, // LF1
eMJQZoneCode = 6, // MJQ data center
eYNZoneCode = 7, // YN
eMJQ3ZoneCode = 11, // MJQ3
eLF3ZoneCode = 12, // LF3
eHKTHZoneCode = 13, // THA
eGZ1ZoneCode = 14, // GZ1
eLF4ZoneCode = 15, // LF4
eLF5ZoneCode = 17, // LF4
eKEPLERZoneCode = 21, // kepler
eHT3ZoneCode = 28, // HT3
eMaxZoneCode
};
typedef struct InstanceInfo
{
std::string sAccessKey; // used by IP
std::string sIpWithPort;
int64_t sExpiredTimestamp;
InstanceInfo(
const std::string& key,
const std::string& addr,
const int64_t expiredWhen)
:
sAccessKey(key),
sIpWithPort(addr),
sExpiredTimestamp(expiredWhen + sExpiredTimeoffset)
{
}
bool isExpired(const int64_t nowTimestamp) const
{
return sExpiredTimestamp <= nowTimestamp;
}
bool operator < (const InstanceInfo& other) const
{
if ((sAccessKey != "" && other.sAccessKey != "")
&& (sAccessKey != other.sAccessKey))
{
return sAccessKey < other.sAccessKey;
}
return sIpWithPort < other.sIpWithPort;
}
}InstanceInfo_t;
struct ProtocalHeader {
uint32_t magic;
uint32_t length;
uint32_t cmd;
};
protected:
CPollThread* mDetectPoll;
std::string mReportUrl;
int mDriverTimeout; // ms
int mDetectStep;
int mPrevDetectIndex;
std::set<InstanceInfo_t> mTroubleInstances;
std::set<std::string> m_TroubleHandledIpSet;
// db instance
static DBInstance* mDBInstance;
// region control relevant
static std::string mGetPhysicalInfoUrl;
static std::string mSelfAddr;
static LirsCache* mPhysicalRegionCode; // mapping physicalId to region code
static std::bitset<DetectHandlerBase::eMaxZoneCode> mRegionCodeSet; // region codes
static int64_t mCacheExpiredWhen;
// statistic
std::stringstream mDetectSummary;
public:
DetectHandlerBase(CPollThread* poll)
{
mDetectPoll = poll;
mCacheExpiredWhen = GET_TIMESTAMP() + sCacheExpiredInterval;
mDetectSummary.str(""); // clear streaming
srand(time(NULL));
}
virtual ~DetectHandlerBase();
virtual void addTimerEvent();
virtual void TimerNotify(void);
void reportAlarm(const std::string& errMessage);
static int getInstanceTimeout(
const DetectHandlerBase::DetectType type,
const std::string& detectorAddr,
const std::string& nodeAddr);
template<typename T>
static std::string toString(const T& val)
{
std::stringstream ss;
ss << val;
return ss.str();
}
protected:
static int selectTimeoutWithZone(
const DetectHandlerBase::DetectType type,
const std::string& selfZoneCode,
const std::string& peerZoneCode);
static void getNodeInfo(
const std::string& phyAddr,
std::string& zoneCode);
static void stringSplit(
const std::string& src,
const std::string& delim,
std::vector<std::string>& dst);
static bool initHandler();
static void registerRegionCode();
static bool isNodeExpired(const std::string& nodeInfo);
};
#endif // __DETECT_HANDLER_BASE_H__

209
src/tools/search_monitor/src/DetectUtil.cpp Normal file
View File

@ -0,0 +1,209 @@
/////////////////////////////////////////////////////
//
// for detect Agent and Dtc instance
// create by qiuyu on Nov 27, 2018
//
/////////////////////////////////////////////////////
#include "DetectUtil.h"
#include "sockaddr.h"
#include "log.h"
#include "detector_instance.h"
#include "DtcMonitorConfigMgr.h"
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <sstream>
#include <errno.h>
bool DetectUtil::connectServer(
int& fd,
const std::string& ip,
const int port)
{
#if 0
CSocketAddress addr;
std::stringstream bindAddr;
bindAddr << ip << ":" << port << "/tcp";
addr.SetAddress(bindAddr.str().c_str(), (const char*)NULL);
if (addr.SocketFamily() != 0)
{
monitor_log_error("invalid addr.");
return false;
}
fd = addr.CreateSocket();
if (fd < 0)
{
monitor_log_error("create socket failed. errno:%d", errno);
return false;
}
fcntl(fd, F_SETFL, O_RDWR|O_NONBLOCK);
int ret = addr.ConnectSocket(fd);
if (ret != 0)
{
monitor_log_error("connect to server failed. ip:%s, portL%d, errno:%d", ip.c_str(), port, errno);
return false;
}
#else
monitor_log_info("connect to server. ip:%s, port:%d", ip.c_str(), port);
fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0)
{
monitor_log_error("create socket failed. ip:%s, port:%d", ip.c_str(), port);
return false;
}
struct sockaddr_in net_addr;
net_addr.sin_addr.s_addr = inet_addr(ip.c_str());
net_addr.sin_family = AF_INET;
net_addr.sin_port = htons(port);
// block to connect
int ret = connect(fd, (struct sockaddr *)&net_addr, sizeof(struct sockaddr));
if (ret < 0)
{
monitor_log_error("connect to server failed, fd:%d, errno:%d, ip:%s, port:%d", fd, errno, ip.c_str(), port);
close(fd);
return false;
}
// set socket to non-block
fcntl(fd, F_SETFL, O_RDWR|O_NONBLOCK);
#endif
return true;
}
int DetectUtil::recieveMessage(
const int fd,
char* data,
const int dataLen)
{
int readNum = 0;
int nRead = 0;
int nRet = 0;
do {
nRet = read(fd, data + nRead, dataLen - nRead);
if (nRet > 0)
{
nRead += nRet;
if (nRead == dataLen) return nRead;
}
else if (nRet == 0)
{
// close the connection
monitor_log_error("client close the socket, fd:%d", fd);
// return nRead;
return -1;
}
else
{
if (readNum < 1000 && (errno == EAGAIN || errno == EINTR))
{
readNum++;
continue;
}
else
{
// close the connection
monitor_log_error("client close the socket, fd:%d", fd);
// return nRead;
return -1;
}
}
}while (nRead < dataLen);
return dataLen;
}
int DetectUtil::sendMessage(
const int netfd,
char* data,
const int dataLen)
{
int sendNum = 0;
int nWrite = 0;
int nRet = 0;
do {
nRet = write(netfd, data + nWrite, dataLen - nWrite);
if (nRet > 0)
{
nWrite += nRet;
if (dataLen == nWrite) return nWrite;
}
else if (nRet < 0)
{
if (sendNum < 1000 && (errno == EINTR || errno == EAGAIN))
{
sendNum++;
continue;
}
else
{
// connection has issue, need to close the socket
monitor_log_error("write socket failed, fd:%d, errno:%d", netfd, errno);
return -1;
// return nWrite;
}
}
else
{
monitor_log_error("write socket failed, fd:%d, errno:%d", netfd, errno);
return -1;
}
}
while(nWrite < dataLen);
return dataLen;
}
bool DetectUtil::detectAgentInstance(
const std::string& accessKey,
const std::string& ipWithPort,
const int timeout,
bool& isAlive,
int& errCode)
{
return CDetectorInstance::DetectAgent(accessKey, ipWithPort, timeout, isAlive, errCode);
}
bool DetectUtil::detectDtcInstance(
const std::string& ipWithPort,
const int timeout,
bool& isAlive,
int& errCode)
{
return CDetectorInstance::DetectDTC(ipWithPort, timeout, isAlive, errCode);
}
// network endian is big endian
void DetectUtil::translateByteOrder(uint64_t &value)
{
#if __BYTE_ORDER == __BIG_ENDIAN
// do noting, network endian is big endian
#elif __BYTE_ORDER == __LITTLE_ENDIAN
// translate to little endian
unsigned char *val, temp;
val = (unsigned char*)&value;
temp = val[0];
val[0] = val[7];
val[7] = temp;
temp = val[1];
val[1] = val[6];
val[6] = temp;
temp = val[2];
val[2] = val[5];
val[5] = temp;
temp = val[3];
val[3] = val[4];
val[4] = temp;
#else
#error unkown endian
#endif
return;
}

48
src/tools/search_monitor/src/DetectUtil.h Normal file
View File

@ -0,0 +1,48 @@
/////////////////////////////////////////////////////
//
// for detect Agent and Dtc instance
// create by qiuyu on Nov 27, 2018
//
/////////////////////////////////////////////////////
#ifndef __DETECT_UTIL_H__
#define __DETECT_UTIL_H__
#include <string>
#include <stdint.h>
class DetectUtil
{
public:
static bool connectServer(
int& fd,
const std::string& ip,
const int port);
static int recieveMessage(
const int fd,
char* data,
const int dataLen);
static int sendMessage(
const int fd,
char* data,
const int dataLen);
static bool detectAgentInstance(
const std::string& accessKey,
const std::string& ipWithPort,
const int timeout,
bool& isAlive,
int& errCode);
static bool detectDtcInstance(
const std::string& ipWithPort,
const int timeout,
bool& isAlive,
int& errCode);
static void translateByteOrder(uint64_t& value);
};
#endif // __DETECT_UTIL_H__

1050
src/tools/search_monitor/src/DtcDetectHandler.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

107
src/tools/search_monitor/src/DtcDetectHandler.h Normal file
View File

@ -0,0 +1,107 @@
///////////////////////////////////////////////////
//
// the dtc detector
// create by qiuyu on Nov 27, 2018
// Modify: chenyujie
//////////////////////////////////////////////////
#ifndef __DTC_DETECT_HANDLER_H__
#define __DTC_DETECT_HANDLER_H__
#include <vector>
#include <string>
#include "DetectHandlerBase.h"
#include "config_center_parser.h"
#define DISTRIBUTE_LOCK_SOLE_IP "127.0.0.1"
#define DISTRIBUTE_LOCK_SOLE_PORT "8888"
class DtcDetectHandler : public DetectHandlerBase
{
enum RetCode {
RT_INIT_ERR = 10001,
RT_PARSE_CONF_ERR,
RT_PARSE_JSON_ERR,
RT_PRE_RUN_ERR,
RT_DB_ERR,
RT_HTTP_ERR,
};
private:
static int64_t adminInfoIdx;
private:
DtcClusterContextType mDtcList;
SearchCoreClusterContextType mDtcMap;
int mPrevDetectedIndex;
DtcClusterContextType mCheckDtcList;
ParserBase* m_pCurrentParser;
bool m_bReportEnable;
public:
DtcDetectHandler(CPollThread* poll);
~DtcDetectHandler();
virtual void addTimerEvent();
virtual void TimerNotify(void);
private:
void initHandler();
bool batchDetect(
const int startIndex,
const int endIndex);
bool procSingleDetect(
const std::string& addr,
bool& isAlive,
int& errCode);
bool broadCastConfirm(
const std::string& addr,
bool& needSwitch);
bool reportDtcAlarm(
const DetectHandlerBase::ReportType type,
const std::string& addr,
const int errCode);
bool doDtcSwitch(
const std::string& sIP,
const std::string& addr,
const int errCode);
void verifyAllExpired(const int line);
int verifySingleExpired(const std::string& sIP, const std::string& addr);
bool loadDtcClusterInfo();
bool loadDtcClusterInfoNoOpr();
bool loadSearchCoreClusterInfo();
int Connect(const std::string& ip, const uint32_t port);
char* MakeAdminPackage(const std::string& shardingName,
const std::string& ip,
const std::string& port,
int& msgsize);
int NotifyRouteAdmin(const std::string& ip,
const uint32_t port,
const std::string& sharding,
const std::string& localip,
const std::string& localport);
int NotifyConfigCenter();
bool CheckIndexGenIsMaster(const std::string& sIP);
int getDistributedLockForConsistency(
const std::string& ip,
const std::string& port);
void releaseDistributedLock(
const std::string& ip,
const std::string& port);
};
#endif // __DTC_DETECT_HANDLER_H__

109
src/tools/search_monitor/src/DtcMonitor.cpp Normal file
View File

@ -0,0 +1,109 @@
/////////////////////////////////////////////////////////
//
// This class detect the DTC instance for a internal time
// created by qiuyu on Nov 26, 2018
////////////////////////////////////////////////////////
#include "DtcMonitor.h"
#include "poll_thread.h"
#include "MonitorVoteListener.h"
#include "DtcDetectHandler.h"
// DtcMonitor* DtcMonitor::mSelf = NULL;
// CMutex* DtcMonitor::mMutexLock = new CMutex();
DtcMonitor::DtcMonitor()
:
mDetectPoll(new CPollThread("DtcDetectPoll")),
mListener(NULL),
mDtcDetector(NULL)
{
}
DtcMonitor::~DtcMonitor()
{
// if (mMutexLock)
// delete mMutexLock;
if (mDetectPoll)
delete mDetectPoll;
if (mListener)
delete mListener;
if (mDtcDetector)
delete mDtcDetector;
}
bool DtcMonitor::initMonitor()
{
// add poller to poller thread
if (!mDetectPoll)
{
monitor_log_error("create thread poll failed.");
return false;
}
int ret = mDetectPoll->InitializeThread();
if (ret < 0)
{
monitor_log_error("initialize thread poll failed.");
return false;
}
// add listener to poll
bool rslt = addListenerToPoll();
if (!rslt) return false;
rslt = addTimerEventToPoll();
if (!rslt) return false;
return true;
}
bool DtcMonitor::startMonitor()
{
bool rslt = initMonitor();
if (!rslt) return false;
mDetectPoll->RunningThread();
monitor_log_info("start DtcMonitor successful.");
return true;
}
bool DtcMonitor::addListenerToPoll()
{
mListener = new MonitorVoteListener(mDetectPoll);
if (!mListener)
{
monitor_log_error("create listener instance failed");
return false;
}
int ret = mListener->Bind();
if (ret < 0)
{
monitor_log_error("bind address failed.");
return false;
}
ret = mListener->attachThread();
if (ret < 0)
{
monitor_log_error("add listener to poll failed.");
return false;
}
return true;
}
// add detect agent, detect dtc event to the poll
bool DtcMonitor::addTimerEventToPoll()
{
mDtcDetector = new DtcDetectHandler(mDetectPoll);
if (!mDtcDetector)
{
monitor_log_error("create dtc detector failed.");
return false;
}
mDtcDetector->addTimerEvent();
return true;
}

48
src/tools/search_monitor/src/DtcMonitor.h Normal file
View File

@ -0,0 +1,48 @@
/////////////////////////////////////////////////////////
//
// This class detect the DTC instance for a internal time
// created by qiuyu on Nov 26, 2018
////////////////////////////////////////////////////////
#ifndef __DTC_MONITOR_H__
#define __DTC_MONITOR_H__
#include "singleton.h"
class DetectHandlerBase;
class CPollThread;
class MonitorVoteListener;
class DtcMonitor
{
private:
// static DtcMonitor* mSelf;
// static CMutex* mMutexLock;
CPollThread* mDetectPoll;
MonitorVoteListener* mListener;
DetectHandlerBase* mDtcDetector;
public:
DtcMonitor();
virtual ~DtcMonitor();
static DtcMonitor* getInstance()
{
return CSingleton<DtcMonitor>::Instance();
}
// private:
// DtcMonitor();
// DtcMonitor(const DtcMonitor&);
// DtcMonitor& operator=(const DtcMonitor&);
public:
bool startMonitor();
void stopMonitor();
private:
bool initMonitor();
bool addListenerToPoll();
bool addTimerEventToPoll();
};
#endif // __DTC_MONITOR_H__

581
src/tools/search_monitor/src/DtcMonitorConfigMgr.cpp Normal file
View File

@ -0,0 +1,581 @@
/////////////////////////////////////////////////////////////////
//
// Handle All human configurable config
// created by qiuyu on Nov 26, 2018
////////////////////////////////////////////////////////////////
#include "DtcMonitorConfigMgr.h"
#include "log.h"
#include <sstream>
#include <unistd.h>
#include <fstream>
#include <algorithm>
#include <errno.h>
// variable for gdb debug
int sgPhysicalId = 0;
bool DtcMonitorConfigMgr::init(const std::string& path)
{
if (mHasInit)
{
monitor_log_info("has init this config yet.");
return true;
}
bool rslt = false;
Json::Reader reader;
Json::Value jsonStyleValue;
std::ifstream conf(path.c_str());
if (!conf)
{
monitor_log_error("open config file failed. fileName:%s", path.c_str());
return false;
}
rslt = reader.parse(conf, jsonStyleValue);
if (!rslt)
{
monitor_log_error("parse config to json failed!");
return false;
}
rslt = parseLogLevel(jsonStyleValue);
if (!rslt) return false;
rslt = parseLogFilePath(jsonStyleValue);
if (!rslt) return false;
rslt = parseGlobalPhysicalId(jsonStyleValue);
if (!rslt) return false;
rslt = parseInvokeTimeout(jsonStyleValue);
if (!rslt) return false;
// parse listen addr
rslt = parseListenAddr(jsonStyleValue);
if (!rslt) return false;
// parse cluster host address
rslt = parseClusterHostsInfo(jsonStyleValue);
if (!rslt) return false;
rslt = parseAdminClusterInfo(jsonStyleValue);
if (!rslt) return false;
rslt = parseDtcConf(jsonStyleValue);
if (!rslt) return false;
rslt = parseReportAlarmUrl(jsonStyleValue);
if (!rslt) return false;
rslt = parseAlarmReceivers(jsonStyleValue);
if (!rslt) return false;
rslt = parsePhysicalInfoUrl(jsonStyleValue);
if (!rslt) return false;
rslt = parseConfigCenterContext(jsonStyleValue);
if (!rslt) return false;
mHasInit = true;
monitor_log_info("load customer config successful.");
return true;
}
bool DtcMonitorConfigMgr::parseLogLevel(const Json::Value& jsonValue)
{
// Json::Value dtc_config;
if (!jsonValue.isMember("logLevel") || !jsonValue["logLevel"].isInt())
{
monitor_log_error("incorrect field in config.");
return false;
}
mConfs.sLogLevel = jsonValue["logLevel"].asInt();
return true;
}
bool DtcMonitorConfigMgr::parseLogFilePath(const Json::Value& jsonValue)
{
// Json::Value dtc_config;
if (!jsonValue.isMember("logFilePath") || !jsonValue["logFilePath"].isString())
{
monitor_log_error("incorrect field in config.");
return false;
}
mConfs.sLogFilePath = jsonValue["logFilePath"].asString();
return true;
}
bool DtcMonitorConfigMgr::parseGlobalPhysicalId(const Json::Value& jsonValue)
{
// 1.Because of the cluster was started up by website, so user configurable field
// will not be allowed, remove this field from the config
// 2.hard code the physical id will cause no bad influence, it only be used for
// distinguishing the sequenceId created by which physical, use kernel thread id
// to replace it
sgPhysicalId = syscall(__NR_gettid);
mConfs.sGlobalPhysicalId = sgPhysicalId % (0x7F); // one byte for physical id
monitor_log_crit("Only print it out for debugging, physicalId:%d", sgPhysicalId);
return true;
if (!jsonValue.isMember("physicalId") || !jsonValue["physicalId"].isInt())
{
monitor_log_error("incorrect field in config.");
return false;
}
int phyId = jsonValue["physicalId"].asInt();
if (phyId < 0)
{
monitor_log_error("physical id can not be negative, physicalId:%d", phyId);
return false;
}
mConfs.sGlobalPhysicalId = phyId;
return true;
}
// Due to distinguish the the physical zone for diffrent detecting timeout, the invoke
// paramter sames to be useless, we should evaluate it dynamically
bool DtcMonitorConfigMgr::parseInvokeTimeout(const Json::Value& jsonValue)
{
return true;
if (!jsonValue.isMember("invokeTimeout") || !jsonValue["invokeTimeout"].isInt())
{
monitor_log_error("incorrect field in config.");
return false;
}
int timeout = jsonValue["invokeTimeout"].asInt();
if (timeout < 0)
{
monitor_log_error("invokeTimeout can not be negative, invokeTimeout:%d", timeout);
return false;
}
mConfs.sInvokeTimeout = timeout;
return true;
}
bool DtcMonitorConfigMgr::parseListenAddr(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("listenAddr") || !jsonValue["listenAddr"].isObject())
{
monitor_log_error("incorrect field in config.");
return false;
}
const Json::Value& listenAddr = jsonValue["listenAddr"];
if (!listenAddr.isMember("ip") || !listenAddr["ip"].isString())
{
monitor_log_error("parse listenAddr ip failed.");
return false;
}
std::string ip = listenAddr["ip"].asString();
if (ip.empty())
{
monitor_log_error("ip can not be empty.");
return false;
}
trimBlank(ip);
if (ip == "*")
{
std::vector<std::string> localIps;
bool rslt = getLocalIps(localIps);
if (!rslt) return false;
ip = localIps[0];
}
if (!listenAddr.isMember("port") || !listenAddr["port"].isInt())
{
monitor_log_error("parse listenAddr port failed.");
return false;
}
int port = listenAddr["port"].asInt();
if (port <= 0)
{
monitor_log_error("parse port failed. port:%d", port);
return false;
}
mConfs.sListenAddr = std::make_pair(ip, port);
return true;
}
bool DtcMonitorConfigMgr::parseClusterHostsInfo(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("clusterHosts") || !jsonValue["clusterHosts"].isArray())
{
monitor_log_error("incorrect field in config.");
return false;
}
std::vector<std::string> localIps;
bool rslt = getLocalIps(localIps);
if (!rslt) return false;
std::string ip;
int port;
const Json::Value& clusterInfo = jsonValue["clusterHosts"];
for (int idx = 0; idx < (int)clusterInfo.size(); idx++)
{
ip = "";
port = -1;
const Json::Value& host = clusterInfo[idx];
if (!host.isMember("ip") || !host["ip"].isString())
{
monitor_log_error("parse host ip failed.");
return false;
}
ip = host["ip"].asString();
if (ip.empty())
{
monitor_log_error("ip can not be empty.");
return false;
}
// filter the local ip from the cluster config
if (std::find(localIps.begin(), localIps.end(), ip) != localIps.end())
{
monitor_log_info("filter local ip:%s", ip.c_str());
continue;
}
if (!host.isMember("port") || !host["port"].isInt())
{
monitor_log_error("parse host port failed.");
return false;
}
port = host["port"].asInt();
if (port <= 0)
{
monitor_log_error("parse port failed. port:%d", port);
return false;
}
mConfs.sClusterInfo.push_back(std::make_pair(ip, port));
}
return true;
}
bool DtcMonitorConfigMgr::parseAdminClusterInfo(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("adminHosts") || !jsonValue["adminHosts"].isArray())
{
monitor_log_error("incorrect field in config.");
return false;
}
const Json::Value& adminHosts = jsonValue["adminHosts"];
for (int idx = 0; idx < (int)adminHosts.size(); idx++)
{
const Json::Value& host = adminHosts[idx];
if (!host.isMember("ip") || !host["ip"].isString())
{
monitor_log_error("parse admin ip failed.");
return false;
}
std::string ip = host["ip"].asString();
if (!host.isMember("port") || !host["port"].isInt())
{
monitor_log_error("parse admin port failed.");
return false;
}
int port = host["port"].asInt();
if (port <= 0)
{
monitor_log_error("parse admin port failed. port:%d", port);
return false;
}
mConfs.sAdminInfo.push_back(std::make_pair(ip, port));
}
if (mConfs.sAdminInfo.size() == 0) {
monitor_log_error("admin info empty.");
return false;
}
return true;
}
bool DtcMonitorConfigMgr::parseDtcConf(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("dtcInfo") || !jsonValue["dtcInfo"].isObject())
{
monitor_log_error("incorrect field in config.");
return false;
}
const Json::Value& dtcInfo = jsonValue["dtcInfo"];
const Json::Value& timeoutSet = dtcInfo["detectTimeoutSet"];
if (!(timeoutSet.isMember("sameZoneTimeout") && timeoutSet["sameZoneTimeout"].isInt())
|| !(timeoutSet.isMember("domesticZoneTimeout") && timeoutSet["domesticZoneTimeout"].isInt())
|| !(timeoutSet.isMember("abroadZoneTimeout") && timeoutSet["abroadZoneTimeout"].isInt()))
{
monitor_log_error("timeoutSet configuration error, check it!");
return false;
}
int& ss = mConfs.sDtcConf.sTimeoutSet.sSameZoneTimeout = timeoutSet["sameZoneTimeout"].asInt();
int& sd = mConfs.sDtcConf.sTimeoutSet.sDomesticZoneTimeout= timeoutSet["domesticZoneTimeout"].asInt();
int& sa = mConfs.sDtcConf.sTimeoutSet.sAbroadZoneTimeout = timeoutSet["abroadZoneTimeout"].asInt();
if (!mConfs.sDtcConf.sTimeoutSet.isValid())
{
monitor_log_error("confiuration error, check it!");
return false;
}
// for risk controlling
ss = ss >= eDtcDefaultTimeout ? ss : eDtcDefaultTimeout;
sd = sd >= eDtcDefaultTimeout ? sd : eDtcDefaultTimeout;
sa = sa >= eDtcDefaultTimeout ? sa : eDtcDefaultTimeout;
// event dirver timeout
if (!(dtcInfo.isMember("detectPeriod") && dtcInfo["detectPeriod"].isInt()))
{
monitor_log_error("configuration error, check it!");
return false;
}
int timeout = dtcInfo["detectPeriod"].asInt();
mConfs.sDtcConf.sEventDriverTimeout = timeout > 0 ? timeout : eDtcDefaultTimeout;
int step;
if (!dtcInfo.isMember("detectStep") || !dtcInfo["detectStep"].isInt())
{
monitor_log_info("maybe missing dtc field.");
// return false;
step = -1;
}
else
{
step = dtcInfo["detectStep"].asInt();
}
step = step > 0 ? step : eDtcDefaultStep;
mConfs.sDtcConf.sDetectStep = step;
return true;
}
bool DtcMonitorConfigMgr::parseReportAlarmUrl(const Json::Value& jsonValue)
{
// Json::Value dtc_config;
if (!jsonValue.isMember("reportAlarmUrl") || !jsonValue["reportAlarmUrl"].isString())
{
monitor_log_error("incorrect field in config.");
return false;
}
std::string reportAlarmUrl = jsonValue["reportAlarmUrl"].asString();
if (reportAlarmUrl.empty())
{
monitor_log_error("reportAlarmUrl can not be empty.");
return false;
}
mConfs.sReportAlarmUrl= reportAlarmUrl;
return true;
}
bool DtcMonitorConfigMgr::parseAlarmReceivers(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("alarmReceivers") || !jsonValue["alarmReceivers"].isArray())
{
monitor_log_error("incorrect field in config.");
return false;
}
std::string reci;
std::string recList = "";
const Json::Value& recieves = jsonValue["alarmReceivers"];
for (int idx = 0; idx < (int)recieves.size(); idx++)
{
if (!recieves[idx].isString())
{
monitor_log_error("parse alarmReceivers failed.");
return false;
}
reci = recieves[idx].asString();
if (reci.empty())
{
monitor_log_error("reciever can not be empty.");
continue;
}
recList.append(reci);
if (idx != (int)recieves.size() - 1)
recList.append(";");
}
if (recList.empty())
{
monitor_log_error("reciever list can not be empty.");
return false;
}
mConfs.sAlarmReceivers = recList;
return true;
}
bool DtcMonitorConfigMgr::parsePhysicalInfoUrl(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("getPhysicalInfoUrl") || !jsonValue["getPhysicalInfoUrl"].isString())
{
monitor_log_error("incorrect field in config.");
return false;
}
std::string physicalInfoUrl = jsonValue["getPhysicalInfoUrl"].asString();
if (physicalInfoUrl.empty())
{
monitor_log_error("reportAlarmUrl can not be empty.");
return false;
}
mConfs.sGetPhysicalInfoUrl = physicalInfoUrl;
return true;
}
bool DtcMonitorConfigMgr::parseConfigCenterContext(const Json::Value& jsonValue)
{
if (!jsonValue.isMember("Config") || !jsonValue["Config"].isObject())
{
monitor_log_error("incorrect centerConfig info in config.");
return false;
}
const Json::Value& oConfigContext = jsonValue["Config"];
if (!oConfigContext.isMember("CaDir") || !oConfigContext["CaDir"].isString())
{
monitor_log_error("parse Config CaDir failed.");
return false;
}
std::string sCaDir = oConfigContext["CaDir"].asString();
if (sCaDir.empty())
{
monitor_log_error("CaDir can not be empty.");
return false;
}
mConfs.oConfigCenterContext.sCaDirPath = sCaDir;
if (!oConfigContext.isMember("CaPid") || !oConfigContext["CaPid"].isInt())
{
monitor_log_error("parse Config CaPid failed.");
return false;
}
int iCaPid = oConfigContext["CaPid"].asInt();
if (iCaPid <= 0)
{
monitor_log_error("parse Config CaPid failed, caPid:%d", iCaPid);
return false;
}
mConfs.oConfigCenterContext.iCaPid = iCaPid;
std::stringstream sTemp;
sTemp << iCaPid;
mConfs.oConfigCenterContext.sValidDir = sCaDir + sTemp.str();
return true;
}
bool DtcMonitorConfigMgr::getLocalIps(std::vector<std::string>& localIps)
{
#ifdef TEST_MONITOR
return true;
#else
localIps.clear();
// this way to get host ip has some issue is the hostname is "localdomain"
// char hname[128];
// struct hostent *hent;
//
// gethostname(hname, sizeof(hname));
// monitor_log_info("local hostname:%s", hname);
//
// hent = gethostbyname(hname);
// for(int idx = 0; hent->h_addr_list[idx]; idx++)
// {
// std::string ip(inet_ntoa(*(struct in_addr*)(hent->h_addr_list[idx])));
// monitor_log_info("local host ip:%s", ip.c_str());
//
// if (ip.empty()) continue;
// localIps.push_back(ip);
// }
//
// if (localIps.size() <= 0)
// {
// monitor_log_error("get local host ip failed, need to check it.");
// return false;
// }
// get hostname from shell
const char* shell = "ifconfig | grep inet | grep -v inet6 |\
grep -v 127 | awk '{print $2}' | awk -F \":\" '{print $2}'";
FILE *fp;
if ((fp = popen(shell, "r")) == NULL)
{
monitor_log_info("open the shell command failed.");
return false;
}
// maybe has multiple network card
char buf[256];
while (fgets(buf, 255, fp) != NULL)
{
monitor_log_info("local ip:%s", buf);
// the main function has ignored the signal SIGCHLD, so the recycle comand
// SIGCHLD sent by child process of popen will be ignored, then pclose will
// return -1 and set errno to ECHILD
if (pclose(fp) == -1 && errno != ECHILD)
{
monitor_log_info("close the file descriptor failed. errno:%d", errno);
return false;
}
// remove the character '\n' from the tail of the buf because the system call
// fgets will get it
std::string ip(buf);
if (ip[ip.length() - 1] != '\n')
{
monitor_log_error("syntax error for fgets, ip:%s", ip.c_str());
return false;
}
ip.erase(ip.begin() + (ip.length() - 1));
monitor_log_info("local ip:%s", ip.c_str());
localIps.push_back(ip);
}
if (localIps.empty())
{
monitor_log_error("get local ips failed.");
return false;
}
return true;
#endif
}
void DtcMonitorConfigMgr::trimBlank(std::string& src)
{
for (std::string::iterator itr = src.begin(); itr != src.end();)
{
if (*itr == ' ' || *itr == '\t' || *itr == '\r' || *itr == '\n')
itr = src.erase(itr);
else
itr++;
}
}

164
src/tools/search_monitor/src/DtcMonitorConfigMgr.h Normal file
View File

@ -0,0 +1,164 @@
/////////////////////////////////////////////////////////////////
//
// Handle All human configurable config
// created by qiuyu on Nov 26, 2018
////////////////////////////////////////////////////////////////
#ifndef __DTC_MONITOR_CONFIG_MGR__
#define __DTC_MONITOR_CONFIG_MGR__
#include "singleton.h"
#include "json/json.h"
#include <vector>
#include <utility>
#include <string>
#include <stdint.h>
//TEST_MONITOR参数用来区分测试(open) / 预发环境与正式环境的mysql信息(close)
//#define TEST_MONITOR
#ifndef monitor_log_error
#define monitor_log_error(format, args...) \
log_error("<%ld>" format, pthread_self(),##args)
#endif
#ifndef monitor_log_info
#define monitor_log_info(format, args...) \
log_info("<%ld>" format, pthread_self(),##args)
#endif
#ifndef monitor_log_crit
#define monitor_log_crit(format, args...) \
log_crit("<%ld>" format, pthread_self(),##args)
#endif
#ifndef monitor_log_warning
#define monitor_log_warning(format, args...) \
log_warning("<%ld>" format, pthread_self(),##args)
#endif
#ifndef monitor_log_debug
#define monitor_log_debug(format, args...) \
log_debug("<%ld>" format, pthread_self(),##args)
#endif
class DtcMonitorConfigMgr
{
public:
typedef std::pair<std::string, int> Pair_t;
typedef std::vector<std::pair<std::string, int> > PairVector_t;
enum
{
eAgentDefaultTimeout = 2000, //ms
eAgentDefaultStep = 50,
eDtcDefaultTimeout = 2000, // ms
eDtcDefaultStep = 70
};
typedef struct TimeoutSet
{
int sSameZoneTimeout; // ms
int sDomesticZoneTimeout;
int sAbroadZoneTimeout;
bool isValid()
{
return sSameZoneTimeout > 0 && sDomesticZoneTimeout > 0 && sAbroadZoneTimeout > 0;
}
}TimeoutSet_t;
private:
typedef struct DtcConfig
{
TimeoutSet_t sTimeoutSet; // ms
int sEventDriverTimeout; // ms
int sDetectStep;
}DtcConf_t;
typedef struct ConfigCenter
{
std::string sCaDirPath;
int iCaPid;
std::string sValidDir;
}ConfigCenterContext;
typedef struct ConfigList
{
int sLogLevel;
std::string sLogFilePath;
int sGlobalPhysicalId; // for creating global sequence id
int sInvokeTimeout;
std::pair<std::string, int> sListenAddr;
PairVector_t sAdminInfo;
PairVector_t sClusterInfo;
DtcConf_t sDtcConf;
std::string sReportAlarmUrl;
std::string sAlarmReceivers;
std::string sGetPhysicalInfoUrl;
ConfigCenterContext oConfigCenterContext;
}ConfList_t;
bool mHasInit;
ConfList_t mConfs;
public:
DtcMonitorConfigMgr() { mHasInit = false;}
static DtcMonitorConfigMgr* getInstance()
{
return CSingleton<DtcMonitorConfigMgr>::Instance();
}
static void Destroy()
{
CSingleton<DtcMonitorConfigMgr>::Destroy();
}
bool init(const std::string& path);
inline int getLogLevel() { return mConfs.sLogLevel; }
inline const std::string& getLogFilePath() { return mConfs.sLogFilePath; }
inline const std::pair<std::string, int>& getListenAddr() { return mConfs.sListenAddr; }
inline int getDtcDriverTimeout() { return mConfs.sDtcConf.sEventDriverTimeout; }
inline int getDtcDetectStep() { return mConfs.sDtcConf.sDetectStep; }
inline const TimeoutSet_t getDtcTimeoutSet() { return mConfs.sDtcConf.sTimeoutSet; }
const PairVector_t& getAdminInfo() { return mConfs.sAdminInfo; }
const PairVector_t& getClusterInfo() { return mConfs.sClusterInfo; }
inline const std::string& getReportAlarmUrl() {return mConfs.sReportAlarmUrl; }
inline const std::string& getReceiverList() { return mConfs.sAlarmReceivers; }
inline int getInvokeTimeout() { return mConfs.sInvokeTimeout; }
inline int getPhysicalId() { return mConfs.sGlobalPhysicalId; }
inline std::string getPhysicalInfoUrl() { return mConfs.sGetPhysicalInfoUrl; }
inline const std::string& getCaDirPath() const { return mConfs.oConfigCenterContext.sCaDirPath;}
inline int getCaPid() { return mConfs.oConfigCenterContext.iCaPid;}
inline const std::string& getValidDir() const { return mConfs.oConfigCenterContext.sValidDir;}
private:
bool parseLogLevel(const Json::Value& jsonValue);
bool parseLogFilePath(const Json::Value& jsonValue);
bool parseGlobalPhysicalId(const Json::Value& jsonValue);
bool parseInvokeTimeout(const Json::Value& jsonValue);
bool parseListenAddr(const Json::Value& jsonValue);
bool parseClusterHostsInfo(const Json::Value& jsonValue);
bool parseAdminClusterInfo(const Json::Value& jsonValue);
bool parseAgentConf(const Json::Value& jsonValue);
bool parseDtcConf(const Json::Value& jsonValue);
bool parseReportAlarmUrl(const Json::Value& jsonValue);
bool parseAlarmReceivers(const Json::Value& jsonValue);
bool parsePhysicalInfoUrl(const Json::Value& jsonValue);
bool parseConfigCenterContext(const Json::Value& jsonValue);
bool getLocalIps(std::vector<std::string>& localIps);
void trimBlank(std::string& src);
};
#endif // __DTC_MONITOR_CONFIG_MGR__

233
src/tools/search_monitor/src/InvokeHandler.cpp Normal file
View File

@ -0,0 +1,233 @@
////////////////////////////////////////////////////////////
//
// invoke client for invoking the peer monitor node
// created by qiuyu on Nov 27, 2018
//
///////////////////////////////////////////////////////////
#include "InvokeHandler.h"
#include "log.h"
#include "DetectUtil.h"
#include "MonitorVoteHandler.h"
#include "InvokeMgr.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
InvokeHandler::InvokeHandler(
CPollThread* poll,
const std::string& ip,
const int port)
:
CPollerObject(poll, 0),
mIp(ip),
mPort(port)
{
}
InvokeHandler::~InvokeHandler()
{
}
bool InvokeHandler::initHandler(const bool isInit)
{
connectToServer();
if (isInit && netfd <= 0) return true;
if (!isInit && netfd <= 0) return false;
return attachThread();
}
bool InvokeHandler::connectToServer()
{
bool rslt = DetectUtil::connectServer(netfd, mIp, mPort);
if (!rslt)
{
// return false;
// if connect failed. may be the server not startup, reconnect it when
// the next time send vote
netfd = -1;
monitor_log_error("connect to the server failed.");
}
else
{
monitor_log_error("connect to server successful, ip:%s, port:%d, netfd:%d", mIp.c_str(), mPort, netfd);
}
return true;
}
bool InvokeHandler::attachThread()
{
if (netfd <= 0)
{
monitor_log_error("invalid socket fd.");
return false;
}
EnableInput();
int ret = CPollerObject::AttachPoller();
if (ret < 0)
{
monitor_log_error("add event to poll failed.");
return false;
}
monitor_log_error("add invoke handler event to the poll successful. netfd:%d", netfd);
return true;
}
/* Notice:
* 1.in current now, not dealing with the sticky package!!!!!!!!
* 2.timeout need to be return with the bigger one between timeout and peer timeout
*/
bool InvokeHandler::invokeVote(
const DetectHandlerBase::DetectType& type,
const std::string& detectedAddr,
const uint64_t sequenceId,
const std::string& data,
int& timeout)
{
if (data.length() <= 0)
{
monitor_log_error("invoke failed. %s:%d, sequenceId:%" PRIu64, mIp.c_str(), mPort, sequenceId);
return false;
}
monitor_log_error("invoke to %s:%d, timeout:%d, sequenceId:%" PRIu64, mIp.c_str(), mPort, timeout, sequenceId);
if (netfd <= 0)
{
// reconnect the socket
bool rslt = initHandler(false);
if (!rslt)
{
monitor_log_error("invoke vote failed. ip:%s, port:%d", mIp.c_str(), mPort);
return false;
}
// bool rslt = DetectUtil::connectServer(netfd, mIp, mPort);
// if (!rslt || netfd <= 0)
// {
// monitor_log_error("invoke vote failed. netfd:%d", netfd);
// return false;
// }
}
// should set socket to block, function call maybe has some issue
char* request = (char*)malloc(sizeof(MonitorVoteHandler::VoteRequest_t) + data.length());
MonitorVoteHandler::VoteRequest_t* pData = (MonitorVoteHandler::VoteRequest_t*)request;
pData->sMagicNum = htons(MonitorVoteHandler::sgMagicNum);
pData->sSequenceId = sequenceId;
DetectUtil::translateByteOrder(pData->sSequenceId);
// calculate the peer timeout value and pick the bigger one
static std::string peerAddr = mIp + DetectHandlerBase::toString<long long int>(mPort); // only c++11 support
int peerTimeout = DetectHandlerBase::getInstanceTimeout(type, peerAddr, detectedAddr);
timeout = peerTimeout > timeout ? peerTimeout : timeout;
pData->sTimeout = htonl(timeout);
pData->sDetectType = (DetectHandlerBase::DetectType)htonl((uint32_t)type);
pData->sDataLen = htons(data.length());
memcpy(pData->sDataBuff, data.c_str(), data.length());
int dataLen = sizeof(MonitorVoteHandler::VoteRequest_t) + data.length();
int ret = DetectUtil::sendMessage(netfd, request, dataLen);
delete request;
if (ret != dataLen)
{
monitor_log_error("invoke vote failed. netfd:%d", netfd);
CPollerObject::DetachPoller();
close(netfd);
netfd = -1;
return false;
}
monitor_log_error("send data successful, fd:%d, sequenceId:%" PRIu64, netfd, sequenceId);
return true;
}
// handle voted response from peer
void InvokeHandler::InputNotify()
{
#if 0
uint16_t magic;
int len = DetectUtil::recieveMessage(netfd, (char*)&magic, sizeof(uint16_t));
if (0 == len)
{
monitor_log_error("client close the fd:%d.", netfd);
CPollerObject::DetachPoller();
// MonitorVoteHandlerMgr::getInstance()->removeHandler(this);
netfd = -1;
callBack(0, false);
return;
}
if (magic != MonitorVoteHandler::sgMagicNum)
{
monitor_log_error("receive message failed.");
callBack(0, false);
return;
}
uint64_t sequenceId;
len = DetectUtil::recieveMessage(netfd, (char*)&sequenceId, sizeof(uint64_t));
if (len != sizeof(uint64_t))
{
monitor_log_error("revieve message failed, fieldLen:%d", len);
callBack(0, false);
return;
}
// result
bool isVote = false;
len = DetectUtil::recieveMessage(netfd, (char*)&isVote, sizeof(bool));
if (len != sizeof(bool))
{
monitor_log_info("recieve message failed. sequenceId:%" PRIu64 , sequenceId);
callBack(0, false);
return;
}
monitor_log_info("call back to caller. sequenceId:%" PRIu64 , sequenceId);
callBack(sequenceId, isVote);
#else
MonitorVoteHandler::VoteResponse_t response;
int len = DetectUtil::recieveMessage(netfd, (char*)&response, sizeof(response));
if (len != sizeof(response))
{
monitor_log_error("client close or read socket failed, fd:%d, len:%d.", netfd, len);
CPollerObject::DetachPoller();
close(netfd);
netfd = -1;
// callBack(0, false);
return;
}
// check magic num
if (ntohs(response.sMagicNum) != MonitorVoteHandler::sgMagicNum)
{
monitor_log_error("receive message failed.");
// callBack(0, false);
return;
}
DetectUtil::translateByteOrder(response.sSequenceId);
monitor_log_error("call back to caller. sequenceId:%" PRIu64 , response.sSequenceId);
callBack(response.sSequenceId, response.sIsVote);
#endif
return;
}
void InvokeHandler::callBack(
const uint64_t sequenceId,
const bool isVote)
{
InvokeMgr::getInstance()->callBack(sequenceId, isVote);
return;
}

52
src/tools/search_monitor/src/InvokeHandler.h Normal file
View File

@ -0,0 +1,52 @@
////////////////////////////////////////////////////////////
//
// invoke client for invoking the peer monitor node
// created by qiuyu on Nov 27, 2018
//
///////////////////////////////////////////////////////////
#ifndef __INVOKE_HANDLER_H__
#define __INVOKE_HANDLER_H__
#include "poller.h"
#include "DetectHandlerBase.h"
#include <string>
class CPollThread;
class InvokeHandler : public CPollerObject
{
private:
std::string mIp;
int mPort;
public:
InvokeHandler(
CPollThread* poll,
const std::string& ip,
const int port);
virtual ~InvokeHandler();
virtual void InputNotify(void);
bool initHandler(const bool isInit);
bool invokeVote(
const DetectHandlerBase::DetectType& type,
const std::string& detectedAddr,
const uint64_t sequenceId,
const std::string& data,
int& timeout);
private:
bool attachThread();
bool connectToServer();
void callBack(
const uint64_t sequenceId,
const bool isVote);
};
#endif // __INVOKE_HANDLER_H__

257
src/tools/search_monitor/src/InvokeMgr.cpp Normal file
View File

@ -0,0 +1,257 @@
//////////////////////////////////////////////////
//
// invoke API for cluster
// created by qiuyu on Nov 27,2018
//
//////////////////////////////////////////////////
#include "InvokeMgr.h"
#include "InvokeHandler.h"
#include <sys/select.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
int InvokeMgr::smClusterNodeSize = -1;
// global sequence id for one process, if it's cluster global will be more better
uint64_t InvokeMgr::sgSequenceId = 0;
InvokeMgr::InvokeMgr()
:
mInvokePoll(new CPollThread("dtcInvokePoll")),
mLock(new CMutex())
{
// unit must be ms
mInvokeTimeout = DtcMonitorConfigMgr::getInstance()->getInvokeTimeout();
monitor_log_error("invokeTimeout:%d", mInvokeTimeout);
mNeedStop = false;
mHasInit = false;
}
InvokeMgr::~InvokeMgr()
{
if (mInvokePoll) delete mInvokePoll;
if (mLock) delete mLock;
for (size_t idx = 0; idx < mInvokeHandlers.size(); idx++)
{
if (mInvokeHandlers[idx]) delete mInvokeHandlers[idx];
}
CallBackDataItr_t itr = mRemaingRequests.begin();
while (itr != mRemaingRequests.end())
{
if (itr->second) delete itr->second;
itr++;
}
mRemaingRequests.clear();
}
bool InvokeMgr::startInvokeMgr()
{
if (mHasInit) return true;
// the highest one byte for physical id
int physicalId = DtcMonitorConfigMgr::getInstance()->getPhysicalId();
sgSequenceId = ((sgSequenceId + physicalId) << GLOBAL_PHYSICAL_ID_SHIFT) + 1;
monitor_log_info("start invokeMgr....");
// start epoll thread
int ret = mInvokePoll->InitializeThread();
if (ret < 0)
{
monitor_log_error("initialize thread poll failed.");
return false;
}
bool rslt = initInvokeHandlers();
if (!rslt) return false;
mInvokePoll->RunningThread();
monitor_log_info("start invokeMgr successful.... begin sequenceId:%" PRIu64, sgSequenceId);
mHasInit = true;
return true;
}
void InvokeMgr::stopInvokeMgr()
{
mNeedStop = true;
}
// in current, this function call can not support reentry
bool InvokeMgr::invokeVoteSync(
const DetectHandlerBase::DetectType type,
const std::string& detectedAddr,
const std::string& invokeData,
const int timeout,
bool& needSwitch)
{
// return true;
bool rslt;
SempData_t* sema = new SempData_t();
uint64_t sequenceId;
for (size_t idx = 0; idx < mInvokeHandlers.size(); idx++)
{
sequenceId = getSequenceId();
// use semaphore to keep synchronize for a temp, in the future
// should change it with event_fd
int waitTime = timeout;
mLock->lock();
rslt = mInvokeHandlers[idx]->invokeVote(type, detectedAddr, sequenceId, invokeData, waitTime);
if (!rslt)
{
mLock->unlock();
continue;
}
// vote for timeout with sempahore, delete it in the callback if needed
sema->sTotalVotes++;
mRemaingRequests[sequenceId] = sema;
mLock->unlock();
monitor_log_info("wait for response. sequenceId:%" PRIu64, sequenceId);
// invoke timeout must be greater than detect timeout in order to wait the objective
// response from peer instead of the subjective timeout, its more correctly
waitTime = (mInvokeTimeout > waitTime + 10000/*offset*/) ? mInvokeTimeout : (waitTime + 10000);
sema->semTimeWait(waitTime);
mLock->lock();
CallBackDataItr_t itr = mRemaingRequests.find(sequenceId);
if (itr != mRemaingRequests.end())
{
if (itr->second->sTotalVotes > ((smClusterNodeSize + 1) >> 1))
{
// gather majority of peer votes
needSwitch = true;
monitor_log_info("erase request, sequenceId:%" PRIu64, sequenceId);
mRemaingRequests.erase(sequenceId);
assert(mRemaingRequests.find(sequenceId) == mRemaingRequests.end());
mLock->unlock();
delete sema;
return true;
}
// remove this invoke request from the remaining map
monitor_log_info("erase request, sequenceId:%" PRIu64, sequenceId);
mRemaingRequests.erase(sequenceId);
assert(mRemaingRequests.find(sequenceId) == mRemaingRequests.end());
}
else
{
monitor_log_error("should never come here, sequenceId:%" PRIu64, sequenceId);
}
mLock->unlock();
}
needSwitch = false;
delete sema;
return true;
}
bool InvokeMgr::callBack(
const uint64_t sequenceId,
const bool isVote)
{
SempData_t* sem = NULL;
mLock->lock();
// no need to deal with the sequenceId 0 because that means network issue and lock
// the meaningless thing will cost extra time
// if (0 == sequenceId)
// {
// monitor_log_error("has network issue, notice!!!!!!");
// mLock->unlock();
// return true;
// }
CallBackDataItr_t itr = mRemaingRequests.find(sequenceId);
if (itr != mRemaingRequests.end())
{
monitor_log_error("async call call back, isVote:%d, sequenceId:%" PRIu64, isVote, sequenceId);
sem = itr->second;
if (!isVote)
{
// remove the vote which was set in invoke in order to handle request timeout
sem->sTotalVotes--;
}
}
else
{
mLock->unlock();
monitor_log_error("this request must be timeout, vote it.sequenceId:%" PRIu64, sequenceId);
return true;
}
if (sem) sem->wakeUp();
mLock->unlock();
return true;
}
uint64_t InvokeMgr::getSequenceId()
{
// the last bit in sequence id indicate the vote result, 0 means refuse
// int temp = sgSequenceId;
return __sync_fetch_and_add(&sgSequenceId, 1);
}
// create poll event and add it to the poll
bool InvokeMgr::initInvokeHandlers()
{
if (mInvokeHandlers.size() != 0)
{
// this means user create this object more than one time,
// this can not be permitted
monitor_log_error("can not create singleton class for more than one time.");
return false;
}
monitor_log_info("begin to init invoke handlers.");
// get cluster info
const DtcMonitorConfigMgr::PairVector_t& clusterInfo = DtcMonitorConfigMgr::getInstance()->getClusterInfo();
if (clusterInfo.size() < 2 || clusterInfo.size() > 0x7FFFFFFF) // the other node is self
{
monitor_log_error("cluster node can not less than 3 or larger than '0x7FFFFFFF'. size:%u", (int)clusterInfo.size());
return false;
}
smClusterNodeSize = clusterInfo.size();
bool rslt;
InvokeHandler* handler;
for (size_t idx = 0; idx < clusterInfo.size(); idx++)
{
handler = NULL;
handler = new InvokeHandler(mInvokePoll, clusterInfo[idx].first, clusterInfo[idx].second);
if (!handler)
{
monitor_log_error("create invokeHandler failed.");
return false;
}
rslt = handler->initHandler(true);
if (!rslt)
{
monitor_log_error("create invoke handler failed.");
return false;
}
mInvokeHandlers.push_back(handler);
}
monitor_log_info("create invoke handler successful.");
return true;
}

Some files were not shown because too many files have changed in this diff Show More