mirrors/isearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add query_parser

Browse Source
This commit is contained in:
shzhulin3 2021-04-22 14:16:00 +08:00
parent f026781a61
commit 1571dbc4ea
14 changed files with 693 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/search_local/index_read/comm.h
View File

@ -236,7 +236,7 @@ struct CacheQueryInfo
enum KeyType
{
MAINKEY,
ORKEY,
ANDKEY,
INVERTKEY,
};

44
src/search_local/index_read/component.cc
View File

@ -20,6 +20,7 @@
#include "split_manager.h"
#include "db_manager.h"
#include "utf8_str.h"
#include "query/bool_query_parser.h"
#include <sstream>
using namespace std;
@ -35,7 +36,6 @@ Component::Component(){
m_snapshot_switch = 0;
m_sort_type = SORT_RELEVANCE;
m_appid = 10001;
m_user_id = "0";
m_last_id = "";
m_last_score = "";
m_search_after = false;
@ -45,7 +45,12 @@ Component::Component(){
}
Component::~Component(){
if(NULL != query_parser){
delete query_parser;
}
if(NULL != query_parser_res){
delete query_parser_res;
}
}
int Component::ParseJson(const char *sz_json, int json_len, Json::Value &recv_packet)
@ -68,12 +73,8 @@ int Component::ParseJson(const char *sz_json, int json_len, Json::Value &recv_pa
m_appid = 10001;
}
if (recv_packet.isMember("userid") && recv_packet["userid"].isString())
{
m_user_id = recv_packet["userid"].asString();
}
else {
m_user_id = "0";
if(recv_packet.isMember("query")){
m_query = recv_packet["query"];
}
if (recv_packet.isMember("key") && recv_packet["key"].isString())
@ -214,9 +215,34 @@ void Component::InitSwitch()
}
void Component::GetQueryWord(uint32_t &m_has_gis){
GetFieldWords(MAINKEY, m_Data, m_appid, m_has_gis);
if(m_query.isObject()){
if(m_query.isMember("bool")){
query_parser = new BoolQueryParser(m_appid, m_query["bool"]);
}
query_parser_res = new QueryParserRes();
query_parser->ParseContent(query_parser_res);
map<uint32_t, vector<FieldInfo> >::iterator field_key_map_iter = query_parser_res->FieldKeysMap().begin();
for(; field_key_map_iter != query_parser_res->FieldKeysMap().end(); field_key_map_iter++){
AddToFieldList(ANDKEY, field_key_map_iter->second);
}
map<uint32_t, vector<FieldInfo> >::iterator or_key_map_iter = query_parser_res->OrFieldKeysMap().begin();
for(; or_key_map_iter != query_parser_res->OrFieldKeysMap().end(); or_key_map_iter++){
AddToFieldList(ORKEY, or_key_map_iter->second);
}
m_has_gis = query_parser_res->HasGis();
if(m_has_gis){
latitude = query_parser_res->Latitude();
longitude = query_parser_res->Longitude();
distance = query_parser_res->Distance();
}
extra_filter_keys.assign(query_parser_res->ExtraFilterKeys().begin(), query_parser_res->ExtraFilterKeys().end());
extra_filter_and_keys.assign(query_parser_res->ExtraFilterAndKeys().begin(), query_parser_res->ExtraFilterAndKeys().end());
extra_filter_invert_keys.assign(query_parser_res->ExtraFilterInvertKeys().begin(), query_parser_res->ExtraFilterInvertKeys().end());
} else {
GetFieldWords(ORKEY, m_Data, m_appid, m_has_gis);
GetFieldWords(ANDKEY, m_Data_and, m_appid, m_has_gis);
GetFieldWords(INVERTKEY, m_Data_invert, m_appid, m_has_gis);
}
}
void Component::GetFieldWords(int type, string dataStr, uint32_t appid, uint32_t &m_has_gis){

5
src/search_local/index_read/component.h
View File

@ -22,6 +22,7 @@
#include "json/json.h"
#include <string>
#include <vector>
#include "query/query_parser.h"
using namespace std;
class Component
@ -100,7 +101,6 @@ private:
uint32_t m_snapshot_switch;
uint32_t m_sort_type;
uint32_t m_appid;
string m_user_id;
string m_sort_field;
string m_last_id;
string m_last_score;
@ -110,5 +110,8 @@ private:
uint32_t m_jdq_switch;
uint32_t m_terminal_tag;
bool m_terminal_tag_valid;
Json::Value m_query;
QueryParser* query_parser;
QueryParserRes* query_parser_res;
};
#endif

69
src/search_local/index_read/query/bool_query_parser.cc Normal file
View File

@ -0,0 +1,69 @@
#include "bool_query_parser.h"
#include "../db_manager.h"
#include "../split_manager.h"
#include "range_query_parser.h"
#include "term_query_parser.h"
#include "match_query_parser.h"
const char* const BoolQueryParser::NAME ="bool";
const char* const BoolQueryParser::MUST ="must";
const char* const BoolQueryParser::SHOULD ="should";
const char* const BoolQueryParser::TERM ="term";
const char* const BoolQueryParser::MATCH ="match";
const char* const BoolQueryParser::RANGE ="range";
BoolQueryParser::BoolQueryParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
}
BoolQueryParser::~BoolQueryParser(){
if(NULL != range_query_parser){
delete range_query_parser;
}
if(NULL != term_query_parser){
delete term_query_parser;
}
if(NULL != match_query_parser){
delete match_query_parser;
}
}
void BoolQueryParser::DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res){
if(value.isMember(TERM)){
term_query_parser = new TermQueryParser(appid, value[TERM]);
term_query_parser->ParseContent(query_parser_res, type);
} else if(value.isMember(MATCH)){
match_query_parser = new MatchQueryParser(appid, value[MATCH]);
match_query_parser->ParseContent(query_parser_res, type);
} else if(value.isMember(RANGE)){
range_query_parser = new RangeQueryParser(appid, value[RANGE]);
range_query_parser->ParseContent(query_parser_res, type);
}
}
void BoolQueryParser::ParseContent(QueryParserRes* query_parser_res){
if(value.isMember(MUST)){
int type = ANDKEY;
Json::Value must = value[MUST];
if(must.isArray()){
for(int i = 0; i < (int)must.size(); i++){
DoJobByType(must[i], type, query_parser_res);
}
} else if (must.isObject()){
DoJobByType(must, type, query_parser_res);
}
} else if (value.isMember(SHOULD)){
int type = ORKEY;
Json::Value should = value[SHOULD];
if(should.isArray()){
for(int i = 0; i < (int)should.size(); i++){
DoJobByType(should[i], type, query_parser_res);
}
} else if (should.isObject()){
DoJobByType(should, type, query_parser_res);
}
}
return;
}

52
src/search_local/index_read/query/bool_query_parser.h Normal file
View File

@ -0,0 +1,52 @@
/*
* =====================================================================================
*
* Filename: bool_query_parser.h
*
* Description: bool_query_parser class definition.
*
* Version: 1.0
* Created: 05/03/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __BOOL_QUERY_PARSER_H__
#define __BOOL_QUERY_PARSER_H__
#include "query_parser.h"
#include "json/json.h"
class RangeQueryParser;
class TermQueryParser;
class MatchQueryParser;
class BoolQueryParser : public QueryParser
{
public:
BoolQueryParser(uint32_t a, Json::Value& v);
~BoolQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
private:
void DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res);
private:
uint32_t appid;
Json::Value value;
static const char* const NAME;
static const char* const MUST;
static const char* const SHOULD;
static const char* const TERM;
static const char* const MATCH;
static const char* const RANGE;
RangeQueryParser* range_query_parser;
TermQueryParser* term_query_parser;
MatchQueryParser* match_query_parser;
};
#endif

111
src/search_local/index_read/query/geo_distance_parser.cc Normal file
View File

@ -0,0 +1,111 @@
#include "geo_distance_parser.h"
#include "../db_manager.h"
#include <sstream>
const char* const DISTANCE ="distance";
GeoDistanceParser::GeoDistanceParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
}
GeoDistanceParser::~GeoDistanceParser(){
}
vector<double> splitDouble(const string& src, string separate_character)
{
vector<double> strs;
//分割字符串的长度,这样就可以支持如“,,”多字符串的分隔符
int separate_characterLen = separate_character.size();
int lastPosition = 0, index = -1;
string str;
int pos = 0;
while (-1 != (index = src.find(separate_character, lastPosition)))
{
if (src.substr(lastPosition, index - lastPosition) != " ") {
str = src.substr(lastPosition, index - lastPosition);
pos = atof(str.c_str());
strs.push_back(pos);
}
lastPosition = index + separate_characterLen;
}
string lastString = src.substr(lastPosition);//截取最后一个分隔符后的内容
if (!lastString.empty() && lastString != " ")
pos = atof(lastString.c_str());
strs.push_back(pos);//如果最后一个分隔符后还有内容就入队
return strs;
}
void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
vector<FieldInfo> fieldInfos;
double distance = 0;
string fieldname;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
for(; iter != member.end(); iter++){
Json::Value geo_value = value[*iter];
if(DISTANCE == *iter){
if(geo_value.isString()){
distance = atof(geo_value.asString().c_str());
}
} else {
fieldname = *iter;
if(geo_value.isString()){
string geo_str = geo_value.asString();
vector<double> res = splitDouble(geo_str, ",");
if(res.size() >= 2){
geo.lat = res[0];
geo.lon = res[1];
}
} else if (geo_value.isArray()){
if(geo_value.size() >= 2){
if(geo_value[0].isDouble()){
geo.lon = geo_value[0].asDouble();
}
if(geo_value[1].isDouble()){
geo.lat = geo_value[1].asDouble();
}
}
} else if (geo_value.isObject()){
if(geo_value.isMember("lat") && geo_value["lat"].isDouble()){
geo.lat = geo_value["lat"].asDouble();
}
if(geo_value.isMember("lon") && geo_value["lon"].isDouble()){
geo.lon = geo_value["lon"].asDouble();
}
}
}
}
vector<string> gisCode = GetArroundGeoHash(geo, distance, 6);
if(gisCode.size() > 0){
vector<FieldInfo> fieldInfos;
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
uint32_t field = DBManager::Instance()->GetWordField(segment_tag, appid, fieldname, fieldInfo);
if (field != 0 && segment_tag == 0) {
query_parser_res->HasGis() = 1;
for (size_t index = 0; index < gisCode.size(); index++) {
FieldInfo info;
info.field = fieldInfo.field;
info.field_type = fieldInfo.field_type;
info.segment_tag = fieldInfo.segment_tag;
info.word = gisCode[index];
fieldInfos.push_back(info);
}
}
if (fieldInfos.size() != 0) {
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
stringstream sslat;
stringstream sslon;
sslat << geo.lat;
sslon << geo.lon;
query_parser_res->Latitude() = sslat.str();
query_parser_res->Longitude() = sslon.str();
query_parser_res->Distance() = distance;
}
return;
}

39
src/search_local/index_read/query/geo_distance_parser.h Normal file
View File

@ -0,0 +1,39 @@
/*
* =====================================================================================
*
* Filename: geo_distance_parser.h
*
* Description: geo_distance_parser class definition.
*
* Version: 1.0
* Created: 20/04/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __GEO_DISTANCE_PARSER_H__
#define __GEO_DISTANCE_PARSER_H__
#include "query_parser.h"
#include "json/json.h"
#include "geohash.h"
class GeoDistanceParser : public QueryParser
{
public:
GeoDistanceParser(uint32_t a, Json::Value& v);
~GeoDistanceParser();
void ParseContent(QueryParserRes* query_parser_res);
private:
uint32_t appid;
Json::Value value;
GeoPoint geo;
double distance;
};
#endif

63
src/search_local/index_read/query/match_query_parser.cc Normal file
View File

@ -0,0 +1,63 @@
#include "match_query_parser.h"
#include "../db_manager.h"
#include "../split_manager.h"
MatchQueryParser::MatchQueryParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
}
MatchQueryParser::~MatchQueryParser(){
}
void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
}
void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
string fieldname;
Json::Value field_value;
if(iter != member.end()){ // 一个match下只对应一个字段
fieldname = *iter;
field_value = value[fieldname];
} else {
return;
}
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
uint32_t field = DBManager::Instance()->GetWordField(segment_tag, appid, fieldname, fieldInfo);
if (field != 0 && segment_tag == 1)
{
string split_data = SplitManager::Instance()->split(field_value.asString(), appid);
log_debug("split_data: %s", split_data.c_str());
vector<string> split_datas = splitEx(split_data, "|");
for(size_t index = 0; index < split_datas.size(); index++)
{
FieldInfo info;
info.field = fieldInfo.field;
info.field_type = fieldInfo.field_type;
info.word = split_datas[index];
info.segment_tag = fieldInfo.segment_tag;
fieldInfos.push_back(info);
}
}
else if (field != 0)
{
fieldInfo.word = field_value.asString();
fieldInfos.push_back(fieldInfo);
}
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
return;
}

37
src/search_local/index_read/query/match_query_parser.h Normal file
View File

@ -0,0 +1,37 @@
/*
* =====================================================================================
*
* Filename: match_query_parser.h
*
* Description: match_query_parser class definition.
*
* Version: 1.0
* Created: 20/04/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __MATCH_QUERY_PARSER_H__
#define __MATCH_QUERY_PARSER_H__
#include "query_parser.h"
#include "json/json.h"
class MatchQueryParser : public QueryParser
{
public:
MatchQueryParser(uint32_t a, Json::Value& v);
~MatchQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;
Json::Value value;
};
#endif

74
src/search_local/index_read/query/query_parser.h Normal file
View File

@ -0,0 +1,74 @@
/*
* =====================================================================================
*
* Filename: query_parser.h
*
* Description: query_parser class definition.
*
* Version: 1.0
* Created: 19/04/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __QUERY_PARSER_H__
#define __QUERY_PARSER_H__
#include "../comm.h"
#include <map>
class QueryParserRes{
public:
QueryParserRes(){
m_has_gis = 0;
}
map<uint32_t, vector<FieldInfo> >& FieldKeysMap(){
return field_keys_map;
}
map<uint32_t, vector<FieldInfo> >& OrFieldKeysMap(){
return or_field_keys_map;
}
vector<ExtraFilterKey>& ExtraFilterKeys(){
return extra_filter_keys;
}
vector<ExtraFilterKey>& ExtraFilterAndKeys(){
return extra_filter_and_keys;
}
vector<ExtraFilterKey>& ExtraFilterInvertKeys(){
return extra_filter_invert_keys;
}
uint32_t& HasGis(){
return m_has_gis;
}
string& Latitude(){
return latitude;
}
string& Longitude(){
return latitude;
}
double& Distance(){
return distance;
}
private:
uint32_t m_has_gis;
string latitude;
string longitude;
double distance;
map<uint32_t, vector<FieldInfo> > field_keys_map;
map<uint32_t, vector<FieldInfo> > or_field_keys_map;
vector<ExtraFilterKey> extra_filter_keys;
vector<ExtraFilterKey> extra_filter_and_keys;
vector<ExtraFilterKey> extra_filter_invert_keys;
};
class QueryParser{
public:
virtual void ParseContent(QueryParserRes* query_parser_res) = 0;
virtual ~QueryParser() {};
};
#endif

69
src/search_local/index_read/query/range_query_parser.cc Normal file
View File

@ -0,0 +1,69 @@
#include "range_query_parser.h"
#include "../db_manager.h"
const char* const GTE ="gte";
const char* const GT ="gt";
const char* const LTE ="lte";
const char* const LT ="lt";
RangeQueryParser::RangeQueryParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
}
RangeQueryParser::~RangeQueryParser(){
}
void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
}
void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
if(iter != member.end()){ // 一个range下只对应一个字段
string fieldname = *iter;
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
DBManager::Instance()->GetWordField(segment_tag, appid, fieldname, fieldInfo);
Json::Value field_value = value[fieldname];
if(field_value.isObject()){
FieldInfo info;
Json::Value start;
Json::Value end;
if(field_value.isMember(GTE)){
start = field_value[GTE];
if(field_value.isMember(LTE)){
end = field_value[LTE];
info.range_type = RANGE_GELE;
} else if(field_value.isMember(LT)){
end = field_value[LT];
info.range_type = RANGE_GELT;
}
} else if(field_value.isMember(GT)){
start = field_value[GT];
if(field_value.isMember(LTE)){
end = field_value[LTE];
info.range_type = RANGE_GTLE;
} else if(field_value.isMember(LT)){
end = field_value[LT];
info.range_type = RANGE_GTLT;
}
}
info.start = start.asInt();
info.end = end.asInt();
fieldInfos.push_back(info);
}
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
}
return;
}

37
src/search_local/index_read/query/range_query_parser.h Normal file
View File

@ -0,0 +1,37 @@
/*
* =====================================================================================
*
* Filename: range_query_parser.h
*
* Description: range_query_parser class definition.
*
* Version: 1.0
* Created: 19/04/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __RANGE_QUERY_PARSER_H__
#define __RANGE_QUERY_PARSER_H__
#include "query_parser.h"
#include "json/json.h"
class RangeQueryParser : public QueryParser
{
public:
RangeQueryParser(uint32_t a, Json::Value& v);
~RangeQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;
Json::Value value;
};
#endif

63
src/search_local/index_read/query/term_query_parser.cc Normal file
View File

@ -0,0 +1,63 @@
#include "term_query_parser.h"
#include "../db_manager.h"
TermQueryParser::TermQueryParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
}
TermQueryParser::~TermQueryParser(){
}
void TermQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
}
void TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
string fieldname;
string field_value;
Json::Value json_value;
if(iter != member.end()){ // 一个term下只对应一个字段
fieldname = *iter;
json_value = value[fieldname];
field_value = json_value.asString();
} else {
return;
}
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
uint32_t field = DBManager::Instance()->GetWordField(segment_tag, appid, fieldname, fieldInfo);
if(field != 0 && fieldInfo.index_tag == 0){
ExtraFilterKey extra_filter_key;
extra_filter_key.field_name = fieldname;
extra_filter_key.field_value = field_value;
extra_filter_key.field_type = fieldInfo.field_type;
if(type == ORKEY){
query_parser_res->ExtraFilterKeys().push_back(extra_filter_key);
} else if (type == ANDKEY) {
query_parser_res->ExtraFilterAndKeys().push_back(extra_filter_key);
} else if (type == INVERTKEY) {
query_parser_res->ExtraFilterInvertKeys().push_back(extra_filter_key);
}
return;
}
if (field != 0)
{
fieldInfo.word = field_value;
fieldInfos.push_back(fieldInfo);
}
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
return;
}

37
src/search_local/index_read/query/term_query_parser.h Normal file
View File

@ -0,0 +1,37 @@
/*
* =====================================================================================
*
* Filename: term_query_parser.h
*
* Description: term_query_parser class definition.
*
* Version: 1.0
* Created: 20/04/2021
* Revision: none
* Compiler: gcc
*
* Author: zhulin, shzhulin3@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __TERM_QUERY_PARSER_H__
#define __TERM_QUERY_PARSER_H__
#include "query_parser.h"
#include "json/json.h"
class TermQueryParser : public QueryParser
{
public:
TermQueryParser(uint32_t a, Json::Value& v);
~TermQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;
Json::Value value;
};
#endif