mirrors/isearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

complete BoolQueryParser

Browse Source
This commit is contained in:
shzhulin3 2021-04-28 11:23:09 +08:00
parent d0f422521a
commit 108af11443
17 changed files with 198 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/search_local/index_read/comm.h
View File

@ -52,6 +52,7 @@ enum RetCode{
RT_GET_DOC_ERR,
RT_ADD_DICT_ERR,
RT_GET_FIELD_ERROR,
RT_PARSE_CONTENT_ERROR,
};
typedef enum CMD {

14
src/search_local/index_read/component.cc
View File

@ -218,7 +218,7 @@ void Component::InitSwitch()
}
}
void Component::GetQueryWord(uint32_t &m_has_gis){
int Component::GetQueryWord(uint32_t &m_has_gis, string &err_msg){
if(m_query.isObject()){
if(m_query.isMember("bool")){
query_parser = new BoolQueryParser(m_appid, m_query["bool"]);
@ -232,7 +232,12 @@ void Component::GetQueryWord(uint32_t &m_has_gis){
query_parser = new TermQueryParser(m_appid, m_query["term"]);
}
query_parser_res = new QueryParserRes();
query_parser->ParseContent(query_parser_res);
int ret = query_parser->ParseContent(query_parser_res);
if(ret != 0){
err_msg = query_parser_res->ErrMsg();
log_error("query_parser ParseContent error, ret: %d", ret);
return ret;
}
map<uint32_t, vector<FieldInfo> >::iterator field_key_map_iter = query_parser_res->FieldKeysMap().begin();
for(; field_key_map_iter != query_parser_res->FieldKeysMap().end(); field_key_map_iter++){
AddToFieldList(ANDKEY, field_key_map_iter->second);
@ -241,6 +246,10 @@ void Component::GetQueryWord(uint32_t &m_has_gis){
for(; or_key_map_iter != query_parser_res->OrFieldKeysMap().end(); or_key_map_iter++){
AddToFieldList(ORKEY, or_key_map_iter->second);
}
map<uint32_t, vector<FieldInfo> >::iterator invert_key_map_iter = query_parser_res->InvertFieldKeysMap().begin();
for(; invert_key_map_iter != query_parser_res->InvertFieldKeysMap().end(); invert_key_map_iter++){
AddToFieldList(INVERTKEY, invert_key_map_iter->second);
}
m_has_gis = query_parser_res->HasGis();
if(m_has_gis){
latitude = query_parser_res->Latitude();
@ -256,6 +265,7 @@ void Component::GetQueryWord(uint32_t &m_has_gis){
GetFieldWords(ANDKEY, m_Data_and, m_appid, m_has_gis);
GetFieldWords(INVERTKEY, m_Data_invert, m_appid, m_has_gis);
}
return 0;
}
void Component::GetFieldWords(int type, string dataStr, uint32_t appid, uint32_t &m_has_gis){

2
src/search_local/index_read/component.h
View File

@ -31,7 +31,7 @@ public:
Component();
~Component();
void GetQueryWord(uint32_t &m_has_gis);
int GetQueryWord(uint32_t &m_has_gis, string &err_msg);
const vector<vector<FieldInfo> >& Keys();
const vector<vector<FieldInfo> >& AndKeys();
const vector<vector<FieldInfo> >& InvertKeys();

5
src/search_local/index_read/process_task.cc
View File

@ -39,9 +39,12 @@ ProcessTask::~ProcessTask()
}
string ProcessTask::GenReplyStr(RSPCODE code) {
string ProcessTask::GenReplyStr(RSPCODE code, string err_msg) {
Json::Value m_ReplyPacket;
m_ReplyPacket["code"] = code;
if(err_msg != ""){
m_ReplyPacket["message"] = err_msg;
}
Json::StyledWriter _sw;
string resultStr = _sw.write(m_ReplyPacket);
return resultStr;

2
src/search_local/index_read/process_task.h
View File

@ -37,7 +37,7 @@ public:
ProcessTask();
virtual ~ProcessTask();
virtual int Process(CTaskRequest *request);
string GenReplyStr(RSPCODE code);
string GenReplyStr(RSPCODE code, string err_msg = "");
};
#endif

71
src/search_local/index_read/query/bool_query_parser.cc
View File

@ -9,11 +9,17 @@
const char* const NAME ="bool";
const char* const MUST ="must";
const char* const SHOULD ="should";
const char* const MUST_NOT ="must_not";
const char* const TERM ="term";
const char* const MATCH ="match";
const char* const RANGE ="range";
const char* const GEODISTANCE ="geo_distance";
void SetErrMsg(QueryParserRes* query_parser_res, string err_msg){
log_error(err_msg.c_str());
query_parser_res->ErrMsg() = err_msg;
}
BoolQueryParser::BoolQueryParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
{
@ -35,43 +41,84 @@ BoolQueryParser::~BoolQueryParser(){
}
}
void BoolQueryParser::DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res){
int BoolQueryParser::DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res){
if(value.isMember(TERM)){
term_query_parser = new TermQueryParser(appid, value[TERM]);
term_query_parser->ParseContent(query_parser_res, type);
return term_query_parser->ParseContent(query_parser_res, type);
} else if(value.isMember(MATCH)){
match_query_parser = new MatchQueryParser(appid, value[MATCH]);
match_query_parser->ParseContent(query_parser_res, type);
return match_query_parser->ParseContent(query_parser_res, type);
} else if(value.isMember(RANGE)){
range_query_parser = new RangeQueryParser(appid, value[RANGE]);
range_query_parser->ParseContent(query_parser_res, type);
return range_query_parser->ParseContent(query_parser_res, type);
} else if(value.isMember(GEODISTANCE)){
geo_query_parser = new GeoDistanceParser(appid, value[GEODISTANCE]);
geo_query_parser->ParseContent(query_parser_res);
return geo_query_parser->ParseContent(query_parser_res);
} else {
SetErrMsg(query_parser_res, "BoolQueryParser only support term/match/range/geo_distance!");
return -RT_PARSE_CONTENT_ERROR;
}
return 0;
}
void BoolQueryParser::ParseContent(QueryParserRes* query_parser_res){
int BoolQueryParser::ParseContent(QueryParserRes* query_parser_res){
int ret = 0;
if(value.isMember(MUST)){
int type = ANDKEY;
Json::Value must = value[MUST];
if(must.isArray()){
for(int i = 0; i < (int)must.size(); i++){
DoJobByType(must[i], type, query_parser_res);
ret = DoJobByType(must[i], type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
}
} else if (must.isObject()){
DoJobByType(must, type, query_parser_res);
ret = DoJobByType(must, type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
} else if (value.isMember(SHOULD)){
}
}
if (value.isMember(SHOULD)){
int type = ORKEY;
Json::Value should = value[SHOULD];
if(should.isArray()){
for(int i = 0; i < (int)should.size(); i++){
DoJobByType(should[i], type, query_parser_res);
ret = DoJobByType(should[i], type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
}
} else if (should.isObject()){
DoJobByType(should, type, query_parser_res);
ret = DoJobByType(should, type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
}
return;
}
if (value.isMember(MUST_NOT)){
int type = INVERTKEY;
Json::Value must_not = value[MUST_NOT];
if(must_not.isArray()){
for(int i = 0; i < (int)must_not.size(); i++){
ret = DoJobByType(must_not[i], type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
}
} else if (must_not.isObject()) {
ret = DoJobByType(must_not, type, query_parser_res);
if(ret != 0){
log_error("DoJobByType error!");
return -RT_PARSE_CONTENT_ERROR;
}
}
}
return 0;
}

4
src/search_local/index_read/query/bool_query_parser.h
View File

@ -31,10 +31,10 @@ public:
BoolQueryParser(uint32_t a, Json::Value& v);
~BoolQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
int ParseContent(QueryParserRes* query_parser_res);
private:
void DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res);
int DoJobByType(Json::Value& value, uint32_t type, QueryParserRes* query_parser_res);
private:
uint32_t appid;

39
src/search_local/index_read/query/geo_distance_parser.cc
View File

@ -3,6 +3,8 @@
#include <sstream>
const char* const DISTANCE ="distance";
const int GEO_PRECISION = 6;
const int DEFAULT_DISTANCE = 2;
GeoDistanceParser::GeoDistanceParser(uint32_t a, Json::Value& v)
:appid(a),value(v)
@ -22,7 +24,7 @@ vector<double> splitDouble(const string& src, string separate_character)
int separate_characterLen = separate_character.size();
int lastPosition = 0, index = -1;
string str;
int pos = 0;
double pos = 0;
while (-1 != (index = src.find(separate_character, lastPosition)))
{
if (src.substr(lastPosition, index - lastPosition) != " ") {
@ -33,13 +35,19 @@ vector<double> splitDouble(const string& src, string separate_character)
lastPosition = index + separate_characterLen;
}
string lastString = src.substr(lastPosition);//截取最后一个分隔符后的内容
if (!lastString.empty() && lastString != " ")
if (!lastString.empty() && lastString != " "){
pos = atof(lastString.c_str());
strs.push_back(pos);//如果最后一个分隔符后还有内容就入队
}
return strs;
}
void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
void SetErrMsg(QueryParserRes* query_parser_res, string err_msg){
log_error(err_msg.c_str());
query_parser_res->ErrMsg() = err_msg;
}
int GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
vector<FieldInfo> fieldInfos;
double distance = 0;
string fieldname;
@ -50,6 +58,9 @@ void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
if(DISTANCE == *iter){
if(geo_value.isString()){
distance = atof(geo_value.asString().c_str());
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser distance should be string, the unit is km.");
return -RT_PARSE_CONTENT_ERROR;
}
} else {
fieldname = *iter;
@ -59,6 +70,9 @@ void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
if(res.size() >= 2){
geo.lat = res[0];
geo.lon = res[1];
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser format error.");
return -RT_PARSE_CONTENT_ERROR;
}
} else if (geo_value.isArray()){
if(geo_value.size() >= 2){
@ -68,18 +82,33 @@ void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
if(geo_value[1].isDouble()){
geo.lat = geo_value[1].asDouble();
}
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser format error.");
return -RT_PARSE_CONTENT_ERROR;
}
} else if (geo_value.isObject()){
if(geo_value.isMember("lat") && geo_value["lat"].isDouble()){
geo.lat = geo_value["lat"].asDouble();
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser lat format error.");
return -RT_PARSE_CONTENT_ERROR;
}
if(geo_value.isMember("lon") && geo_value["lon"].isDouble()){
geo.lon = geo_value["lon"].asDouble();
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser lon format error.");
return -RT_PARSE_CONTENT_ERROR;
}
} else {
SetErrMsg(query_parser_res, "GeoDistanceParser error, value is not string/array/object.");
return -RT_PARSE_CONTENT_ERROR;
}
}
}
if(distance == 0){
distance = DEFAULT_DISTANCE;
}
vector<string> gisCode = GetArroundGeoHash(geo, distance, 6);
vector<string> gisCode = GetArroundGeoHash(geo, distance, GEO_PRECISION);
if(gisCode.size() > 0){
vector<FieldInfo> fieldInfos;
uint32_t segment_tag = 0;
@ -107,5 +136,5 @@ void GeoDistanceParser::ParseContent(QueryParserRes* query_parser_res){
query_parser_res->Longitude() = sslon.str();
query_parser_res->Distance() = distance;
}
return;
return 0;
}

2
src/search_local/index_read/query/geo_distance_parser.h
View File

@ -27,7 +27,7 @@ class GeoDistanceParser : public QueryParser
public:
GeoDistanceParser(uint32_t a, Json::Value& v);
~GeoDistanceParser();
void ParseContent(QueryParserRes* query_parser_res);
int ParseContent(QueryParserRes* query_parser_res);
private:
uint32_t appid;

15
src/search_local/index_read/query/match_query_parser.cc
View File

@ -12,11 +12,11 @@ MatchQueryParser::~MatchQueryParser(){
}
void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
int MatchQueryParser::ParseContent(QueryParserRes* query_parser_res){
return ParseContent(query_parser_res, ORKEY);
}
void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
int MatchQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
@ -26,7 +26,8 @@ void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t t
fieldname = *iter;
field_value = value[fieldname];
} else {
return;
log_error("TermQueryParser error, value is null");
return -RT_PARSE_CONTENT_ERROR;
}
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
@ -55,9 +56,11 @@ void MatchQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t t
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
} else if(type == ANDKEY){
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else if(type == INVERTKEY){
query_parser_res->InvertFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
return;
return 0;
}

4
src/search_local/index_read/query/match_query_parser.h
View File

@ -26,8 +26,8 @@ class MatchQueryParser : public QueryParser
public:
MatchQueryParser(uint32_t a, Json::Value& v);
~MatchQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
int ParseContent(QueryParserRes* query_parser_res);
int ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;

10
src/search_local/index_read/query/query_parser.h
View File

@ -32,6 +32,9 @@ public:
map<uint32_t, vector<FieldInfo> >& OrFieldKeysMap(){
return or_field_keys_map;
}
map<uint32_t, vector<FieldInfo> >& InvertFieldKeysMap(){
return invert_field_keys_map;
}
vector<ExtraFilterKey>& ExtraFilterKeys(){
return extra_filter_keys;
}
@ -53,13 +56,18 @@ public:
double& Distance(){
return distance;
}
string& ErrMsg(){
return err_msg;
}
private:
uint32_t m_has_gis;
string latitude;
string longitude;
double distance;
string err_msg;
map<uint32_t, vector<FieldInfo> > field_keys_map;
map<uint32_t, vector<FieldInfo> > or_field_keys_map;
map<uint32_t, vector<FieldInfo> > invert_field_keys_map;
vector<ExtraFilterKey> extra_filter_keys;
vector<ExtraFilterKey> extra_filter_and_keys;
vector<ExtraFilterKey> extra_filter_invert_keys;
@ -67,7 +75,7 @@ private:
class QueryParser{
public:
virtual void ParseContent(QueryParserRes* query_parser_res) = 0;
virtual int ParseContent(QueryParserRes* query_parser_res) = 0;
virtual ~QueryParser() {};
};

39
src/search_local/index_read/query/range_query_parser.cc
View File

@ -16,11 +16,11 @@ RangeQueryParser::~RangeQueryParser(){
}
void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
int RangeQueryParser::ParseContent(QueryParserRes* query_parser_res){
return ParseContent(query_parser_res, ORKEY);
}
void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
int RangeQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
@ -42,6 +42,8 @@ void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t t
} else if(field_value.isMember(LT)){
end = field_value[LT];
info.range_type = RANGE_GELT;
} else {
info.range_type = RANGE_GE;
}
} else if(field_value.isMember(GT)){
start = field_value[GT];
@ -51,20 +53,43 @@ void RangeQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t t
} else if(field_value.isMember(LT)){
end = field_value[LT];
info.range_type = RANGE_GTLT;
} else {
info.range_type = RANGE_GT;
}
} else if(field_value.isMember(LTE)){
end = field_value[LTE];
info.range_type = RANGE_LE;
} else if(field_value.isMember(LT)){
end = field_value[LT];
info.range_type = RANGE_LT;
}
fieldInfo.start = start.asInt();
fieldInfo.end = end.asInt();
if(!start.isInt() && !start.isNull()){
log_error("range query only support integer");
return -RT_PARSE_CONTENT_ERROR;
}
if(!end.isInt() && !end.isNull()){
log_error("range query only support integer");
return -RT_PARSE_CONTENT_ERROR;
}
if(start.isInt() || end.isInt()){
fieldInfo.start = start.isInt() ? start.asInt() : 0;
fieldInfo.end = end.isInt() ? end.asInt() : 0;
fieldInfo.range_type = info.range_type;
fieldInfos.push_back(fieldInfo);
}
}
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
} else if(type == ANDKEY){
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else if(type == INVERTKEY){
query_parser_res->InvertFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
} else {
log_error("RangeQueryParser error, value is null");
return -RT_PARSE_CONTENT_ERROR;
}
return;
return 0;
}

4
src/search_local/index_read/query/range_query_parser.h
View File

@ -26,8 +26,8 @@ class RangeQueryParser : public QueryParser
public:
RangeQueryParser(uint32_t a, Json::Value& v);
~RangeQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
int ParseContent(QueryParserRes* query_parser_res);
int ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;

19
src/search_local/index_read/query/term_query_parser.cc
View File

@ -12,11 +12,11 @@ TermQueryParser::~TermQueryParser(){
}
void TermQueryParser::ParseContent(QueryParserRes* query_parser_res){
ParseContent(query_parser_res, ORKEY);
int TermQueryParser::ParseContent(QueryParserRes* query_parser_res){
return ParseContent(query_parser_res, ORKEY);
}
void TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
int TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t type){
vector<FieldInfo> fieldInfos;
Json::Value::Members member = value.getMemberNames();
Json::Value::Members::iterator iter = member.begin();
@ -28,7 +28,8 @@ void TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t ty
json_value = value[fieldname];
field_value = json_value.asString();
} else {
return;
log_error("TermQueryParser error, value is null");
return -RT_PARSE_CONTENT_ERROR;
}
uint32_t segment_tag = 0;
FieldInfo fieldInfo;
@ -45,9 +46,7 @@ void TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t ty
} else if (type == INVERTKEY) {
query_parser_res->ExtraFilterInvertKeys().push_back(extra_filter_key);
}
return;
}
if (field != 0)
} else if (field != 0)
{
fieldInfo.word = field_value;
fieldInfos.push_back(fieldInfo);
@ -55,9 +54,11 @@ void TermQueryParser::ParseContent(QueryParserRes* query_parser_res, uint32_t ty
if(fieldInfos.size() != 0){
if(type == ORKEY){
query_parser_res->OrFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else {
} else if(type == ANDKEY) {
query_parser_res->FieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
} else if(type == INVERTKEY){
query_parser_res->InvertFieldKeysMap().insert(make_pair(fieldInfo.field, fieldInfos));
}
}
return;
return 0;
}

4
src/search_local/index_read/query/term_query_parser.h
View File

@ -26,8 +26,8 @@ class TermQueryParser : public QueryParser
public:
TermQueryParser(uint32_t a, Json::Value& v);
~TermQueryParser();
void ParseContent(QueryParserRes* query_parser_res);
void ParseContent(QueryParserRes* query_parser_res, uint32_t type);
int ParseContent(QueryParserRes* query_parser_res);
int ParseContent(QueryParserRes* query_parser_res, uint32_t type);
private:
uint32_t appid;

11
src/search_local/index_read/search_task.cc
View File

@ -597,7 +597,14 @@ int SearchTask::Process(CTaskRequest *request)
skipList.InitList();
component->InitSwitch();
log_debug("m_Data: %s", m_Primary_Data.c_str());
component->GetQueryWord(m_has_gis);
string err_msg = "";
int ret = component->GetQueryWord(m_has_gis, err_msg);
if (ret != 0) {
string str = GenReplyStr(PARAMETER_ERR, err_msg);
request->setResult(str);
common::ProfilerMonitor::GetInstance().FunctionError(caller_info);
return ret;
}
if(component->TerminalTag() == 1 && component->TerminalTagValid() == false){
log_error("TerminalTag is 1 and TerminalTagValid is false.");
common::ProfilerMonitor::GetInstance().FunctionError(caller_info);
@ -634,7 +641,7 @@ int SearchTask::Process(CTaskRequest *request)
}
}
logical_operate = new LogicalOperate(m_appid, m_sort_type, m_has_gis, component->CacheSwitch());
int ret = DoJob(request);
ret = DoJob(request);
if (ret != 0) {
string str = GenReplyStr(PARAMETER_ERR);
request->setResult(str);