872 lines
25 KiB
C++
872 lines
25 KiB
C++
|
#include "split_tool.h"
|
|||
|
|
#include <fstream>
|
|||
|
|
#include <vector>
|
|||
|
|
#include <stdio.h>
|
|||
|
|
#include <stdint.h>
|
|||
|
|
#include <math.h>
|
|||
|
|
#include <stdlib.h>
|
|||
|
|
#include <float.h>
|
|||
|
|
#include "log.h"
|
|||
|
|
using namespace std;
|
|||
|
|
#define MAX_WORD_LEN 8
|
|||
|
|
#define TOTAL 8000000
|
|||
|
|
#define ALPHA_DIGIT "0123456789１２３４５６７８９０\
|
|||
|
|
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZａｂｃｄｅｆｇｈｉｇｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＵＶＷＸＹＺ"
|
|||
|
|
|
|||
|
|
|
|||
|
|
bool isAllAlphaOrDigit(string str) {
|
|||
|
|
bool flag = true;
|
|||
|
|
size_t i = 0;
|
|||
|
|
for (; i < str.size(); i++) {
|
|||
|
|
if (!isupper(str[i]) && !islower(str[i]) && !isdigit(str[i])) {
|
|||
|
|
flag = false;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return flag;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
FBSegment::FBSegment() {
|
|||
|
|
punct_set.clear();
|
|||
|
|
train_cnt = 0;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::isAlphaOrDigit(string str) {
|
|||
|
|
if (alpha_set.find(str) != alpha_set.end())
|
|||
|
|
{
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
bool FBSegment::Init() {
|
|||
|
|
|
|||
|
|
string en_punct = ",.!?/'\"<>\\:;\n";
|
|||
|
|
string punct = "，。！？、；：“”‘’（）《》 ";
|
|||
|
|
punct = punct.append(en_punct);
|
|||
|
|
iutf8string utf8_punct(punct);
|
|||
|
|
for (int i = 0; i < utf8_punct.length(); i++) {
|
|||
|
|
punct_set.insert(utf8_punct[i]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
string alphadigit = ALPHA_DIGIT;
|
|||
|
|
iutf8string utf8_alpha(alphadigit);
|
|||
|
|
for (int i = 0; i < utf8_alpha.length(); i++) {
|
|||
|
|
alpha_set.insert(utf8_alpha[i]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::Init2(string train_path) {
|
|||
|
|
bool ret = Init();
|
|||
|
|
if (ret == false) {
|
|||
|
|
return ret;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
string str;
|
|||
|
|
ifstream train_infile;
|
|||
|
|
train_infile.open(train_path.c_str());
|
|||
|
|
if (train_infile.is_open() == false) {
|
|||
|
|
log_error("open file error: %s.\n", train_path.c_str());
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
string beg_tag = "<BEG>";
|
|||
|
|
string end_tag = "<END>";
|
|||
|
|
while (getline(train_infile, str))
|
|||
|
|
{
|
|||
|
|
vector<string> str_vec = splitEx(str, " ");
|
|||
|
|
vector<string> line_list;
|
|||
|
|
vector<string>::iterator iter = str_vec.begin();
|
|||
|
|
for (; iter != str_vec.end(); iter++) {
|
|||
|
|
if (punct_set.find(*iter) == punct_set.end() && *iter != "") {
|
|||
|
|
line_list.push_back(*iter);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
train_cnt += line_list.size();
|
|||
|
|
for (int i = -1; i < (int)line_list.size(); i++) {
|
|||
|
|
string word1;
|
|||
|
|
string word2;
|
|||
|
|
if (i == -1) {
|
|||
|
|
word1 = beg_tag;
|
|||
|
|
word2 = line_list[i + 1];
|
|||
|
|
}
|
|||
|
|
else if (i == (int)line_list.size() - 1) {
|
|||
|
|
word1 = line_list[i];
|
|||
|
|
word2 = end_tag;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
word1 = line_list[i];
|
|||
|
|
word2 = line_list[i + 1];
|
|||
|
|
}
|
|||
|
|
if (next_dict.find(word1) == next_dict.end()) {
|
|||
|
|
map<string, int> dict;
|
|||
|
|
next_dict[word1] = dict;
|
|||
|
|
}
|
|||
|
|
if (next_dict[word1].find(word2) == next_dict[word1].end()) {
|
|||
|
|
next_dict[word1][word2] = 1;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
next_dict[word1][word2] += 1;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
train_infile.close();
|
|||
|
|
|
|||
|
|
ret = train_corpus.Init(train_path);
|
|||
|
|
if (ret == false) {
|
|||
|
|
log_error("train_corpus init error.");
|
|||
|
|
return ret;
|
|||
|
|
}
|
|||
|
|
log_info("total training words length is: %u, next_dict count: %d.", train_cnt, (int)next_dict.size());
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::Init3(string train_path, string word_path) {
|
|||
|
|
bool ret = Init2(train_path);
|
|||
|
|
if (ret == false) {
|
|||
|
|
return ret;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
string str;
|
|||
|
|
ifstream word_infile;
|
|||
|
|
word_infile.open(word_path.c_str());
|
|||
|
|
if (word_infile.is_open() == false) {
|
|||
|
|
log_error("open file error: %s.\n", word_path.c_str());
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
uint32_t word_id = 0;
|
|||
|
|
uint32_t appid = 0;
|
|||
|
|
string word;
|
|||
|
|
uint32_t word_freq = 0;
|
|||
|
|
while (getline(word_infile, str))
|
|||
|
|
{
|
|||
|
|
vector<string> str_vec = splitEx(str, "\t");
|
|||
|
|
word_id = atoi(str_vec[0].c_str());
|
|||
|
|
word = str_vec[1];
|
|||
|
|
appid = atoi(str_vec[2].c_str());
|
|||
|
|
word_freq = atoi(str_vec[3].c_str());
|
|||
|
|
WordInfo word_info;
|
|||
|
|
word_info.appid = appid;
|
|||
|
|
word_info.word_freq = word_freq;
|
|||
|
|
word_info.word_id = word_id;
|
|||
|
|
word_dict[word][appid] = word_info;
|
|||
|
|
}
|
|||
|
|
log_info("word_dict count: %d", (int)word_dict.size());
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::WordValid(string word, uint32_t appid) {
|
|||
|
|
if(punct_set.find(word) != punct_set.end()){
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
if (word_dict.find(word) != word_dict.end()) {
|
|||
|
|
map<uint32_t, WordInfo> wordInfo = word_dict[word];
|
|||
|
|
if (wordInfo.find(0) != wordInfo.end() || wordInfo.find(appid) != wordInfo.end()) {
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::GetWordInfo(string word, uint32_t appid, WordInfo &word_info) {
|
|||
|
|
if (word_dict.find(word) != word_dict.end()) {
|
|||
|
|
map<uint32_t, WordInfo> wordInfo = word_dict[word];
|
|||
|
|
if (wordInfo.find(0) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[0];
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
if (wordInfo.find(appid) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[appid];
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool FBSegment::GetWordInfoFromDictOnly(string word, uint32_t appid, WordInfo &word_info) {
|
|||
|
|
if (word_dict.find(word) != word_dict.end()) {
|
|||
|
|
map<uint32_t, WordInfo> wordInfo = word_dict[word];
|
|||
|
|
if (wordInfo.find(0) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[0];
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
if (wordInfo.find(appid) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[appid];
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::FMM2(iutf8string &phrase, uint32_t appid, vector<string> &fmm_list) {
|
|||
|
|
int maxlen = MAX_WORD_LEN;
|
|||
|
|
int len_phrase = phrase.length();
|
|||
|
|
int i = 0, j = 0;
|
|||
|
|
|
|||
|
|
while (i < len_phrase) {
|
|||
|
|
int end = i + maxlen;
|
|||
|
|
if (end >= len_phrase)
|
|||
|
|
end = len_phrase;
|
|||
|
|
iutf8string phrase_sub = phrase.utf8substr(i, end - i);
|
|||
|
|
for (j = phrase_sub.length(); j >= 0; j--) {
|
|||
|
|
if (j == 1)
|
|||
|
|
break;
|
|||
|
|
iutf8string key = phrase_sub.utf8substr(0, j);
|
|||
|
|
if (WordValid(key.stlstring(), appid) == true) {
|
|||
|
|
fmm_list.push_back(key.stlstring());
|
|||
|
|
i += key.length() - 1;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (j == 1) {
|
|||
|
|
fmm_list.push_back(phrase_sub[0]);
|
|||
|
|
}
|
|||
|
|
i += 1;
|
|||
|
|
}
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// not query from DTC
|
|||
|
|
void FBSegment::BMM(iutf8string &phrase, uint32_t appid, vector<string> &bmm_list) {
|
|||
|
|
int maxlen = MAX_WORD_LEN - 2;
|
|||
|
|
int len_phrase = phrase.length();
|
|||
|
|
int i = len_phrase, j = 0;
|
|||
|
|
|
|||
|
|
while (i > 0) {
|
|||
|
|
int start = i - maxlen;
|
|||
|
|
if (start < 0)
|
|||
|
|
start = 0;
|
|||
|
|
iutf8string phrase_sub = phrase.utf8substr(start, i - start);
|
|||
|
|
for (j = 0; j < phrase_sub.length(); j++) {
|
|||
|
|
if (j == phrase_sub.length() - 1)
|
|||
|
|
break;
|
|||
|
|
iutf8string key = phrase_sub.utf8substr(j, phrase_sub.length() - j);
|
|||
|
|
string word = key.stlstring();
|
|||
|
|
if (WordValid(word, appid) == true) {
|
|||
|
|
vector<string>::iterator iter = bmm_list.begin();
|
|||
|
|
bmm_list.insert(iter, key.stlstring());
|
|||
|
|
i -= key.length() - 1;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (j == phrase_sub.length() - 1) {
|
|||
|
|
vector<string>::iterator iter = bmm_list.begin();
|
|||
|
|
bmm_list.insert(iter, "" + phrase_sub[j]);
|
|||
|
|
}
|
|||
|
|
i -= 1;
|
|||
|
|
}
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::BMM2(iutf8string &phrase, uint32_t appid, vector<string> &bmm_list) {
|
|||
|
|
int maxlen = MAX_WORD_LEN;
|
|||
|
|
int len_phrase = phrase.length();
|
|||
|
|
int i = len_phrase, j = 0;
|
|||
|
|
|
|||
|
|
while (i > 0) {
|
|||
|
|
int start = i - maxlen;
|
|||
|
|
if (start < 0)
|
|||
|
|
start = 0;
|
|||
|
|
iutf8string phrase_sub = phrase.utf8substr(start, i-start);
|
|||
|
|
for (j = 0; j < phrase_sub.length(); j++) {
|
|||
|
|
if (j == phrase_sub.length() - 1)
|
|||
|
|
break;
|
|||
|
|
iutf8string key = phrase_sub.utf8substr(j, phrase_sub.length()-j);
|
|||
|
|
if (WordValid(key.stlstring(), appid) == true) {
|
|||
|
|
vector<string>::iterator iter = bmm_list.begin();
|
|||
|
|
bmm_list.insert(iter, key.stlstring());
|
|||
|
|
i -= key.length() - 1;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (j == phrase_sub.length() - 1) {
|
|||
|
|
vector<string>::iterator iter = bmm_list.begin();
|
|||
|
|
bmm_list.insert(iter, "" + phrase_sub[j]);
|
|||
|
|
}
|
|||
|
|
i -= 1;
|
|||
|
|
}
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
vector<string> FBSegment::segment_part(iutf8string &phrase, uint32_t appid) {
|
|||
|
|
vector<string> fmm_list;
|
|||
|
|
FMM2(phrase, appid, fmm_list); // 正向最大匹配
|
|||
|
|
vector<string> bmm_list;
|
|||
|
|
BMM2(phrase, appid, bmm_list); // 反向最大匹配
|
|||
|
|
//如果正反向分词结果词数不同，则取分词数量较少的那个
|
|||
|
|
if (fmm_list.size() != bmm_list.size()) {
|
|||
|
|
if (fmm_list.size() > bmm_list.size())
|
|||
|
|
return bmm_list;
|
|||
|
|
else return fmm_list;
|
|||
|
|
}
|
|||
|
|
//如果分词结果词数相同
|
|||
|
|
else {
|
|||
|
|
//如果正反向的分词结果相同，就说明没有歧义，可返回任意一个
|
|||
|
|
int i, FSingle = 0, BSingle = 0;
|
|||
|
|
bool isSame = true;
|
|||
|
|
for (i = 0; i < (int)fmm_list.size(); i++) {
|
|||
|
|
if (fmm_list.at(i) != (bmm_list.at(i)))
|
|||
|
|
isSame = false;
|
|||
|
|
if (fmm_list.at(i).length() == 1)
|
|||
|
|
FSingle += 1;
|
|||
|
|
if (bmm_list.at(i).length() == 1)
|
|||
|
|
BSingle += 1;
|
|||
|
|
}
|
|||
|
|
if (isSame)
|
|||
|
|
return fmm_list;
|
|||
|
|
else {
|
|||
|
|
//分词结果不同，返回其中单字较少的那个
|
|||
|
|
if (BSingle > FSingle)
|
|||
|
|
return fmm_list;
|
|||
|
|
else return bmm_list;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::segment2(iutf8string &phrase, uint32_t appid, vector<string> &new_res_all, string mode, bool Hmm_flag) {
|
|||
|
|
vector<string> sen_list;
|
|||
|
|
set<string> special_set; // 记录英文和数字字符串
|
|||
|
|
string tmp_words = "";
|
|||
|
|
bool flag = false; // 记录是否有英文或者数字的flag
|
|||
|
|
for (int i = 0; i < phrase.length(); i++) {
|
|||
|
|
if (isAlphaOrDigit(phrase[i])) {
|
|||
|
|
if (tmp_words != "" and flag == false) {
|
|||
|
|
sen_list.push_back(tmp_words);
|
|||
|
|
tmp_words = "";
|
|||
|
|
}
|
|||
|
|
flag = true;
|
|||
|
|
tmp_words += phrase[i];
|
|||
|
|
}
|
|||
|
|
else if(punct_set.find(phrase[i]) != punct_set.end()){
|
|||
|
|
if (tmp_words != "") {
|
|||
|
|
sen_list.push_back(tmp_words);
|
|||
|
|
sen_list.push_back(phrase[i]);
|
|||
|
|
if (flag == true) {
|
|||
|
|
special_set.insert(tmp_words);
|
|||
|
|
flag = false;
|
|||
|
|
}
|
|||
|
|
tmp_words = "";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
if (flag == true) {
|
|||
|
|
sen_list.push_back(tmp_words);
|
|||
|
|
special_set.insert(tmp_words);
|
|||
|
|
flag = false;
|
|||
|
|
tmp_words = phrase[i];
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
tmp_words += phrase[i];
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (tmp_words != "") {
|
|||
|
|
sen_list.push_back(tmp_words);
|
|||
|
|
if (flag == true) {
|
|||
|
|
special_set.insert(tmp_words);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
tmp_words = "";
|
|||
|
|
vector<string> res_all;
|
|||
|
|
for (int i = 0; i < (int)sen_list.size(); i++) {
|
|||
|
|
// special_set中保存了连续的字母数字串，不需要进行分词
|
|||
|
|
if (special_set.find(sen_list[i]) == special_set.end() && punct_set.find(sen_list[i]) == punct_set.end()) {
|
|||
|
|
iutf8string utf8_str(sen_list[i]);
|
|||
|
|
vector<string> parse_list;
|
|||
|
|
if (mode == "Pre") {
|
|||
|
|
FMM2(utf8_str, appid, parse_list);
|
|||
|
|
}
|
|||
|
|
else if (mode == "Post") {
|
|||
|
|
BMM2(utf8_str, appid, parse_list);
|
|||
|
|
}
|
|||
|
|
else if (mode == "DAG") {
|
|||
|
|
__cut_DAG_NO_HMM(sen_list[i], appid, parse_list);
|
|||
|
|
}
|
|||
|
|
else if (mode == "Cache") { // word dict in cache, not from DTC
|
|||
|
|
BMM(utf8_str, appid, parse_list);
|
|||
|
|
}
|
|||
|
|
else { // PrePostNGram
|
|||
|
|
vector<string> parse_list1;
|
|||
|
|
vector<string> parse_list2;
|
|||
|
|
FMM2(utf8_str, appid, parse_list1);
|
|||
|
|
BMM2(utf8_str, appid, parse_list2);
|
|||
|
|
parse_list1.insert(parse_list1.begin(), "<BEG>");
|
|||
|
|
parse_list1.push_back("<END>");
|
|||
|
|
parse_list2.insert(parse_list2.begin(), "<BEG>");
|
|||
|
|
parse_list2.push_back("<END>");
|
|||
|
|
|
|||
|
|
// CalList1和CalList2分别记录两个句子词序列不同的部分
|
|||
|
|
vector<string> cal_list1;
|
|||
|
|
vector<string> cal_list2;
|
|||
|
|
// pos1和pos2记录两个句子的当前字的位置，cur1和cur2记录两个句子的第几个词
|
|||
|
|
uint32_t pos1 = 0;
|
|||
|
|
uint32_t pos2 = 0;
|
|||
|
|
uint32_t cur1 = 0;
|
|||
|
|
uint32_t cur2 = 0;
|
|||
|
|
while (1) {
|
|||
|
|
if (cur1 == parse_list1.size() && cur2 == parse_list2.size()) {
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
// 如果当前位置一样
|
|||
|
|
if (pos1 == pos2) {
|
|||
|
|
// 当前位置一样，并且词也一样
|
|||
|
|
if (parse_list1[cur1].size() == parse_list2[cur2].size()) {
|
|||
|
|
pos1 += parse_list1[cur1].size();
|
|||
|
|
pos2 += parse_list2[cur2].size();
|
|||
|
|
// 说明此时得到两个不同的词序列，根据bigram选择概率大的
|
|||
|
|
// 注意算不同的时候要考虑加上前面一个词和后面一个词，拼接的时候再去掉即可
|
|||
|
|
if (cal_list1.size() > 0) {
|
|||
|
|
cal_list1.insert(cal_list1.begin(), parse_list[parse_list.size() - 1]);
|
|||
|
|
cal_list2.insert(cal_list2.begin(), parse_list[parse_list.size() - 1]);
|
|||
|
|
if (cur1 < parse_list1.size()-1) {
|
|||
|
|
cal_list1.push_back(parse_list1[cur1]);
|
|||
|
|
cal_list2.push_back(parse_list2[cur2]);
|
|||
|
|
}
|
|||
|
|
double p1 = CalSegProbability(cal_list1);
|
|||
|
|
double p2 = CalSegProbability(cal_list2);
|
|||
|
|
|
|||
|
|
vector<string> cal_list = (p1 > p2) ? cal_list1 : cal_list2;
|
|||
|
|
cal_list.erase(cal_list.begin());
|
|||
|
|
if (cur1 < parse_list1.size() - 1) {
|
|||
|
|
cal_list.pop_back();
|
|||
|
|
}
|
|||
|
|
parse_list.insert(parse_list.end(), cal_list.begin(), cal_list.end());
|
|||
|
|
cal_list1.clear();
|
|||
|
|
cal_list2.clear();
|
|||
|
|
}
|
|||
|
|
parse_list.push_back(parse_list1[cur1]);
|
|||
|
|
cur1++;
|
|||
|
|
cur2++;
|
|||
|
|
}
|
|||
|
|
// pos相同，len(ParseList1[cur1])不同，向后滑动，不同的添加到list中
|
|||
|
|
else if (parse_list1[cur1].size() > parse_list2[cur2].size()) {
|
|||
|
|
cal_list2.push_back(parse_list2[cur2]);
|
|||
|
|
pos2 += parse_list2[cur2].size();
|
|||
|
|
cur2++;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
cal_list1.push_back(parse_list1[cur1]);
|
|||
|
|
pos1 += parse_list1[cur1].size();
|
|||
|
|
cur1++;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
// pos不同，而结束的位置相同，两个同时向后滑动
|
|||
|
|
if (pos1 + parse_list1[cur1].size() == pos2 + parse_list2[cur2].size()) {
|
|||
|
|
cal_list1.push_back(parse_list1[cur1]);
|
|||
|
|
cal_list2.push_back(parse_list2[cur2]);
|
|||
|
|
pos1 += parse_list1[cur1].size();
|
|||
|
|
pos2 += parse_list2[cur2].size();
|
|||
|
|
cur1++;
|
|||
|
|
cur2++;
|
|||
|
|
}
|
|||
|
|
else if (pos1 + parse_list1[cur1].size() > pos2 + parse_list2[cur2].size()) {
|
|||
|
|
cal_list2.push_back(parse_list2[cur2]);
|
|||
|
|
pos2 += parse_list2[cur2].size();
|
|||
|
|
cur2++;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
cal_list1.push_back(parse_list1[cur1]);
|
|||
|
|
pos1 += parse_list1[cur1].size();
|
|||
|
|
cur1++;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
parse_list.erase(parse_list.begin());
|
|||
|
|
parse_list.pop_back();
|
|||
|
|
}
|
|||
|
|
res_all.insert(res_all.end(), parse_list.begin(), parse_list.end());
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
if(punct_set.find(sen_list[i]) == punct_set.end()){
|
|||
|
|
res_all.push_back(sen_list[i]);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (Hmm_flag == false) {
|
|||
|
|
new_res_all.assign(res_all.begin(), res_all.end());
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
// 使用HMM发现新词
|
|||
|
|
string buf = "";
|
|||
|
|
for (size_t i = 0; i < res_all.size(); i++) {
|
|||
|
|
iutf8string utf8_str(res_all[i]);
|
|||
|
|
if (utf8_str.length() == 1 && punct_set.find(res_all[i]) == punct_set.end() && res_all[i].length() > 1) { // 确保res_all[i]是汉字
|
|||
|
|
buf += res_all[i];
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
if (buf.length() > 0) {
|
|||
|
|
iutf8string utf8_buf(buf);
|
|||
|
|
if (utf8_buf.length() == 1) {
|
|||
|
|
new_res_all.push_back(buf);
|
|||
|
|
}
|
|||
|
|
else if (WordValid(buf, appid) == false) { // 连续的单字组合起来，使用HMM算法进行分词
|
|||
|
|
vector<string> vec = HMM_split(buf);
|
|||
|
|
new_res_all.insert(new_res_all.end(), vec.begin(), vec.end());
|
|||
|
|
}
|
|||
|
|
else { // 是否有这种情况
|
|||
|
|
new_res_all.push_back(buf);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
buf = "";
|
|||
|
|
new_res_all.push_back(res_all[i]);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (buf.length() > 0) {
|
|||
|
|
iutf8string utf8_buf(buf);
|
|||
|
|
if (utf8_buf.length() == 1) {
|
|||
|
|
new_res_all.push_back(buf);
|
|||
|
|
}
|
|||
|
|
else if (WordValid(buf, appid) == false) { // 连续的单字组合起来，使用HMM算法进行分词
|
|||
|
|
vector<string> vec = HMM_split(buf);
|
|||
|
|
new_res_all.insert(new_res_all.end(), vec.begin(), vec.end());
|
|||
|
|
}
|
|||
|
|
else { // 是否有这种情况
|
|||
|
|
new_res_all.push_back(buf);
|
|||
|
|
}
|
|||
|
|
buf = "";
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::cut_for_search(iutf8string &phrase, uint32_t appid, vector<vector<string> > &search_res_all, string mode) {
|
|||
|
|
// 搜索引擎模式
|
|||
|
|
vector<string> new_res_all;
|
|||
|
|
segment2(phrase, appid, new_res_all, mode);
|
|||
|
|
for (size_t i = 0; i < new_res_all.size(); i++) {
|
|||
|
|
vector<string> vec;
|
|||
|
|
iutf8string utf8_str(new_res_all[i]);
|
|||
|
|
if (utf8_str.length() > 2 && isAllAlphaOrDigit(new_res_all[i]) == false) {
|
|||
|
|
for (int j = 0; j < utf8_str.length() - 1; j++) {
|
|||
|
|
string tmp_str = utf8_str.substr(j, 2);
|
|||
|
|
if (WordValid(tmp_str, appid) == true) {
|
|||
|
|
vec.push_back(tmp_str);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (utf8_str.length() > 3 && isAllAlphaOrDigit(new_res_all[i]) == false) {
|
|||
|
|
for (int j = 0; j < utf8_str.length() - 2; j++) {
|
|||
|
|
string tmp_str = utf8_str.substr(j, 3);
|
|||
|
|
if (WordValid(tmp_str, appid) == true) {
|
|||
|
|
vec.push_back(tmp_str);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
vec.push_back(new_res_all[i]);
|
|||
|
|
search_res_all.push_back(vec);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::cut_ngram(iutf8string &phrase, vector<string> &search_res, uint32_t n) {
|
|||
|
|
uint32_t N = (n > (uint32_t)phrase.length()) ? (uint32_t)phrase.length() : n;
|
|||
|
|
for (size_t i = 1; i <= N; i++) {
|
|||
|
|
for (size_t j = 0; j < (size_t)phrase.length() - i + 1; j++) {
|
|||
|
|
string tmp_str = phrase.substr(j, i);
|
|||
|
|
search_res.push_back(tmp_str);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
vector<char> FBSegment::viterbi(string sentence) {
|
|||
|
|
iutf8string utf8_str(sentence);
|
|||
|
|
vector< map<char, double> > V;
|
|||
|
|
map<char, vector<char> > path;
|
|||
|
|
char states[4] = { 'B','M','E','S' };
|
|||
|
|
map<char, double> prob_map;
|
|||
|
|
for (size_t i = 0; i < sizeof(states); i++) {
|
|||
|
|
char y = states[i];
|
|||
|
|
double emit_value = train_corpus.MinEmit();
|
|||
|
|
if (train_corpus.emit_dict[y].find(utf8_str[0]) != train_corpus.emit_dict[y].end()) {
|
|||
|
|
emit_value = train_corpus.emit_dict[y].at(utf8_str[0]);
|
|||
|
|
}
|
|||
|
|
prob_map[y] = train_corpus.start_dict[y] * emit_value; // 在位置0，以y状态为末尾的状态序列的最大概率
|
|||
|
|
path[y].push_back(y);
|
|||
|
|
}
|
|||
|
|
V.push_back(prob_map);
|
|||
|
|
for (int j = 1; j < utf8_str.length(); j++) {
|
|||
|
|
map<char, vector<char> > new_path;
|
|||
|
|
prob_map.clear();
|
|||
|
|
for (size_t k = 0; k < sizeof(states); k++) {
|
|||
|
|
char y = states[k];
|
|||
|
|
double max_prob = 0.0;
|
|||
|
|
char state = ' ';
|
|||
|
|
for (size_t m = 0; m < sizeof(states); m++) {
|
|||
|
|
char y0 = states[m]; // 从y0 -> y状态的递归
|
|||
|
|
//cout << j << " " << y0 << " " << y << " " << V[j - 1][y0] << " " << train_corpus.trans_dict[y0][y] << " " << train_corpus.emit_dict[y].at(utf8_str[j]) << endl;
|
|||
|
|
double emit_value = train_corpus.MinEmit();
|
|||
|
|
if (train_corpus.emit_dict[y].find(utf8_str[j]) != train_corpus.emit_dict[y].end()) {
|
|||
|
|
emit_value = train_corpus.emit_dict[y].at(utf8_str[j]);
|
|||
|
|
}
|
|||
|
|
double prob = V[j - 1][y0] * train_corpus.trans_dict[y0][y] * emit_value;
|
|||
|
|
if (prob > max_prob) {
|
|||
|
|
max_prob = prob;
|
|||
|
|
state = y0;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
prob_map[y] = max_prob;
|
|||
|
|
new_path[y] = path[state];
|
|||
|
|
new_path[y].push_back(y);
|
|||
|
|
}
|
|||
|
|
V.push_back(prob_map);
|
|||
|
|
path = new_path;
|
|||
|
|
}
|
|||
|
|
double max_prob = 0.0;
|
|||
|
|
char state = ' ';
|
|||
|
|
for (size_t i = 0; i < sizeof(states); i++) {
|
|||
|
|
char y = states[i];
|
|||
|
|
if (V[utf8_str.length() - 1][y] > max_prob) {
|
|||
|
|
max_prob = V[utf8_str.length() - 1][y];
|
|||
|
|
state = y;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return path[state];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
vector<string> FBSegment::HMM_split(string sentence) {
|
|||
|
|
|
|||
|
|
vector<char> pos_list = viterbi(sentence);
|
|||
|
|
string result;
|
|||
|
|
iutf8string utf8_str(sentence);
|
|||
|
|
for (size_t i = 0; i < pos_list.size(); i++) {
|
|||
|
|
result += utf8_str[i];
|
|||
|
|
if (pos_list[i] == 'E') {
|
|||
|
|
std::size_t found = result.find_last_of(" ");
|
|||
|
|
string new_word = result.substr(found + 1);
|
|||
|
|
//printf("new word: %s\n", new_word.c_str());
|
|||
|
|
}
|
|||
|
|
if (pos_list[i] == 'E' || pos_list[i] == 'S') {
|
|||
|
|
result += ' ';
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (result[result.size()-1] == ' ') {
|
|||
|
|
result = result.substr(0, result.size() - 1);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return splitEx(result, " ");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
double FBSegment::CalSegProbability(const vector<string> &vec) {
|
|||
|
|
double p = 0;
|
|||
|
|
string word1;
|
|||
|
|
string word2;
|
|||
|
|
// 由于概率很小，对连乘做了取对数处理转化为加法
|
|||
|
|
for (int pos = 0; pos < (int)vec.size(); pos++) {
|
|||
|
|
if (pos != (int)vec.size() - 1) {
|
|||
|
|
// 乘以后面词的条件概率
|
|||
|
|
word1 = vec[pos];
|
|||
|
|
word2 = vec[pos + 1];
|
|||
|
|
if (next_dict.find(word1) == next_dict.end()) {
|
|||
|
|
// 加1平滑
|
|||
|
|
p += log(1.0 / train_cnt);
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
double numerator = 1.0;
|
|||
|
|
uint32_t denominator = train_cnt;
|
|||
|
|
map<string, int>::iterator iter = next_dict[word1].begin();
|
|||
|
|
for (; iter != next_dict[word1].end(); iter++) {
|
|||
|
|
if (iter->first == word2) {
|
|||
|
|
numerator += iter->second;
|
|||
|
|
}
|
|||
|
|
denominator += iter->second;
|
|||
|
|
}
|
|||
|
|
p += log(numerator / denominator);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// 乘以第一个词的概率
|
|||
|
|
if ((pos == 0 && vec[pos] != "<BEG>") || (pos == 1 && vec[0] == "<BEG>")) {
|
|||
|
|
uint32_t word_freq = 0;
|
|||
|
|
WordInfo word_info;
|
|||
|
|
if (GetWordInfo(vec[pos], 0, word_info)) {
|
|||
|
|
word_freq = word_info.word_freq;
|
|||
|
|
p += log(word_freq + 1.0 / next_dict.size() + train_cnt);
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
p += log(1.0 / next_dict.size() + train_cnt);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return p;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
vector<string> FBSegment::segment(iutf8string &phrase, uint32_t appid) {
|
|||
|
|
vector<string> res;
|
|||
|
|
int last = 0;
|
|||
|
|
for (int i = 0; i < phrase.length(); i++) {
|
|||
|
|
if (punct_set.find(phrase[i]) != punct_set.end()) {
|
|||
|
|
iutf8string fragment = phrase.utf8substr(last, i - last);
|
|||
|
|
for (int j = 0; j < fragment.length(); ) { // 继续拆分，将连续的中文或者连续的英文字母合并
|
|||
|
|
if (fragment[j].length() == 1) {
|
|||
|
|
string tmp = fragment[j];
|
|||
|
|
int k = 1;
|
|||
|
|
for (; k < fragment.length() - j; k++) {
|
|||
|
|
if (fragment[j + k].size() > 1) { // 连续英文，遇到非英文字母停止
|
|||
|
|
res.push_back(tmp);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
else { // 非英文或数字，断开
|
|||
|
|
char frag_char = fragment[j + k][0];
|
|||
|
|
if (!isupper(frag_char) && !islower(frag_char) && !isdigit(frag_char)) {
|
|||
|
|
res.push_back(tmp);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
tmp += fragment[j + k];
|
|||
|
|
}
|
|||
|
|
if (k == fragment.length() - j) { // 如果循环完没有break，则插入到res
|
|||
|
|
res.push_back(tmp);
|
|||
|
|
}
|
|||
|
|
j = j + k;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
string tmp = fragment[j];
|
|||
|
|
int k = 1;
|
|||
|
|
for (; k < fragment.length() - j; k++) {
|
|||
|
|
if (fragment[j + k].size() == 1) { // 连续中文，遇到非中文字母停止
|
|||
|
|
res.push_back(tmp);
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
tmp += fragment[j + k];
|
|||
|
|
}
|
|||
|
|
if (k == fragment.length() - j) {
|
|||
|
|
res.push_back(tmp);
|
|||
|
|
}
|
|||
|
|
j = j + k;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
last = i + 1;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (last < phrase.length()) {
|
|||
|
|
string fragment = phrase.substr(last, phrase.length() - last);
|
|||
|
|
res.push_back(fragment);
|
|||
|
|
}
|
|||
|
|
vector<string> res_all;
|
|||
|
|
vector<string>::iterator iter = res.begin();
|
|||
|
|
for (; iter != res.end(); iter++) {
|
|||
|
|
vector<string> res;
|
|||
|
|
string str = *iter;
|
|||
|
|
if (isAllAlphaOrDigit(str)) { // 英文不进行分词
|
|||
|
|
res_all.push_back(str);
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
iutf8string utf8_str(*iter);
|
|||
|
|
res = segment_part(utf8_str, appid);
|
|||
|
|
res_all.insert(res_all.end(), res.begin(), res.end());
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return res_all;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::get_DAG(string sentence, uint32_t appid, map<uint32_t, vector<uint32_t> > &DAG) {
|
|||
|
|
iutf8string utf8_str(sentence);
|
|||
|
|
uint32_t N = utf8_str.length();
|
|||
|
|
for (uint32_t k = 0; k < N; k++) {
|
|||
|
|
uint32_t i = k;
|
|||
|
|
vector<uint32_t> tmplist;
|
|||
|
|
string frag = utf8_str[k];
|
|||
|
|
while (i < N) {
|
|||
|
|
if (WordValid(frag, appid) == true) {
|
|||
|
|
tmplist.push_back(i);
|
|||
|
|
}
|
|||
|
|
i++;
|
|||
|
|
frag = utf8_str.substr(k, i + 1 - k);
|
|||
|
|
}
|
|||
|
|
if (tmplist.empty()) {
|
|||
|
|
tmplist.push_back(k);
|
|||
|
|
}
|
|||
|
|
DAG[k] = tmplist;
|
|||
|
|
}
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::calc(string sentence, const map<uint32_t, vector<uint32_t> > &DAG, map<uint32_t, RouteValue> &route, uint32_t appid) {
|
|||
|
|
iutf8string utf8_str(sentence);
|
|||
|
|
uint32_t N = utf8_str.length();
|
|||
|
|
RouteValue route_N;
|
|||
|
|
route[N] = route_N;
|
|||
|
|
double logtotal = log(TOTAL);
|
|||
|
|
for (int i = N - 1; i > -1; i--) {
|
|||
|
|
vector<uint32_t> vec = DAG.at(i);
|
|||
|
|
double max_route = -DBL_MAX;
|
|||
|
|
uint32_t max_idx = 0;
|
|||
|
|
for (size_t t = 0; t < vec.size(); t++) {
|
|||
|
|
string word = utf8_str.substr(i, vec[t] + 1 - i);
|
|||
|
|
WordInfo word_info;
|
|||
|
|
uint32_t word_freq = 1;
|
|||
|
|
/* 不查DTC，改为从本地词库查询
|
|||
|
|
if (word_manager.WordValid(word, appid, word_info) == true) {
|
|||
|
|
word_freq = word_info.word_freq;
|
|||
|
|
}
|
|||
|
|
*/
|
|||
|
|
if (word_dict.find(word) != word_dict.end()) {
|
|||
|
|
map<uint32_t, WordInfo> wordInfo = word_dict[word];
|
|||
|
|
if (wordInfo.find(0) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[0];
|
|||
|
|
word_freq = word_info.word_freq;
|
|||
|
|
}
|
|||
|
|
if (wordInfo.find(appid) != wordInfo.end()) {
|
|||
|
|
word_info = wordInfo[appid];
|
|||
|
|
word_freq = word_info.word_freq;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
double route_value = log(word_freq) - logtotal + route[vec[t] + 1].max_route;
|
|||
|
|
if (route_value > max_route) {
|
|||
|
|
max_route = route_value;
|
|||
|
|
max_idx = vec[t];
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
RouteValue route_value;
|
|||
|
|
route_value.max_route = max_route;
|
|||
|
|
route_value.idx = max_idx;
|
|||
|
|
route[i] = route_value;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void FBSegment::__cut_DAG_NO_HMM(string sentence, uint32_t appid, vector<string> &vec) {
|
|||
|
|
map<uint32_t, vector<uint32_t> > DAG;
|
|||
|
|
get_DAG(sentence, appid, DAG);
|
|||
|
|
map<uint32_t, RouteValue> route;
|
|||
|
|
calc(sentence, DAG, route, appid);
|
|||
|
|
iutf8string utf8_str(sentence);
|
|||
|
|
uint32_t N = utf8_str.length();
|
|||
|
|
uint32_t i = 0;
|
|||
|
|
string buf = "";
|
|||
|
|
while (i < N) {
|
|||
|
|
uint32_t j = route[i].idx + 1;
|
|||
|
|
string l_word = utf8_str.substr(i, j - i);
|
|||
|
|
if (isAllAlphaOrDigit(l_word)) {
|
|||
|
|
buf += l_word;
|
|||
|
|
i = j;
|
|||
|
|
}
|
|||
|
|
else {
|
|||
|
|
if (!buf.empty()) {
|
|||
|
|
vec.push_back(buf);
|
|||
|
|
buf = "";
|
|||
|
|
}
|
|||
|
|
vec.push_back(l_word);
|
|||
|
|
i = j;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (!buf.empty()) {
|
|||
|
|
vec.push_back(buf);
|
|||
|
|
buf = "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return;
|
|||
|
|
}
|