diff --git a/src/search_local/index_storage/Make.conf b/src/search_local/index_storage/Make.conf new file mode 100644 index 0000000..0e769d1 --- /dev/null +++ b/src/search_local/index_storage/Make.conf @@ -0,0 +1,41 @@ +#for memchecker only, developer-only +ifeq ($(LIB_PATH),) +LIB_PATH = ../.. +endif +ifeq ($(MEMCHECK),1) +CFLAGS += -DMEMCHECK=1 +endif + +#for gprof only +ifeq ($(GPROF),1) +CFLAGS += -pg +LDFLAGS += -pg +endif + +# slk10 don't accept: -fvisibility=hidden +CFLAGS += -g -fno-strict-aliasing -fpermissive + +GCCVER := $(shell $(CC) -dumpversion | awk -F. '{ print $$1"."$$2}') +PLATFORM := $(shell $(CC) -dumpmachine) + +ifeq ($(findstring x86_64,$(PLATFORM)),) +CFLAGS += -march=pentiumpro +endif +LD := $(CXX) + +ifeq ($(findstring x86_64, $(PLATFORM)),) + ZINC := -I$(LIB_PATH)/3rdlib/zlib/include32/ + Z_LIB := $(LIB_PATH)/3rdlib/zlib/lib/libz32.a + DTC_API_LIB = ../3rdlib/dtc_api/lib32/libdtc-gcc-4.4-r66b566b.so + CA_API_LIB = $(LIB_PATH)/3rdlib/CA_API/libapp-client.a +else + ZINC := -I$(LIB_PATH)/3rdlib/zlib/include64/ + Z_LIB := $(LIB_PATH)/3rdlib/zlib/lib/libz64.a + DTC_API_LIB = ../3rdlib/dtc_api/lib64/libdtc-gcc-4.4-r66b566b.so + CA_API_LIB = $(LIB_PATH)/3rdlib/CA_API/libapp-client.a +endif + +#custom-helper +CUSTOMHLP=1 + +all:: diff --git a/src/search_local/index_storage/Make.rules b/src/search_local/index_storage/Make.rules new file mode 100644 index 0000000..7de52cf --- /dev/null +++ b/src/search_local/index_storage/Make.rules @@ -0,0 +1,391 @@ +.DEFAULT_GOAL := all +.PHONY: all clean install count FORCE + +export MAKEFLAGS +MAKEFLAGS += --no-print-directory + +ifneq (,$(DIRNAME)) +target_external := +endif + +I=$(if $(filter install,$(MAKECMDGOALS)),1,0) +ifeq (,$(MAKEFILE_LIST)) +MAKEFILE_LIST := Makefile +endif + +_INDEX := 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 +EXTDIRS := $(foreach p,$(target_external),$(if $(wildcard $p),,$(dir $p))) +EXTIDXS := $(addprefix make_extdir_,$(wordlist 1,$(words $(EXTDIRS)),$(_INDEX))) +all:: $(addprefix make_all_,$(SUBDIRS)) $(EXTIDXS) $(target) $(target_only); +install:: $(addprefix make_all_,$(SUBDIRS)) $(EXTIDXS) $(target) $(target_only); +ifeq ($I,1) +all:: $(addprefix install_,$(target_install)); +install:: $(addprefix install_,$(target_install)); +endif +FORCE:; +clean:: $(addprefix make_clean_,$(SUBDIRS)); + +#override CFLAGS += -Wall -D_GNU_SOURCE -D_REENTRANT -g -O2 +override CFLAGS += -Wall -D_GNU_SOURCE -D_REENTRANT -g + +ifeq ($(DEBUG), ) +override CFLAGS += -O2 +endif + +ECHO = /bin/echo + +ifneq (0,$C) +RED = \\e[1m\\e[31m +DARKRED = \\e[31m +GREEN = \\e[1m\\e[32m +DARKGREEN = \\e[32m +BLUE = \\e[1m\\e[34m +DARKBLUE = \\e[34m +YELLOW = \\e[1m\\e[33m +DARKYELLOW = \\e[33m +MAGENTA = \\e[1m\\e[35m +DARKMAGENTA = \\e[35m +CYAN = \\e[1m\\e[36m +DARKCYAN = \\e[36m +RESET = \\e[m + +color: + @$(ECHO) -e $(RED)RED$(RESET) + @$(ECHO) -e $(GREEN)GREEN$(RESET) + @$(ECHO) -e $(BLUE)BLUE$(RESET) + @$(ECHO) -e $(YELLOW)YELLOW$(RESET) + @$(ECHO) -e $(MAGENTA)MAGENTA$(RESET) + @$(ECHO) -e $(CYAN)CYAN $(RESET) + @$(ECHO) -e $(DARKRED)DARKRED$(RESET) + @$(ECHO) -e $(DARKGREEN)DARKGREEN$(RESET) + @$(ECHO) -e $(DARKBLUE)DARKBLUE$(RESET) + @$(ECHO) -e $(DARKYELLOW)DARKYELLOW$(RESET) + @$(ECHO) -e $(DARKMAGENTA)DARKMAGENTA$(RESET) + @$(ECHO) -e $(DARKCYAN)DARKCYAN $(RESET) +endif + +export P + +PSX = $(if $(findstring -j,-$(MAKEFLAGS)),$(GREEN)$(EPREFIX) ) + +ifneq (,$(findstring clean,$(MAKECMDGOALS))) +override P := 0 +EXTIDXS := +endif + +ifeq (0,$P) +_TOTAL := +_TOTAL2 := +override TOTAL := +else +_TOTAL := $(foreach p,$(basename $(filter-out %.so,$(target))),$($p_objs)) +_TOTAL2 := $(foreach p,$(basename $(filter %.so,$(target))),$($p_objs:.o=.pic.o)) +_TOTAL2 := $(words $(sort $(target) $(_TOTAL) $(_TOTAL2))) +_TOTAL := $(SUBDIRS) +endif + +$(foreach p,$(_TOTAL),$(eval $(shell $(MAKE) -s -C $p count DIRNAME=$p P=$P))) +$(foreach p,$(EXTIDXS),$(eval $(shell $(MAKE) -s -C $(word $(subst make_extdir_,,$p),$(EXTDIRS)) count DIRNAME=$p P=$P))) +_TOTAL += $(EXTIDXS) +_TOTAL := $(if $(_TOTAL),$(words $(_TOTAL))*2 $(patsubst %,+$$(TOTAL_%),$(sort $(_TOTAL)))) +$(if $(_TOTAL),$(eval _TOTAL2:=$$(shell $(ECHO) $$$$(($(_TOTAL2)+$(_TOTAL)))))) +$(if $(TOTAL),,$(eval TOTAL:=$(_TOTAL2))) + +ifeq (0,$(TOTAL)) +TOTAL := +endif + +ifeq (,$(TOTAL)) +override PCT := +override PCT0 := +override PCT1 := +override inc := +else +ifneq (,$(findstring -j,-$(MAKEFLAGS))) +override PCT := +override PCT0 := +override PCT1 := +override inc := +else +ifeq ($(START),) +STEP := 0 00 +else +STEP := $(START) $(shell B=$$(($(START)*100/$(TOTAL))); [ $$B -le 9 ] && B=0$$B; $(ECHO) $$B) +endif +START = $(firstword $(STEP)) +PCT0 = $(if $(findstring -j,-$(MAKEFLAGS)),," ") +ifeq (2,$P) +PCT1 = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(TOTAL)/$(TOTAL)]) +PCT = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(firstword $(STEP))/$(TOTAL)]) +else +PCT1 = $(if $(findstring -j,-$(MAKEFLAGS)),,[100]) +PCT = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(word 2,$(STEP))%]) +endif +define inc +$(if $(findstring -j,-$(MAKEFLAGS)),,STEP := $(shell A=$$(($(firstword $(STEP))+$(if $1,$1,1))); B=$$(($$A*995/$(TOTAL)/10)); [ $$B -le 9 ] && B=0$$B; $(ECHO) $$A $$B)) +endef +ifeq (0,$P) +inc := 0 00 +endif +endif +endif + +ifeq (default,$(origin LD)) +LD := $(CC) +endif +$(eval LD := $(CXX) -Wl,--as-needed) +$(eval MMD = -MMD -MF .dep.$$@) + +define CC +@$(ECHO) -e $(PCT) $(eval $(inc)) " CC "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET) +@gcc +endef + +define CCSO +@$(ECHO) -e $(PCT) $(eval $(inc)) " CC [PIC] "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET) +@gcc -fPIC +endef + +define CXX +@$(ECHO) -e $(PCT) $(eval $(inc)) " CXX "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET) +@g++ +endef + +define CXXSO +@$(ECHO) -e $(PCT) $(eval $(inc)) " CXX [PIC] "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET) +@g++ -fPIC +endef + +define MLD +@$(ECHO) -e $(PCT) $(eval $(inc)) " LD "$(PSX)$(MAGENTA)$@ $(RESET) +@$(LD) +endef + +define MLDSO +@$(ECHO) -e $(PCT) $(eval $(inc)) " LD [PIC] "$(PSX)$(MAGENTA)$@ $(RESET) +@$(LD) -shared +endef + +define AR +@$(ECHO) -e $(PCT) $(eval $(inc)) " AR "$(PSX)$(MAGENTA)$@ $(RESET) +@ar +endef + +define BIN2C +@$(ECHO) -e $(PCT) $(eval $(inc)) " BIN2C "$(PSX)$(YELLOW)$@ $(RESET) +@xxd -i +endef + +ENTER = @$(ECHO) -e $(PCT) $(eval $(inc)) " ENTER ==> "$(GREEN) +LEAVE = @$(ECHO) -e $(PCT) $(eval $(inc)) " LEAVE <== "$(GREEN) + +define RM +@$(ECHO) -e $(PCT0) " CLEAN "$(PSX)$(MAGENTA)$(if $(CLEAN),$(CLEAN),"<$(notdir $(CURDIR))>") $(RESET) +@rm -f +endef + +define INST +@$(ECHO) -e $(PCT0) " INSTALL "$(PSX)$(YELLOW)$* $(RESET) +@install +endef + +DONE = @$(ECHO) -e $(PCT1) " DONE "$(MAGENTA)"<$(notdir $(CURDIR))>" $(RESET) +BAD = ($(ECHO) -e $(PCT0) " BAD "$(PSX)$(RED)$(1) $(RESET); /bin/false) +BAD = ($(ECHO) -e $(PCT0) " BAD "$(PSX)$(RED)$(1) $(RESET); /bin/false) +MDEP = $(ECHO) -e $(PCT0) " MAKEDEP "$(YELLOW)$(EPREFIX)$(firstword $(MAKEFILE_LIST)) $(RESET) 1>&2 + +tmp.%.c: %.c + @ln -s $< $@ + +tmp.%.cc: %.cc + @ln -s $< $@ + +tmp.%.cpp: %.cpp + @ln -s $< $@ + +%.pic.o: %.c + $(CCSO) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $< + +%_pic.o: %.c + $(CCSO) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $< + +%.o: %.c + $(CC) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $< + +%.pic.o: %.cc + $(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%_pic.o: %.cc + $(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%.o: %.cc + $(CXX) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%.pic.o: %.cpp + $(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%_pic.o: %.cpp + $(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%.o: %.cpp + $(CXX) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $< + +%.bin.c: % + $(BIN2C) $< >$@ + +make_all_%:: + $(ENTER) $(EPREFIX)$* $(RESET) + @$(MAKE) -s -C $* all DONE= I=$I P=$P TOTAL=$(TOTAL) START=$(START) EPREFIX=$(EPREFIX)$*/ + @$(eval $(call inc,$(TOTAL_$(strip $*)))) + $(LEAVE) $(EPREFIX)$* $(RESET) + +make_extdir_%:: + $(ENTER) $(word $*,$(EXTDIRS)) $(RESET) + @$(MAKE) -s -C $(word $*,$(EXTDIRS)) all DONE= P=$P TOTAL=$(TOTAL) START=$(START) + @$(eval $(call inc,$(TOTAL_make_extdir_$(strip $*)))) + $(LEAVE) $(word $*,$(EXTDIRS)) $(RESET) + +make_clean_%: + @$(MAKE) -s -C $* clean CLEAN=$(if $(CLEAN),$(CLEAN)/)$* + +install_%.so: %.so + @[ -f $(install_dir) ] || mkdir -p $(install_dir) + $(INST) $(if $($*_soname),$($*_soname),$*.so) $(install_dir) + $(if $($*_soname),@ln -sf $($*_soname) $(install_dir)/$*.so) + +install_%: % + @[ -f $(install_dir) ] || mkdir -p $(install_dir) + $(INST) $* $(if $(install_mode),-m $(install_mode),) $(install_dir) + +%: %.c +%: %.cc +%: %.cpp +%: %.o + +_COMMA=, +_method := +ifneq (,$(findstring second-expansion,$(.FEATURES))) +# for gmake v3.81 or above, second-expansion method +.SECONDEXPANSION: +_SUB=:.o=.pic.o +%.pic.a: $$(%_objs$$(_SUB)) + $(AR) cr $@ $^ + +%.so: $$(%_objs$$(_SUB)) +# $(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname),$($*_soname) -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS) + $(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname),$(if $($*_filename), $($*_filename), $($*_soname)) -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS) + $(if $($*_soname),@ln -sf $($*_soname) $@) + $(if $($*_soname),$(if $($*_filename), @ln -sf $($*_filename) $($*_soname))) + +# libdtc.so: $$(%_objs$$(_SUB)) +# $(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname), $filename -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS) +# $(if $($*_soname),@ln -sf $(filename) libdtc.so.1) +# $(if $($*_soname),@ln -sf libdtc.so.1 $@) + +%.a: $$(%_objs) + $(AR) cr $@ $^ + +%: $$(%_objs) + @test -n "$^" || $(call BAD,$@) + $(MLD) $(LDFLAGS) -o $@ $^ $($*_libs) $(LIBPATH) $(LIBS) + +_method := second-expansion +endif + +$(if $(_method),,$(if $(findstring i486,$(shell gcc -dumpmachine)),,$(eval _method := eval))) + +ifeq (eval,$(_method)) +# for gmake v3.80, $(eval) method +define rule-piclib +$p: $($(p:.pic.a=)_objs:.o=.pic.o) + $$(AR) cr $p $$^ +endef + +define rule-lib +$p: $($(p:.a=)_objs) + $$(AR) cr $p $$^ +endef + +define rule-so +$p: $($(p:.so=)_objs:.o=.pic.o) + $$(MLDSO) -z defs $$(LDFLAGS) -o $(if $($(p:.so=)_soname),$($(p:.so=)_soname) -Wl$(_COMMA)-soname=$($(p:.so=)_soname),$p) $$^ $$($(p:.so=)_libs) $$(LIBPATH) $$(LIBS) + $(if $($(p:.so=)_soname),@ln -s $($(p:.so=)_soname) $p) +endef + +define rule-bin +$p: $($p_objs) + $$(MLD) $$(LDFLAGS) -o $p $$($p_objs) $$($p_libs) $$(LIBPATH) $$(LIBS) +endef + +endif +# gmake 3.80 don't allow ($eval) inside if/endif block +$(foreach p,$(filter %.pic.a,$(target)),$(if $($(p:.pic.a=)_objs),$(eval $(rule-piclib)))) +$(foreach p,$(filter %.a,$(target)),$(if $($(p:.a=)_objs),$(eval $(rule-lib)))) +$(foreach p,$(filter %.so,$(target)),$(if $($(p:.so=)_objs),$(eval $(rule-so)))) +$(foreach p,$(filter-out %.a %.o %.so,$(target)),$(if $($p_objs),$(eval $(rule-bin)))) + +ifneq (clean,$(MAKECMDGOALS)) +ifneq (count,$(MAKECMDGOALS)) +ifeq (,$(_method)) +# without $(eval), fallback to temporary Makefile +_method := makefile + +define cmd-piclib +$(ECHO) '$p: $($(p:.pic.a=)_objs:.o=.pic.o)' >>.make.dep; \ +$(ECHO) ' $$(AR) cr $p $$^' >>.make.dep +endef + +define cmd-lib +$(ECHO) '$p: $($(p:.a=)_objs)' >>.make.dep; \ +$(ECHO) ' $$(AR) cr $p $$^' >>.make.dep +endef + +define cmd-so +$(ECHO) '$p: $($(p:.so=)_objs:.o=.pic.o)' >>.make.dep; \ +$(ECHO) ' $$(MLDSO) -z defs $$(LDFLAGS) -o $(if $($(p:.so=)_soname),$($(p:.so=)_soname) -Wl$(_COMMA)-soname=$($(p:.so=)_soname),$p) $$^ $$($(p:.so=)_libs) $$(LIBPATH) $$(LIBS)' >>.make.dep; \ +$(ECHO) ' $(if $($(p:.so=)_soname),@ln -s $($(p:.so=)_soname) $p)' >>.make.dep +endef + + +define cmd-bin +$(ECHO) '$p: $($p_objs)' >>.make.dep; \ +$(ECHO) ' $$(MLD) $$(LDFLAGS) -o $p $$($p_objs) $$($p_libs) $$(LIBPATH) $$(LIBS)' >>.make.dep +endef + +ifeq (OK,$(shell [ $(firstword $(MAKEFILE_LIST)) -nt .make.dep ] && $(ECHO) OK)) +$(shell $(MDEP)) +$(shell >.make.dep) +$(foreach p,$(filter %.pic.a,$(target)),$(if $($(p:.pic.a=)_objs),$(shell $(cmd-piclib)))) +$(foreach p,$(filter %.a,$(target)),$(if $($(p:.a=)_objs),$(shell $(cmd-lib)))) +$(foreach p,$(filter %.so,$(target)),$(if $($(p:.so=)_objs),$(shell $(cmd-so)))) +$(foreach p,$(filter-out %.a %.o %.so,$(target)),$(if $($p_objs),$(shell $(cmd-bin)))) +endif +include .make.dep + +endif +endif +endif + + +.SECONDARY: + +all:: + $(DONE) + +install:: all; + +clean:: + $(foreach p,$(filter %.so,$(target)),@-rm -f $($(p:.so=)_soname)) +clean:: + $(RM) .make.* *.o *.a *.d *.so .dep.* tmp.* core core.* *.log $(target) $(TSTS) + +count:: + @$(ECHO) TOTAL_$(if $(DIRNAME),$(DIRNAME),all)=$(TOTAL) + +method:: + @$(ECHO) rule method is $(_method). + +ifneq ($(wildcard .dep.*),) +-include $(wildcard .dep.*) +endif + diff --git a/src/search_local/index_storage/Makefile b/src/search_local/index_storage/Makefile new file mode 100644 index 0000000..e5d5f12 --- /dev/null +++ b/src/search_local/index_storage/Makefile @@ -0,0 +1,12 @@ +include Make.conf + +SUBDIRS := common api stat watchdog cache rocksdb_helper + +make_all_stat:: make_all_common + +make_all_cache:: make_all_common make_all_api make_all_stat make_all_watchdog + +make_all_proxy:: make_all_common make_all_stat + +include Make.rules + diff --git a/src/search_local/index_storage/api/Makefile b/src/search_local/index_storage/api/Makefile new file mode 100644 index 0000000..6a3d478 --- /dev/null +++ b/src/search_local/index_storage/api/Makefile @@ -0,0 +1,5 @@ +include ../Make.conf + +SUBDIRS := c_api c_api_cc + +include ../Make.rules diff --git a/src/search_local/index_storage/api/c_api/Makefile b/src/search_local/index_storage/api/c_api/Makefile new file mode 100644 index 0000000..607d5fe --- /dev/null +++ b/src/search_local/index_storage/api/c_api/Makefile @@ -0,0 +1,39 @@ +LIB_PATH = ../../../.. + +include ../../Make.conf + +SVN_REVISION = $(shell test -d .svn && (svn info | grep "Last Changed Rev" | cut -d " " -f 4)) +ifeq "$(SVN_REVISION)a" "a" + SVN_REVISION = "(unknown)" +endif + +VPATH = ../../common +################compile############# +target = libdtc.a container_api.pic.o version.pic.o somain.pic.o libdtc.so libdtc.pic.a + +# CLIENTAPI macro use for scope test only +CFLAGS += -DCLIENTAPI +CFLAGS += -pthread -I../../common -I../../stat $(ZINC) +LIBS = $(Z_LIB) -ldl -lpthread + + +filelist := dtc_req dtc_srv dtc_pool dtc_wrap dtc_wrapp \ + poller timer_list key_list table_def \ + log_client mem_check md5 value \ + section decode encode field_api \ + packet_base packet_client \ + task_base task_const net_addr udp_pool compress buffer thread + +# for auto ln -sf +libdtc_objs := $(patsubst %,%.o,$(filelist)) +libdtc.so: LDFLAGS += -Wl,--version-script,dtc_api.lst -e _so_start container_api.pic.o version.pic.o somain.pic.o +libdtc_filename := libdtc-gcc-$(GCCVER)-r$(SVN_REVISION).so + +###############install############## +target_install = libdtc.a libdtc.pic.a libdtc.so dtc_api.h +install_dir = ../../../bin +%.a: install_dir = ../../../lib +%.h: install_dir = ../../../include + +include ../../Make.rules + diff --git a/src/search_local/index_storage/api/c_api/container_api.cc b/src/search_local/index_storage/api/c_api/container_api.cc new file mode 100644 index 0000000..d9af54f --- /dev/null +++ b/src/search_local/index_storage/api/c_api/container_api.cc @@ -0,0 +1,123 @@ +/* + * ===================================================================================== + * + * Filename: container_api.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "container.h" +#include "version.h" +#include "dtc_int.h" + +typedef IInternalService *(*QueryInternalServiceFunctionType)(const char *name, const char *instance); + +IInternalService *query_internal_service(const char *name, const char *instance) +{ + QueryInternalServiceFunctionType entry = NULL; + entry = (QueryInternalServiceFunctionType)dlsym(RTLD_DEFAULT, "_QueryInternalService"); + if (entry == NULL) + return NULL; + return entry(name, instance); +} + +static inline int field_type_to_key_type(int t) +{ + switch (t) + { + case DField::Signed: + case DField::Unsigned: + return DField::Signed; + + case DField::String: + case DField::Binary: + return DField::String; + } + + return DField::None; +} + +void NCServer::check_internal_service(void) +{ + if (NCResultInternal::verify_class() == 0) + return; + + IInternalService *inst = query_internal_service("dtcd", this->tablename); + + // not inside dtcd or tablename not found + if (inst == NULL) + return; + + // version mismatch, internal service is unusable + const char *version = inst->query_version_string(); + if (version == NULL || strcasecmp(version_detail, version) != 0) + return; + + // cast to DTC service + IDTCService *inst1 = static_cast(inst); + + DTCTableDefinition *tdef = inst1->query_table_definition(); + + // verify tablename etc + if (tdef->is_same_table(tablename) == 0) + return; + + // verify and save key type + int kt = field_type_to_key_type(tdef->key_type()); + if (kt == DField::None) + // bad key type + return; + + if (keytype == DField::None) + { + keytype = kt; + } + else if (keytype != kt) + { + badkey = 1; + errstr = "Key Type Mismatch"; + return; + } + + if (keyinfo.key_fields() != 0) + { + // FIXME: checking keyinfo + + // ZAP key info, use ordered name from server + keyinfo.Clear(); + } + // add NCKeyInfo + for (int i = 0; i < tdef->key_fields(); i++) + { + kt = field_type_to_key_type(tdef->field_type(i)); + if (kt == DField::None) + // bad key type + return; + keyinfo.add_key(tdef->field_name(i), kt); + } + + // OK, save it. + // old tdef always NULL, because tablename didn't set, Server didn't complete + this->tdef = tdef; + this->admin_tdef = inst1->query_admin_table_definition(); + // useless here, internal DTCTableDefinition don't maintent a usage count + tdef->INC(); + this->iservice = inst1; + this->completed = 1; + if (get_address() && iservice->match_listening_ports(get_address(), NULL)) + { + executor = iservice->query_task_executor(); + } +} diff --git a/src/search_local/index_storage/api/c_api/dtc_api.h b/src/search_local/index_storage/api/c_api/dtc_api.h new file mode 100644 index 0000000..28c0192 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_api.h @@ -0,0 +1,974 @@ +/* + * ===================================================================================== + * + * Filename: dtc_api.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DistributedTableCache +{ + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int change_node_address = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + const int ClearCache = 21; + const int MigrateDB = 22; + const int MigrateDBSwitch = 23; + const int ColExpandStatus = 24; + const int col_expand = 25; + const int ColExpandDone = 26; + const int ColExpandKey = 27; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log(const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) +#ifndef WIN32 + void write_log(int level, + const char *file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf, 5, 6))); +#endif + + class Result; + class Server + { + private: + void *addr; + long check; + + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void clone_tab_def(const Server &source); + int set_address(const char *host, const char *port = 0); + int set_table_name(const char *); + //for compress + void set_compress_level(int); + //get address and tablename set by user + const char *get_address(void) const; + const char *get_table_name(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char *get_server_address(void) const; + const char *get_server_table_name(void) const; + int int_key(void); + int binary_key(void); + int string_key(void); + int add_key(const char *name, int type); + int field_type(const char *name); + const char *error_message(void) const; + void set_timeout(int); + void set_m_timeout(int); + int Connect(void); + void Close(void); + int ping(void); + void auto_ping(void); + void SetFD(int); // UNSUPPORTED API + void set_auto_update_tab(bool autoUpdate); + void set_auto_reconnect(int autoReconnect); + int decode_packet(Result &, const char *, int); + int check_packet_size(const char *, int); + + void set_access_key(const char *token); + }; + + class Request + { + private: + void *addr; + long check; + Request(const Request &); + + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int attach_server(Server *srv); + + void set_admin_code(int code); + void set_hot_backup_id(long long); + void set_master_hb_timestamp(long long); + void set_slave_hb_timestamp(long long); + +#define _REDIR_(op, t) \ + int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) \ + int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int field_type(const char *); + void no_cache(void); + void no_next_server(void); + void limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int compress_set(const char *, const char *, int); + //just compress and set. Don't need compressflag + int compress_set_force(const char *, const char *, int); + + //bits op + int set_multi_bits(const char *, int, int, unsigned int); + int set_bit(const char *f, int o) { return set_multi_bits(f, o, 1, 1); } + int clear_bit(const char *f, int o) { return set_multi_bits(f, o, 1, 0); } + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void unset_key(void); + int set_key(long long); + int set_key(const char *); + int set_key(const char *, int); +#define _REDIR_(t) \ + int set_key(t a) { return set_key((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int add_key_value(const char *name, long long v); + int add_key_value(const char *name, const char *str); + int add_key_value(const char *name, const char *ptr, int len); +#define _REDIR_(t) \ + int add_key_value(const char *name, t a) { return add_key_value(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *execute(void); + Result *execute(long long); + Result *execute(const char *); + Result *execute(const char *, int); + +#define _REDIR_(t) \ + Result *execute(t a) { return execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int execute(Result &); + int execute(Result &, long long); + int execute(Result &, const char *); + int execute(Result &, const char *, int); + +#define _REDIR_(t) \ + int execute(Result &r, t a) { return execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int encode_packet(char *&, int &, long long &); + int encode_packet(char *&, int &, long long &, long long); + int encode_packet(char *&, int &, long long &, const char *); + int encode_packet(char *&, int &, long long &, const char *, int); + +#define _REDIR_(t) \ + int encode_packet(char *&p, int &l, long long &m, t a) { return encode_packet(p, l, m, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int set_cache_id(long long); +#define _REDIR_(t) \ + int set_cache_id(t a) { return set_cache_id((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *error_message(void) const; + + //无源模式超时时间 + int set_expire_time(const char *key, int time); + int get_expire_time(const char *key); + }; + + class GetRequest : public Request + { + public: + GetRequest(Server *srv) : Request(srv, RequestGet) {} + GetRequest() : Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request + { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request + { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request + { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request + { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest : public Request + { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request + { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest : public Request + { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest : public Request + { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest : public Request + { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor) {} + }; + + class Result + { + private: + void *addr; + long check; + Result(const Result &); + char *server_info() const; + + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void set_error(int errcode, const char *from, const char *detail); // from will not dupped + int result_code(void) const; + const char *error_message(void) const; + const char *error_from(void) const; + long long hot_backup_id() const; + long long master_hb_timestamp() const; + long long slave_hb_timestamp() const; + long long binlog_id() const; + long long binlog_offset() const; + long long mem_size() const; + long long data_size() const; + int num_rows(void) const; + int total_rows(void) const; + int affected_rows(void) const; + int num_fields(void) const; + const char *field_name(int n) const; + int field_present(const char *name) const; + int field_type(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long server_timestamp(void) const; + long long insert_id(void) const; + long long int_key(void) const; + const char *binary_key(void) const; + const char *binary_key(int *) const; + const char *binary_key(int &) const; + const char *string_key(void) const; + const char *string_key(int *) const; + const char *string_key(int &) const; + long long int_value(const char *) const; + double float_value(const char *) const; + const char *string_value(const char *) const; + const char *string_value(const char *, int *) const; + const char *string_value(const char *, int &) const; + const char *binary_value(const char *) const; + const char *binary_value(const char *, int *) const; + const char *binary_value(const char *, int &) const; + int uncompress_binary_value(const char *name, char **buf, int *lenp); + //Uncompress Binary Value without check compressflag + int uncompress_binary_value_force(const char *name, char **buf, int *lenp); + const char *uncompress_error_message() const; + long long int_value(int) const; + double float_value(int) const; + const char *string_value(int) const; + const char *string_value(int, int *) const; + const char *string_value(int, int &) const; + const char *binary_value(int) const; + const char *binary_value(int, int *) const; + const char *binary_value(int, int &) const; + int fetch_row(void); + int rewind(void); + }; + + class ServerPool + { + private: + void *addr; + long check; + ServerPool(ServerPool &); + + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int get_epoll_fd(int size); + int add_server(Server *srv, int mReq = 1, int mConn = 0); + int add_request(Request *, long long); + int add_request(Request *, long long, long long); + int add_request(Request *, long long, const char *); + int add_request(Request *, long long, const char *, int); + int add_request(Request *, void *); + int add_request(Request *, void *, long long); + int add_request(Request *, void *, const char *); + int add_request(Request *, void *, const char *, int); + int execute(int msec); + int execute_all(int msec); + int cancel_request(int); + int cancel_all_request(int type); + int abort_request(int); + int abort_all_request(int type); + Result *get_result(void); + Result *get_result(int); + int get_result(Result &); + int get_result(Result &, int); + + int server_count(void) const; + int request_count(int type) const; + int request_state(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT | SEND | RECV | DONE; + + enum + { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + + EC_TASK_TIMEOUT, + + EC_BUSINESS_WITHOUT_EXPIRETIME, //62 + EC_EMPTY_TBDEF, //63 + EC_INVALID_KEY_VALUE, //64 + EC_INVALID_EXPIRETIME, //65 + + EC_GET_EXPIRETIME_END_OF_RESULT, //66 + + EC_ERR_MIGRATEDB_ILLEGAL, + EC_ERR_MIGRATEDB_DUPLICATE, + EC_ERR_MIGRATEDB_HELPER, + + EC_ERR_MIGRATEDB_MIGRATING, // 70 + EC_ERR_MIGRATEDB_NOT_START, + EC_ERR_MIGRATEDB_DISTINCT, + EC_ERR_COL_EXPANDING, + EC_ERR_COL_EXPAND_DUPLICATE, + + EC_ERR_COL_EXPAND_DONE_DUPLICATE, // 75 + EC_ERR_COL_EXPAND_DONE_DISTINCT, + EC_ERR_COL_EXPAND_NO_MEM, + EC_ERR_COL_EXPAND_COLD, + }; + + enum + { + ER_HASHCHK = 1000, + ER_NISAMCHK = 1001, + ER_NO = 1002, + ER_YES = 1003, + ER_CANT_CREATE_FILE = 1004, + ER_CANT_CREATE_TABLE = 1005, + ER_CANT_CREATE_DB = 1006, + ER_DB_CREATE_EXISTS = 1007, + ER_DB_DROP_EXISTS = 1008, + ER_DB_DROP_DELETE = 1009, + ER_DB_DROP_RMDIR = 1010, + ER_CANT_DELETE_FILE = 1011, + ER_CANT_FIND_SYSTEM_REC = 1012, + ER_CANT_GET_STAT = 1013, + ER_CANT_GET_WD = 1014, + ER_CANT_LOCK = 1015, + ER_CANT_OPEN_FILE = 1016, + ER_FILE_NOT_FOUND = 1017, + ER_CANT_READ_DIR = 1018, + ER_CANT_SET_WD = 1019, + ER_CHECKREAD = 1020, + ER_DISK_FULL = 1021, + ER_DUP_KEY = 1022, + ER_ERROR_ON_CLOSE = 1023, + ER_ERROR_ON_READ = 1024, + ER_ERROR_ON_RENAME = 1025, + ER_ERROR_ON_WRITE = 1026, + ER_FILE_USED = 1027, + ER_FILSORT_ABORT = 1028, + ER_FORM_NOT_FOUND = 1029, + ER_GET_ERRNO = 1030, + ER_ILLEGAL_HA = 1031, + ER_KEY_NOT_FOUND = 1032, + ER_NOT_FORM_FILE = 1033, + ER_NOT_KEYFILE = 1034, + ER_OLD_KEYFILE = 1035, + ER_OPEN_AS_READONLY = 1036, + ER_OUTOFMEMORY = 1037, + ER_OUT_OF_SORTMEMORY = 1038, + ER_UNEXPECTED_EOF = 1039, + ER_CON_COUNT_ERROR = 1040, + ER_OUT_OF_RESOURCES = 1041, + ER_BAD_HOST_ERROR = 1042, + ER_HANDSHAKE_ERROR = 1043, + ER_DBACCESS_DENIED_ERROR = 1044, + ER_ACCESS_DENIED_ERROR = 1045, + ER_NO_DB_ERROR = 1046, + ER_UNKNOWN_COM_ERROR = 1047, + ER_BAD_NULL_ERROR = 1048, + ER_BAD_DB_ERROR = 1049, + ER_TABLE_EXISTS_ERROR = 1050, + ER_BAD_TABLE_ERROR = 1051, + ER_NON_UNIQ_ERROR = 1052, + ER_SERVER_SHUTDOWN = 1053, + ER_BAD_FIELD_ERROR = 1054, + ER_WRONG_FIELD_WITH_GROUP = 1055, + ER_WRONG_GROUP_FIELD = 1056, + ER_WRONG_SUM_SELECT = 1057, + ER_WRONG_VALUE_COUNT = 1058, + ER_TOO_LONG_IDENT = 1059, + ER_DUP_FIELDNAME = 1060, + ER_DUP_KEYNAME = 1061, + ER_DUP_ENTRY = 1062, + ER_WRONG_FIELD_SPEC = 1063, + ER_PARSE_ERROR = 1064, + ER_EMPTY_QUERY = 1065, + ER_NONUNIQ_TABLE = 1066, + ER_INVALID_DEFAULT = 1067, + ER_MULTIPLE_PRI_KEY = 1068, + ER_TOO_MANY_KEYS = 1069, + ER_TOO_MANY_KEY_PARTS = 1070, + ER_TOO_LONG_KEY = 1071, + ER_KEY_COLUMN_DOES_NOT_EXITS = 1072, + ER_BLOB_USED_AS_KEY = 1073, + ER_TOO_BIG_FIELDLENGTH = 1074, + ER_WRONG_AUTO_KEY = 1075, + ER_READY = 1076, + ER_NORMAL_SHUTDOWN = 1077, + ER_GOT_SIGNAL = 1078, + ER_SHUTDOWN_COMPLETE = 1079, + ER_FORCING_CLOSE = 1080, + ER_IPSOCK_ERROR = 1081, + ER_NO_SUCH_INDEX = 1082, + ER_WRONG_FIELD_TERMINATORS = 1083, + ER_BLOBS_AND_NO_TERMINATED = 1084, + ER_TEXTFILE_NOT_READABLE = 1085, + ER_FILE_EXISTS_ERROR = 1086, + ER_LOAD_INFO = 1087, + ER_ALTER_INFO = 1088, + ER_WRONG_SUB_KEY = 1089, + ER_CANT_REMOVE_ALL_FIELDS = 1090, + ER_CANT_DROP_FIELD_OR_KEY = 1091, + ER_INSERT_INFO = 1092, + ER_INSERT_TABLE_USED = 1093, + ER_NO_SUCH_THREAD = 1094, + ER_KILL_DENIED_ERROR = 1095, + ER_NO_TABLES_USED = 1096, + ER_TOO_BIG_SET = 1097, + ER_NO_UNIQUE_LOGFILE = 1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099, + ER_TABLE_NOT_LOCKED = 1100, + ER_BLOB_CANT_HAVE_DEFAULT = 1101, + ER_WRONG_DB_NAME = 1102, + ER_WRONG_TABLE_NAME = 1103, + ER_TOO_BIG_SELECT = 1104, + ER_UNKNOWN_ERROR = 1105, + ER_UNKNOWN_PROCEDURE = 1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108, + ER_UNKNOWN_TABLE = 1109, + ER_FIELD_SPECIFIED_TWICE = 1110, + ER_INVALID_GROUP_FUNC_USE = 1111, + ER_UNSUPPORTED_EXTENSION = 1112, + ER_TABLE_MUST_HAVE_COLUMNS = 1113, + ER_RECORD_FILE_FULL = 1114, + ER_UNKNOWN_CHARACTER_SET = 1115, + ER_TOO_MANY_TABLES = 1116, + ER_TOO_MANY_FIELDS = 1117, + ER_TOO_BIG_ROWSIZE = 1118, + ER_STACK_OVERRUN = 1119, + ER_WRONG_OUTER_JOIN = 1120, + ER_NULL_COLUMN_IN_INDEX = 1121, + ER_CANT_FIND_UDF = 1122, + ER_CANT_INITIALIZE_UDF = 1123, + ER_UDF_NO_PATHS = 1124, + ER_UDF_EXISTS = 1125, + ER_CANT_OPEN_LIBRARY = 1126, + ER_CANT_FIND_DL_ENTRY = 1127, + ER_FUNCTION_NOT_DEFINED = 1128, + ER_HOST_IS_BLOCKED = 1129, + ER_HOST_NOT_PRIVILEGED = 1130, + ER_PASSWORD_ANONYMOUS_USER = 1131, + ER_PASSWORD_NOT_ALLOWED = 1132, + ER_PASSWORD_NO_MATCH = 1133, + ER_UPDATE_INFO = 1134, + ER_CANT_CREATE_THREAD = 1135, + ER_WRONG_VALUE_COUNT_ON_ROW = 1136, + ER_CANT_REOPEN_TABLE = 1137, + ER_INVALID_USE_OF_NULL = 1138, + ER_REGEXP_ERROR = 1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140, + ER_NONEXISTING_GRANT = 1141, + ER_TABLEACCESS_DENIED_ERROR = 1142, + ER_COLUMNACCESS_DENIED_ERROR = 1143, + ER_ILLEGAL_GRANT_FOR_TABLE = 1144, + ER_GRANT_WRONG_HOST_OR_USER = 1145, + ER_NO_SUCH_TABLE = 1146, + ER_NONEXISTING_TABLE_GRANT = 1147, + ER_NOT_ALLOWED_COMMAND = 1148, + ER_SYNTAX_ERROR = 1149, + ER_DELAYED_CANT_CHANGE_LOCK = 1150, + ER_TOO_MANY_DELAYED_THREADS = 1151, + ER_ABORTING_CONNECTION = 1152, + ER_NET_PACKET_TOO_LARGE = 1153, + ER_NET_READ_ERROR_FROM_PIPE = 1154, + ER_NET_FCNTL_ERROR = 1155, + ER_NET_PACKETS_OUT_OF_ORDER = 1156, + ER_NET_UNCOMPRESS_ERROR = 1157, + ER_NET_READ_ERROR = 1158, + ER_NET_READ_INTERRUPTED = 1159, + ER_NET_ERROR_ON_WRITE = 1160, + ER_NET_WRITE_INTERRUPTED = 1161, + ER_TOO_LONG_STRING = 1162, + ER_TABLE_CANT_HANDLE_BLOB = 1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164, + ER_DELAYED_INSERT_TABLE_LOCKED = 1165, + ER_WRONG_COLUMN_NAME = 1166, + ER_WRONG_KEY_COLUMN = 1167, + ER_WRONG_MRG_TABLE = 1168, + ER_DUP_UNIQUE = 1169, + ER_BLOB_KEY_WITHOUT_LENGTH = 1170, + ER_PRIMARY_CANT_HAVE_NULL = 1171, + ER_TOO_MANY_ROWS = 1172, + ER_REQUIRES_PRIMARY_KEY = 1173, + ER_NO_RAID_COMPILED = 1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175, + ER_KEY_DOES_NOT_EXITS = 1176, + ER_CHECK_NO_SUCH_TABLE = 1177, + ER_CHECK_NOT_IMPLEMENTED = 1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179, + ER_ERROR_DURING_COMMIT = 1180, + ER_ERROR_DURING_ROLLBACK = 1181, + ER_ERROR_DURING_FLUSH_LOGS = 1182, + ER_ERROR_DURING_CHECKPOINT = 1183, + ER_NEW_ABORTING_CONNECTION = 1184, + ER_DUMP_NOT_IMPLEMENTED = 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED = 1186, + ER_INDEX_REBUILD = 1187, + ER_MASTER = 1188, + ER_MASTER_NET_READ = 1189, + ER_MASTER_NET_WRITE = 1190, + ER_FT_MATCHING_KEY_NOT_FOUND = 1191, + ER_LOCK_OR_ACTIVE_TRANSACTION = 1192, + ER_UNKNOWN_SYSTEM_VARIABLE = 1193, + ER_CRASHED_ON_USAGE = 1194, + ER_CRASHED_ON_REPAIR = 1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196, + ER_TRANS_CACHE_FULL = 1197, + ER_SLAVE_MUST_STOP = 1198, + ER_SLAVE_NOT_RUNNING = 1199, + ER_BAD_SLAVE = 1200, + ER_MASTER_INFO = 1201, + ER_SLAVE_THREAD = 1202, + ER_TOO_MANY_USER_CONNECTIONS = 1203, + ER_SET_CONSTANTS_ONLY = 1204, + ER_LOCK_WAIT_TIMEOUT = 1205, + ER_LOCK_TABLE_FULL = 1206, + ER_READ_ONLY_TRANSACTION = 1207, + ER_DROP_DB_WITH_READ_LOCK = 1208, + ER_CREATE_DB_WITH_READ_LOCK = 1209, + ER_WRONG_ARGUMENTS = 1210, + ER_NO_PERMISSION_TO_CREATE_USER = 1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212, + ER_LOCK_DEADLOCK = 1213, + ER_TABLE_CANT_HANDLE_FULLTEXT = 1214, + ER_CANNOT_ADD_FOREIGN = 1215, + ER_NO_REFERENCED_ROW = 1216, + ER_ROW_IS_REFERENCED = 1217, + ER_CONNECT_TO_MASTER = 1218, + ER_QUERY_ON_MASTER = 1219, + ER_ERROR_WHEN_EXECUTING_COMMAND = 1220, + ER_WRONG_USAGE = 1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222, + ER_CANT_UPDATE_WITH_READLOCK = 1223, + ER_MIXING_NOT_ALLOWED = 1224, + ER_DUP_ARGUMENT = 1225, + ER_USER_LIMIT_REACHED = 1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227, + ER_LOCAL_VARIABLE = 1228, + ER_GLOBAL_VARIABLE = 1229, + ER_NO_DEFAULT = 1230, + ER_WRONG_VALUE_FOR_VAR = 1231, + ER_WRONG_TYPE_FOR_VAR = 1232, + ER_VAR_CANT_BE_READ = 1233, + ER_CANT_USE_OPTION_HERE = 1234, + ER_NOT_SUPPORTED_YET = 1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236, + ER_SLAVE_IGNORED_TABLE = 1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238, + CR_UNKNOWN_ERROR = 1900, + CR_SOCKET_CREATE_ERROR = 1901, + CR_CONNECTION_ERROR = 1902, + CR_CONN_HOST_ERROR = 1903, + CR_IPSOCK_ERROR = 1904, + CR_UNKNOWN_HOST = 1905, + CR_SERVER_GONE_ERROR = 1906, + CR_VERSION_ERROR = 1907, + CR_OUT_OF_MEMORY = 1908, + CR_WRONG_HOST_INFO = 1909, + CR_LOCALHOST_CONNECTION = 1910, + CR_TCP_CONNECTION = 1911, + CR_SERVER_HANDSHAKE_ERR = 1912, + CR_SERVER_LOST = 1913, + CR_COMMANDS_OUT_OF_SYNC = 1914, + CR_NAMEDPIPE_CONNECTION = 1915, + CR_NAMEDPIPEWAIT_ERROR = 1916, + CR_NAMEDPIPEOPEN_ERROR = 1917, + CR_NAMEDPIPESETSTATE_ERROR = 1918, + CR_CANT_READ_CHARSET = 1919, + CR_NET_PACKET_TOO_LARGE = 1920, + CR_EMBEDDED_CONNECTION = 1921, + CR_PROBE_SLAVE_STATUS = 1922, + CR_PROBE_SLAVE_HOSTS = 1923, + CR_PROBE_SLAVE_CONNECT = 1924, + CR_PROBE_MASTER_CONNECT = 1925, + CR_SSL_CONNECTION_ERROR = 1926, + CR_MALFORMED_PACKET = 1927, + CR_WRONG_LICENSE = 1928, + }; +}; // namespace DistributedTableCache + +namespace DTC = DistributedTableCache; +#endif diff --git a/src/search_local/index_storage/api/c_api/dtc_api.lst b/src/search_local/index_storage/api/c_api/dtc_api.lst new file mode 100644 index 0000000..4aea542 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_api.lst @@ -0,0 +1,12 @@ +{ + global: + *DistributedTableCache*; + __invoke_dynamic_linker__; + set_network_mode; + set_server_address; + set_server_tablename; + + local: + *; +}; + diff --git a/src/search_local/index_storage/api/c_api/dtc_int.h b/src/search_local/index_storage/api/c_api/dtc_int.h new file mode 100644 index 0000000..b6d0980 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_int.h @@ -0,0 +1,494 @@ +/* + * ===================================================================================== + * + * Filename: dtc_int.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CHC_CLI_H +#define __CHC_CLI_H + +#include +#include +#include +#include +#include +#include +#include "key_list.h" +#include +#include +#include +#include "udp_pool.h" +#include "compress.h" +#include "poll_thread.h" +#include "lock.h" +#include + +/* + * Goal: + * single transation (*) + * batch transation (*) + * async transation + */ + +class DTCTask; +class Packet; + +class NCResult; +class NCResultInternal; +class NCPool; +struct NCTransation; +class IDTCTaskExecutor; +class IDTCService; + +class NCBase +{ +private: + AtomicU32 _count; + +public: + NCBase(void) {} + ~NCBase(void) {} + int INC(void) { return ++_count; } + int DEC(void) { return --_count; } + int CNT(void) { return _count.get(); }; +}; + +class NCRequest; +class DataConnector; +class NCServer : public NCBase +{ +public: // global server state + NCServer(); + NCServer(const NCServer &); + ~NCServer(void); + + // base settings + SocketAddress addr; + char *tablename; + char *appname; + + static DataConnector *dc; + + //for compress + void set_compress_level(int level) { compressLevel = level; } + int get_compress_level(void) { return compressLevel; } + + //dtc set _server_address and _server_tablename for plugin + static char *_server_address; + static char *_server_tablename; + + int keytype; + int autoUpdateTable; + int autoReconnect; + static int _network_mode; + NCKeyInfo keyinfo; + + void clone_tab_def(const NCServer &source); + int set_address(const char *h, const char *p = NULL); + int set_table_name(const char *); + const char *get_address(void) const { return addr.Name(); } //this addres is set by user + const char *get_server_address(void) const { return _server_address; } //this address is set by dtc + const char *get_server_table_name(void) const { return _server_tablename; } + int is_dgram(void) const { return addr.socket_type() == SOCK_DGRAM; } + int is_udp(void) const { return addr.socket_type() == SOCK_DGRAM && addr.socket_family() == AF_INET; } + const char *get_table_name(void) const { return tablename; } + int int_key(void); + int string_key(void); + int field_type(const char *); + int is_completed(void) const { return completed; } + int add_key(const char *name, uint8_t type); + int key_field_cnt(void) const { return keyinfo.key_fields() ?: keytype != DField::None ? 1 : 0; } + int allow_batch_key(void) const { return keyinfo.key_fields(); } + int simple_batch_key(void) const { return keyinfo.key_fields() == 1 && keytype == keyinfo.key_type(0); } + + void set_auto_update_tab(bool autoUpdate) { autoUpdateTable = autoUpdate ? 1 : 0; } + void set_auto_reconnect(int reconnect) { autoReconnect = reconnect; } + + // error state + unsigned completed : 1; + unsigned badkey : 1; + unsigned badname : 1; + unsigned autoping : 1; + const char *errstr; + + const char *error_message(void) const { return errstr; } + + // table definition + DTCTableDefinition *tdef; + DTCTableDefinition *admin_tdef; + + void save_definition(NCResult *); + DTCTableDefinition *get_tab_def(int cmd) const; + + std::string accessToken; + int set_access_key(const char *token); + +private: // serialNr manupulation + uint64_t lastSN; + +public: + uint64_t NextSerialNr(void) + { + ++lastSN; + if (lastSN == 0) + lastSN++; + return lastSN; + } + uint64_t LastSerialNr(void) { return lastSN; } + +private: // timeout settings + int timeout; + int realtmo; + +public: + void set_m_timeout(int n); + int get_timeout(void) const { return timeout; } + +private: + int compressLevel; + +private: // sync execution + IDTCService *iservice; + IDTCTaskExecutor *executor; + int netfd; + time_t lastAct; + NCRequest *pingReq; + +private: + uint64_t agentTime; + +public: + void set_agent_time(int t) { agentTime = t; } + uint64_t get_agent_time(void) { return agentTime; } + +public: + int Connect(void); + int Reconnect(void); + void Close(void); + void SetFD(int fd) + { + Close(); + netfd = fd; + update_timeout(); + } + // stream, connected + int send_packet_stream(Packet &); + int decode_result_stream(NCResult &); + // dgram, connected or connectless + int send_packet_dgram(SocketAddress *peer, Packet &); + int decode_result_dgram(SocketAddress *peer, NCResult &); + // connectless + NCUdpPort *get_global_port(void); + void put_global_port(NCUdpPort *); + + void TryPing(void); + int ping(void); + void auto_ping(void) + { + if (!is_dgram()) + autoping = 1; + } + NCResultInternal *execute_internal(NCRequest &rq, const DTCValue *kptr) { return executor->task_execute(rq, kptr); } + int has_internal_executor(void) const { return executor != 0; } + +private: + int bind_temp_unix_socket(void); + void update_timeout(void); + // this method is weak, and don't exist in libdtc.a + __attribute__((__weak__)) void check_internal_service(void); + +public: // transation manager, impl at dtcpool.cc + int async_connect(int &); + NCPool *ownerPool; + int ownerId; + void set_owner(NCPool *, int); + + NCResult *decode_buffer(const char *, int); + static int check_packet_size(const char *, int); +}; + +class NCRequest +{ +public: + NCServer *server; + uint8_t cmd; + uint8_t haskey; + uint8_t flags; + int err; + DTCValue key; + NCKeyValueList kvl; + FieldValueByName ui; + FieldValueByName ci; + FieldSetByName fs; + + DTCTableDefinition *tdef; + char *tablename; + int keytype; + + unsigned int limitStart; + unsigned int limitCount; + int adminCode; + + uint64_t hotBackupID; + uint64_t master_hb_timestamp; + uint64_t slave_hb_timestamp; + +public: + NCRequest(NCServer *, int op); + ~NCRequest(void); + int attach_server(NCServer *); + + void enable_no_cache(void) { flags |= DRequest::Flag::no_cache; } + void enable_no_next_server(void) { flags |= DRequest::Flag::no_next_server; } + void enable_no_result(void) { flags |= DRequest::Flag::NoResult; } + int add_condition(const char *n, uint8_t op, uint8_t t, const DTCValue &v); + int add_operation(const char *n, uint8_t op, uint8_t t, const DTCValue &v); + int compress_set(const char *n, const char *v, int len); + int compress_set_force(const char *n, const char *v, int len); + int add_value(const char *n, uint8_t t, const DTCValue &v); + int Need(const char *n, int); + void limit(unsigned int st, unsigned int cnt) + { + if (cnt == 0) + st = 0; + limitStart = st; + limitCount = cnt; + } + + int set_key(int64_t); + int set_key(const char *, int); + int unset_key(void); + int unset_key_value(void); + int field_type(const char *name) { return server ? server->field_type(name) : DField::None; } + int add_key_value(const char *name, const DTCValue &v, uint8_t type); + int set_cache_id(int dummy) { return err = -EINVAL; } + void set_admin_code(int code) { adminCode = code; } + void set_hot_backup_id(uint64_t v) { hotBackupID = v; } + void set_master_hb_timestamp(uint64_t t) { master_hb_timestamp = t; } + void set_slave_hb_timestamp(uint64_t t) { slave_hb_timestamp = t; } + + // never return NULL + NCResult *execute(const DTCValue *key = NULL); + NCResult *execute_stream(const DTCValue *key = NULL); + NCResult *execute_dgram(SocketAddress *peer, const DTCValue *key = NULL); + NCResult *execute_network(const DTCValue *key = NULL); + NCResult *execute_internal(const DTCValue *key = NULL); + NCResult *execute(int64_t); + NCResult *execute(const char *, int); + NCResult *pre_check(const DTCValue *key); // return error result, NULL==success + int set_compress_field_name(void); //Need compress flag for read,or set compressFlag for write + int Encode(const DTCValue *key, Packet *); + // return 1 if tdef changed... + int set_tab_def(void); + + int encode_buffer(char *&ptr, int &len, int64_t &magic, const DTCValue *key = NULL); + int encode_buffer(char *&ptr, int &len, int64_t &magic, int64_t); + int encode_buffer(char *&ptr, int &len, int64_t &magic, const char *, int); + const char *error_message(void) const + { + return _errmsg; + } + + int set_expire_time(const char *key, int t); + int get_expire_time(const char *key); + +private: + int set_compress_flag(const char *name) + { + if (tdef == NULL) + return -EC_NOT_INITIALIZED; + if (tdef->field_id(name) >= 64) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:field id must less than 64"); + return -EC_COMPRESS_ERROR; + } + compressFlag |= (1 << tdef->field_id(name)); + return 0; + } + uint64_t compressFlag; //field flag + DTCCompress *gzip; + int init_compress(void); + char _errmsg[1024]; +}; + +class NCResultLocal +{ +public: + const uint8_t *vidmap; + long long apiTag; + int maxvid; + DTCCompress *gzip; + +public: + NCResultLocal(DTCTableDefinition *tdef) : vidmap(NULL), + apiTag(0), + maxvid(0), + gzip(NULL), + _tdef(tdef), + compressid(-1) + { + } + + virtual ~NCResultLocal(void) + { + FREE_CLEAR(vidmap); + DELETE(gzip); + } + + virtual int field_id_virtual(int id) const + { + return id > 0 && id <= maxvid ? vidmap[id - 1] : 0; + } + + virtual void set_api_tag(long long t) { apiTag = t; } + virtual long long get_api_tag(void) const { return apiTag; } + + void set_virtual_map(FieldSetByName &fs) + { + if (fs.max_virtual_id()) + { + fs.Resolve(_tdef, 0); + vidmap = fs.virtual_map(); + maxvid = fs.max_virtual_id(); + } + } + + virtual int init_compress() + { + if (NULL == _tdef) + { + return -EC_CHECKSUM_MISMATCH; + } + + int iret = 0; + compressid = _tdef->compress_field_id(); + if (compressid < 0) + return 0; + if (gzip == NULL) + NEW(DTCCompress, gzip); + if (gzip == NULL) + return -ENOMEM; + iret = gzip->set_buffer_len(_tdef->max_field_size()); + if (iret) + return iret; + return 0; + } + + virtual const int compress_id(void) const { return compressid; } + +private: + DTCTableDefinition *_tdef; + uint64_t compressid; +}; + +class NCResult : public NCResultLocal, public DTCTask +{ +public: + NCResult(DTCTableDefinition *tdef) : NCResultLocal(tdef), DTCTask(tdef, TaskRoleClient, 0) + { + if (tdef) + tdef->INC(); + mark_allow_remote_table(); + } + + NCResult(int err, const char *from, const char *msg) : NCResultLocal(NULL), DTCTask(NULL, TaskRoleClient, 1) + { + resultInfo.set_error_dup(err, from, msg); + } + virtual ~NCResult() + { + DTCTableDefinition *tdef = table_definition(); + DEC_DELETE(tdef); + } +}; + +class NCResultInternal : public NCResultLocal, public TaskRequest +{ +public: + NCResultInternal(DTCTableDefinition *tdef = NULL) : NCResultLocal(tdef) + { + } + + virtual ~NCResultInternal() + { + } + + static inline int verify_class(void) + { + NCResultInternal *ir = 0; + NCResult *er = reinterpret_cast(ir); + + NCResultLocal *il = (NCResultLocal *)ir; + NCResultLocal *el = (NCResultLocal *)er; + + DTCTask *it = (DTCTask *)ir; + DTCTask *et = (DTCTask *)er; + + long dl = reinterpret_cast(il) - reinterpret_cast(el); + long dt = reinterpret_cast(it) - reinterpret_cast(et); + return dl == 0 && dt == 0; + } +}; + +class DataConnector +{ + struct businessStatistics + { + uint64_t TotalTime; // 10s内请求总耗时 + uint32_t TotalRequests; // 10s内请求总次数 + + public: + businessStatistics() + { + TotalTime = 0; + TotalRequests = 0; + } + }; + + struct bidCurve + { + uint32_t bid; + uint32_t curve; + bool operator<(const bidCurve &that) const + { + int sum1 = bid * 10 + curve; + int sum2 = that.bid * 10 + that.curve; + return sum1 < sum2; + } + }; + +private: + std::map mapBi; + Mutex _lock; // 读写 TotalTime、TotalRequests时，加锁，防止脏数据 + static DataConnector *pDataConnector; + DataConnector(); + ~DataConnector(); + pthread_t threadid; + +public: + static DataConnector *getInstance() + { + if (pDataConnector == NULL) + pDataConnector = new DataConnector(); + return pDataConnector; + }; + +public: + int send_data(); + int set_report_info(const std::string str, const uint32_t curve, const uint64_t t); + void get_report_info(std::map &mapData); + int set_bussiness_id(std::string str); +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api/dtc_pool.cc b/src/search_local/index_storage/api/c_api/dtc_pool.cc new file mode 100644 index 0000000..bfec72f --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_pool.cc @@ -0,0 +1,1075 @@ +/* + * ===================================================================================== + * + * Filename: dtc_pool.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include +#include +#include +#include +#include "myepoll.h" + +#include "dtc_pool.h" + +// must be bitwise +enum +{ + RS_IDLE = 0, + RS_WAIT = 1, + RS_SEND = 2, + RS_RECV = 4, + RS_DONE = 8, +}; + +enum +{ + SS_CONNECTED = 0, + SS_LINGER = 1, + SS_CLOSED = 1, + SS_CONNECTING = 2, +}; + +#define MAXREQID 2147483646L + +NCPool::NCPool(int ms, int mr) + : PollerUnit(1024 + 16) +{ + int i; + + if (ms > (1 << 20)) + ms = 1 << 20; + if (mr > (1 << 20)) + mr = 1 << 20; + + maxServers = ms; + maxRequests = mr; + maxRequestId = MAXREQID / maxRequests; + + serverList = new NCServerInfo[maxServers]; + // needn't zero init + numServers = 0; + + transList = new NCTransation[maxRequests]; + for (i = 0; i < maxRequests; i++) + { + transList[i].ListAdd(&freeList); + ; + } + numRequests = 0; + + doneRequests = 0; + + initFlag = -1; + buf = NULL; +} + +NCPool::~NCPool() +{ + int i; + + DELETE_ARRAY(serverList); + if (transList) + { + for (i = 0; i < maxRequests; i++) + { + } + + DELETE_ARRAY(transList); + } + DELETE_ARRAY(buf); +} + +/* add server to pool */ +int NCPool::add_server(NCServer *server, int mReq, int mConn) +{ + if (server->ownerPool != NULL) + return -EINVAL; + + if (numServers >= maxServers) + return -E2BIG; + + if (server->is_dgram()) + { + if (mConn == 0) + mConn = 1; + else if (mConn != 1) + return -EINVAL; + } + else + { + if (mConn == 0) + mConn = mReq; + else if (mConn > mReq) + return -EINVAL; + } + + server->set_owner(this, numServers); + serverList[numServers].Init(server, mReq, mConn); + numServers++; + return 0; +} + +/* switch basic server to async mode */ +void NCServer::set_owner(NCPool *owner, int id) +{ + Close(); + ownerId = id; + ; + ownerPool = owner; + INC(); +} + +/* async connect, high level */ +int NCServer::async_connect(int &netfd) +{ + int err = -EC_NOT_INITIALIZED; + netfd = -1; + + if (addr.socket_family() != 0) + { + netfd = addr.create_socket(); + if (netfd < 0) + { + err = -errno; + } + else if (addr.socket_family() == AF_UNIX && is_dgram() && bind_temp_unix_socket() < 0) + { + err = -errno; + close(netfd); + netfd = -1; + } + else + { + fcntl(netfd, F_SETFL, O_RDWR | O_NONBLOCK); + if (addr.connect_socket(netfd) == 0) + return 0; + err = -errno; + if (err != -EINPROGRESS) + { + close(netfd); + netfd = -1; + } + } + } + return err; +} + +NCConnection::NCConnection(NCPool *owner, NCServerInfo *si) + : PollerObject(owner), + timer(this) +{ + serverInfo = si; + sreq = NULL; + state = SS_CLOSED; + result = NULL; +} + +NCConnection::~NCConnection(void) +{ + if (state == SS_CONNECTING) + /* decrease connecting count */ + serverInfo->connecting_done(); + /* remove partial result */ + DELETE(result); + /* remove from all connection queue */ + list_del(); + /* abort all associated requests */ + abort_requests(-EC_REQUEST_ABORTED); + serverInfo->more_closed_connection_and_ready(); +} + +/* connection is idle again */ +void NCConnection::switch_to_idle(void) +{ + if (state == SS_CONNECTING) + /* decrease connecting count */ + serverInfo->connecting_done(); + ListMove(&serverInfo->idleList); + serverInfo->connection_idle_and_ready(); + state = SS_CONNECTED; + disable_output(); + enable_input(); + timer.disable_timer(); + /* ApplyEvnets by caller */ +} + +/* prepare connecting state, wait for EPOLLOUT */ +void NCConnection::switch_to_connecting(void) +{ + state = SS_CONNECTING; + timer.attach_timer(serverInfo->timerList); + ListMove(&serverInfo->busyList); + DisableInput(); + enable_output(); + // no apply events, following attach_poller will do it */ +} + +/* try async connecting */ +int NCConnection::Connect(void) +{ + int err = serverInfo->info->async_connect(netfd); + if (err == 0) + { + switch_to_idle(); + /* treat epollsize overflow as fd overflow */ + if (attach_poller() < 0) + return -EMFILE; + /* attach_poller will apply_events automatically */ + } + else if (err == -EINPROGRESS) + { + switch_to_connecting(); + /* treat epollsize overflow as fd overflow */ + if (attach_poller() < 0) + return -EMFILE; + /* attach_poller will apply_events automatically */ + } + + return err; +} + +/* Link connection & transation */ +void NCConnection::process_request(NCTransation *r) +{ + /* linkage between connection & transation */ + sreq = r; + sreq->attach_connection(this); + + /* adjust server connection statistics */ + list_move_tail(&serverInfo->busyList); + serverInfo->request_scheduled(); + + /* initial timing and flushing */ + timer.attach_timer(serverInfo->timerList); + send_request(); +} + +/* abort all requests associated transation */ +void NCConnection::abort_requests(int err) +{ + if (sreq) + { + sreq->Abort(err); + sreq = NULL; + } + while (reqList.ListEmpty() == 0) + { + NCTransation *req = reqList.NextOwner(); + req->Abort(err); + } +} + +/* abort sending transation, linger connection for RECV request */ +void NCConnection::abort_send_side(int err) +{ + if (sreq) + { + sreq->Abort(err); + sreq = NULL; + } + if (!is_async() || reqList.ListEmpty()) + Close(err); + else + { + // async connection, lingering + ListMove(&serverInfo->busyList); + state = SS_LINGER; + timer.attach_timer(serverInfo->timerList); + } +} + +/* close a connection */ +void NCConnection::Close(int err) +{ + abort_requests(err); + if (is_dgram() == 0) + delete this; + /* UDP has not connection close */ +} + +/* a valid result received */ +void NCConnection::done_result(void) +{ + if (result) + { + /* searching SN */ + ListObject *reqPtr = reqList.ListNext(); + while (reqPtr != &reqList) + { + NCTransation *req = reqPtr->ListOwner(); + if (req->MatchSN(result)) + { + /* SN matched, transation is dont */ + req->Done(result); + result = NULL; + break; + } + reqPtr = reqPtr->ListNext(); + } + DELETE(result); + } + if (is_async() == 0) + { + /* + * SYNC server, switch to idle state, + * ASYNC server always idle, no switch needed + */ + switch_to_idle(); + apply_events(); + } + else if (state == SS_LINGER) + { + /* close LINGER connection if all result done */ + if (reqList.ListEmpty()) + delete this; + } + /* + // disabled because async server should trigger by req->Done + else + serverInfo->mark_as_ready(); + */ +} + +/* hangup by recv zero bytes */ +int NCConnection::check_hangup(void) +{ + if (is_async()) + return 0; + char buf[1]; + int n = recv(netfd, buf, sizeof(buf), MSG_DONTWAIT | MSG_PEEK); + return n >= 0; +} + +/* + * sending result, may called by other component + * apply events is required + */ +void NCConnection::send_request(void) +{ + int ret = sreq->Send(netfd); + timer.disable_timer(); + switch (ret) + { + case SendResultMoreData: + /* more data to send, enable EPOLLOUT and timer */ + enable_output(); + apply_events(); + timer.attach_timer(serverInfo->timerList); + break; + + case SendResultDone: + /* send OK, disable output and enable receiving */ + disable_output(); + enable_input(); + sreq->SendOK(&reqList); + sreq = NULL; + serverInfo->request_sent(); + /* fire up receiving timer */ + timer.attach_timer(serverInfo->timerList); + if (is_async()) + { + list_move_tail(&serverInfo->idleList); + serverInfo->connection_idle_and_ready(); + } +#if 0 + else { + /* fire up receiving logic */ + RecvResult(); + } +#endif + apply_events(); + break; + + default: + abort_send_side(-ECONNRESET); + break; + } +} + +int NCConnection::RecvResult(void) +{ + int ret; + if (result == NULL) + { + result = new NCResult(serverInfo->info->tdef); + receiver.attach(netfd); + receiver.erase(); + } + if (is_dgram()) + { + if (serverInfo->owner->buf == NULL) + serverInfo->owner->buf = new char[65536]; + ret = recv(netfd, serverInfo->owner->buf, 65536, 0); + if (ret <= 0) + ret = DecodeFatalError; + else + ret = result->Decode(serverInfo->owner->buf, ret); + } + else + ret = result->Decode(receiver); + timer.disable_timer(); + switch (ret) + { + default: + case DecodeFatalError: + Close(-ECONNRESET); + return -1; // connection bad + break; + + case DecodeDataError: + done_result(); + // more result maybe available + ret = 1; + break; + + case DecodeIdle: + case DecodeWaitData: + // partial or no result available yet + ret = 0; + break; + + case DecodeDone: + serverInfo->info->save_definition(result); + done_result(); + // more result maybe available + ret = 1; + break; + } + if (sreq || !reqList.ListEmpty()) + timer.attach_timer(serverInfo->timerList); + return ret; +} + +void NCConnection::input_notify(void) +{ + switch (state) + { + case SS_CONNECTED: + while (1) + { + if (RecvResult() <= 0) + break; + } + break; + case SS_LINGER: + while (reqList.ListEmpty() == 0) + { + if (RecvResult() <= 0) + break; + } + default: + break; + } +} + +void NCConnection::output_notify(void) +{ + switch (state) + { + case SS_CONNECTING: + switch_to_idle(); + break; + case SS_CONNECTED: + send_request(); + break; + default: + disable_output(); + break; + } +} + +void NCConnection::hangup_notify(void) +{ + if (state == SS_CONNECTING) + { + serverInfo->connecting_failed(); + } + abort_requests(-ECONNRESET); + delete this; +} + +void NCConnection::timer_notify(void) +{ + if (sreq || !reqList.ListEmpty()) + { + Close(-ETIMEDOUT); + } +} + +NCServerInfo::NCServerInfo(void) +{ + info = NULL; +} + +NCServerInfo::~NCServerInfo(void) +{ + while (!idleList.ListEmpty()) + { + NCConnection *conn = idleList.NextOwner(); + delete conn; + } + + while (!busyList.ListEmpty()) + { + NCConnection *conn = busyList.NextOwner(); + delete conn; + } + + while (!reqList.ListEmpty()) + { + NCTransation *trans = reqList.NextOwner(); + trans->Abort(-EC_REQUEST_ABORTED); + } + + if (info && info->DEC() == 0) + delete info; +} + +void NCServerInfo::Init(NCServer *server, int maxReq, int maxConn) +{ + info = server; + reqList.ResetList(); + idleList.ResetList(); + busyList.ResetList(); + + reqWait = 0; + reqSend = 0; + reqRecv = 0; + reqRemain = maxReq; + + connRemain = maxConn; + connTotal = maxConn; + connConnecting = 0; + connError = 0; + owner = server->ownerPool; + int to = server->get_timeout(); + if (to <= 0) + to = 3600 * 1000; // 1 hour + timerList = owner->get_timer_list_by_m_seconds(to); + + mode = server->is_dgram() ? 2 : maxReq == maxConn ? 0 : 1; +} + +int NCServerInfo::Connect(void) +{ + NCConnection *conn = new NCConnection(owner, this); + connRemain--; + int err = conn->Connect(); + if (err == 0) + { + /* pass */; + } + else if (err == -EINPROGRESS) + { + connConnecting++; + err = 0; /* NOT a ERROR */ + } + else + { + connRemain++; + delete conn; + connError++; + } + return err; +} + +void NCServerInfo::abort_wait_queue(int err) +{ + while (!reqList.ListEmpty()) + { + NCTransation *trans = reqList.NextOwner(); + trans->Abort(err); + } +} + +void NCServerInfo::timer_notify() +{ + while (reqRemain > 0 && !reqList.ListEmpty()) + { + if (!idleList.ListEmpty()) + { + // has something to do + NCConnection *conn = idleList.NextOwner(); + + conn->process_request(get_request_from_queue()); + } + else + { + // NO connection available + if (connRemain == 0) + break; + // need more connection to process + if (connConnecting >= reqWait) + break; + + int err; + // connect error, abort processing + if ((err = Connect()) != 0) + { + if (connRemain >= connTotal) + abort_wait_queue(err); + break; + } + } + } + /* no more work, clear bogus ready timer */ + TimerObject::disable_timer(); + /* reset connect error count */ + connError = 0; +} + +void NCServerInfo::request_done_and_ready(int state) +{ + switch (state) + { + case RS_WAIT: + reqWait--; // Abort a queued not really ready + break; + case RS_SEND: + reqSend--; + reqRemain++; + mark_as_ready(); + break; + case RS_RECV: + reqRecv--; + reqRemain++; + mark_as_ready(); + break; + } +} + +NCTransation::NCTransation(void) +{ +} + +NCTransation::~NCTransation(void) +{ + DELETE(packet); + DELETE(result); +} + +/* clear transation state */ +void NCTransation::Clear(void) +{ + // detach from list, mostly freeList + ListObject::ResetList(); + + // increase reqId + genId++; + + // reset local state + state = RS_IDLE; + reqTag = 0; + ; + server = NULL; + packet = NULL; + result = NULL; +} + +/* get transation result */ +NCResult *NCTransation::get_result(void) +{ + NCResult *ret = result; + ret->set_api_tag(reqTag); + DELETE(packet); + Clear(); + return ret; +} + +/* attach new request to transation slot */ +int NCTransation::attach_request(NCServerInfo *info, long long tag, NCRequest *req, DTCValue *key) +{ + state = RS_WAIT; + reqTag = tag; + server = info; + packet = new Packet; + int ret = req->Encode(key, packet); + if (ret < 0) + { + /* encode error */ + result = new NCResult(ret, "API::encoding", "client encode packet error"); + } + else + { + SN = server->info->LastSerialNr(); + } + return ret; +} + +extern inline void NCTransation::attach_connection(NCConnection *c) +{ + state = RS_SEND; + conn = c; +} + +extern inline void NCTransation::SendOK(ListObject *queue) +{ + state = RS_RECV; + ListAddTail(queue); +} + +extern inline void NCTransation::RecvOK(ListObject *queue) +{ + state = RS_DONE; + ListAddTail(queue); +} + +/* abort current transation, zero means succ */ +void NCTransation::Abort(int err) +{ + if (server == NULL) // empty slot or done + return; + + //Don't reverse abort connection, just clear linkage + //NCConnection *c = conn; + //DELETE(c); + conn = NULL; + + NCPool *owner = server->owner; + list_del(); + + server->request_done_and_ready(state); + server = NULL; + + if (err) + { + result = new NCResult(err, "API::aborted", "client abort request"); + } + + owner->transation_finished(this); +} + +NCResult *NCPool::get_transation_result(NCTransation *req) +{ + NCResult *ret = req->get_result(); + doneRequests--; + numRequests--; + req->ListAdd(&freeList); + return ret; +} + +/* get transation slot for new request */ +NCTransation *NCPool::get_transation_slot(void) +{ + NCTransation *trans = freeList.NextOwner(); + trans->Clear(); + trans->round_gen_id(maxRequestId); + numRequests++; + return trans; +} + +void NCPool::transation_finished(NCTransation *trans) +{ + trans->RecvOK(&doneList); + doneRequests++; +} + +int NCPool::add_request(NCRequest *req, long long tag, DTCValue *key) +{ + int ret; + if (numRequests >= maxRequests) + { + return -E2BIG; + } + + if (req->server == NULL) + { + return -EC_NOT_INITIALIZED; + } + + NCServer *server = req->server; + if (server->ownerPool == NULL) + { + ret = add_server(server, 1); + if (ret < 0) + return ret; + } + + if (server->ownerPool != this) + { + return -EINVAL; + } + + if (key == NULL && req->haskey) + key = &req->key; + + NCTransation *trans = get_transation_slot(); + NCServerInfo *info = &serverList[server->ownerId]; + + if (trans->attach_request(info, tag, req, key) < 0) + { + // attach error, result already prepared */ + transation_finished(trans); + } + else + { + info->queue_request(trans); + } + + return (trans - transList) + trans->gen_id() * maxRequests + 1; +} + +void NCPool::execute_one_loop(int timeout) +{ + wait_poller_events(timeout); + uint64_t now = GET_TIMESTAMP(); + process_poller_events(); + check_expired(now); + check_ready(); +} + +int NCPool::execute(int timeout) +{ + if (timeout > 3600000) + timeout = 3600000; + initialize_poller_unit(); + execute_one_loop(0); + if (doneRequests > 0) + return doneRequests; + int64_t till = GET_TIMESTAMP() + timeout * TIMESTAMP_PRECISION / 1000; + while (doneRequests <= 0) + { + int64_t exp = till - GET_TIMESTAMP(); + if (exp < 0) + break; + int msec; +#if TIMESTAMP_PRECISION > 1000 + msec = exp / (TIMESTAMP_PRECISION / 1000); +#else + msec = exp * 1000 / TIMESTAMP_PRECISION; +#endif + execute_one_loop(expire_micro_seconds(msec, 1)); + } + return doneRequests; +} + +int NCPool::execute_all(int timeout) +{ + if (timeout > 3600000) + timeout = 3600000; + initialize_poller_unit(); + execute_one_loop(0); + if (numRequests <= doneRequests) + return doneRequests; + int64_t till = GET_TIMESTAMP() + timeout * TIMESTAMP_PRECISION / 1000; + while (numRequests > doneRequests) + { + int64_t exp = till - GET_TIMESTAMP(); + if (exp < 0) + break; + int msec; +#if TIMESTAMP_PRECISION > 1000 + msec = exp / (TIMESTAMP_PRECISION / 1000); +#else + msec = exp * 1000 / TIMESTAMP_PRECISION; +#endif + execute_one_loop(expire_micro_seconds(msec, 1)); + } + return doneRequests; +} + +int NCPool::initialize_poller_unit(void) +{ + if (initFlag == -1) + { + initFlag = PollerUnit::initialize_poller_unit() >= 0; + } + return initFlag; +} + +int NCPool::get_epoll_fd(int mp) +{ + if (initFlag == -1 && mp > PollerUnit::get_max_pollers()) + PollerUnit::set_max_pollers(mp); + initialize_poller_unit(); + return PollerUnit::get_fd(); +} + +NCTransation *NCPool::Id2Req(int reqId) const +{ + if (reqId <= 0 || reqId > MAXREQID) + return NULL; + reqId--; + NCTransation *req = &transList[reqId % maxRequests]; + if (reqId / maxRequests != req->gen_id()) + return NULL; + return req; +} + +int NCPool::abort_request(NCTransation *req) +{ + if (req->conn) + { + /* send or recv state */ + if (req->server->mode == 0) + { // SYNC, always close connecction + req->conn->Close(-EC_REQUEST_ABORTED); + } + else if (req->server->mode == 1) + { // ASYNC, linger connection for other result, abort send side + if (req->State() == RS_SEND) + req->conn->abort_send_side(-EC_REQUEST_ABORTED); + else + req->Abort(-EC_REQUEST_ABORTED); + } + else + { // UDP, abort request only + req->Abort(-EC_REQUEST_ABORTED); + } + return 1; + } + else if (req->server) + { + /* waiting state, abort request only */ + req->Abort(-EC_REQUEST_ABORTED); + return 1; + } + return 0; +} + +int NCPool::cancel_request(int reqId) +{ + NCTransation *req = Id2Req(reqId); + if (req == NULL) + return -EINVAL; + + abort_request(req); + NCResult *res = get_transation_result(req); + DELETE(res); + return 1; +} + +int NCPool::cancel_all_request(int type) +{ + int n = 0; + if ((type & ~0xf) != 0) + return -EINVAL; + for (int i = 0; i < maxRequests; i++) + { + NCTransation *req = &transList[i]; + if ((type & req->State())) + { + abort_request(req); + NCResult *res = get_transation_result(req); + DELETE(res); + n++; + } + } + return n; +} + +int NCPool::abort_request(int reqId) +{ + NCTransation *req = Id2Req(reqId); + if (req == NULL) + return -EINVAL; + + return abort_request(req); +} + +int NCPool::abort_all_request(int type) +{ + int n = 0; + if ((type & ~0xf) != 0) + return -EINVAL; + type &= ~RS_DONE; + for (int i = 0; i < maxRequests; i++) + { + NCTransation *req = &transList[i]; + if ((type & req->State())) + { + abort_request(req); + n++; + } + } + return n; +} + +NCResult *NCPool::get_result(int reqId) +{ + NCTransation *req; + if (reqId == 0) + { + if (doneList.ListEmpty()) + return (NCResult *)-EAGAIN; + req = doneList.NextOwner(); + } + else + { + req = Id2Req(reqId); + if (req == NULL) + return (NCResult *)-EINVAL; + + if (req->State() != RS_DONE) + return (NCResult *)req->State(); + } + return get_transation_result(req); +} + +int NCServerInfo::count_request_state(int type) const +{ + int n = 0; + if ((type & RS_WAIT)) + n += reqWait; + if ((type & RS_SEND)) + n += reqSend; + if ((type & RS_RECV)) + n += reqRecv; + return n; +} + +int NCPool::count_request_state(int type) const +{ + int n = 0; +#if 0 + for(int i=0; iState(); +} diff --git a/src/search_local/index_storage/api/c_api/dtc_pool.h b/src/search_local/index_storage/api/c_api/dtc_pool.h new file mode 100644 index 0000000..3d95e86 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_pool.h @@ -0,0 +1,315 @@ +/* + * ===================================================================================== + * + * Filename: dtc_pool.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CHC_CLI_POOL_H +#define __CHC_CLI_POOL_H + +#include + +#include "list.h" +#include "poller.h" +#include "timer_list.h" +#include "dtc_int.h" +#include "buffer_error.h" + +class NCServerInfo; +class NCPool; +class NCConnection; + +// transation is a internal async request +class NCTransation : public ListObject +{ +public: + // constructor/destructor equiv + NCTransation(void); + ~NCTransation(void); + /* clear transation state */ + void Clear(void); + /* abort current transation, zero means succ */ + void Abort(int); + /* transation succ with result */ + void Done(NCResult *res) + { + result = res; + Abort(0); + } + /* get transation result */ + NCResult *get_result(void); + /* adjust generation id */ + void round_gen_id(int m) + { + if (genId >= m) + genId = 0; + } + + /* state & info management */ + int State(void) const { return state; } + int gen_id(void) const { return genId; } + int MatchSN(NCResult *res) const { return SN == res->versionInfo.serial_nr(); } + /* send packet management */ + int Send(int fd) { return packet->Send(fd); } + int attach_request(NCServerInfo *s, long long tag, NCRequest *req, DTCValue *key); + void attach_connection(NCConnection *c); + void SendOK(ListObject *); + void RecvOK(ListObject *); + +public: // constant member declare as static + // owner info, associated server + NCServerInfo *server; + // attached connection, SEND, RECV + NCConnection *conn; + +private: // transient members is private + // current transation state, WAIT, SEND, RECV, DONE + int state; + // internal transation generation id + int genId; + // associated request tag + long long reqTag; + // associated request SN, state SEND, RECV + uint64_t SN; + + // sending packet + Packet *packet; + + // execute result + NCResult *result; +}; + +class NCConnection : public ListObject, + public PollerObject +{ +public: + NCConnection(NCPool *, NCServerInfo *); + ~NCConnection(void); + + int is_dgram(void) const; + int is_async(void) const; + /* starting state machine, by NCServerInfo::Connect */ + int Connect(void); + /* attach transation to connection */ + void process_request(NCTransation *); + /* flush send channel */ + void send_request(void); + /* flush recv channel */ + int RecvResult(void); + /* check connection hangup, recv zero bytes */ + int check_hangup(void); + + /* abort all associated request */ + void abort_requests(int err); + /* close connection */ + void Close(int err); + /* abort current sending request, linger recv channel */ + void abort_send_side(int err); + /* a valid result received */ + void done_result(void); + /* connection is idle, usable for transation */ + void switch_to_idle(void); + /* connection is async connecting */ + void switch_to_connecting(void); + + virtual void input_notify(void); + virtual void output_notify(void); + virtual void hangup_notify(void); + virtual void timer_notify(void); + +private: + /* associated server */ + NCServerInfo *serverInfo; + /* queued transation in RECV state */ + ListObject reqList; + /* sending transation */ + NCTransation *sreq; + /* decoding/decoded result */ + NCResult *result; + /* connection state */ + int state; + + int NETFD(void) const { return netfd; } + +private: + TimerMember timer; + SimpleReceiver receiver; +}; + +typedef ListObject NCConnectionList; + +class NCPool : public PollerUnit, + public TimerUnit +{ +public: + NCPool(int maxServers, int maxRequests); + ~NCPool(); + + int initialize_poller_unit(void); + + int get_epoll_fd(int maxpoller); + int add_server(NCServer *srv, int maxReq = 1, int maxConn = 0); + int add_request(NCRequest *req, long long tag, DTCValue *key = 0); + + void execute_one_loop(int timeout); + int execute(int timeout); + int execute_all(int timeout); + + NCTransation *Id2Req(int) const; + int cancel_request(int); + int cancel_all_request(int); + int abort_request(NCTransation *); + int abort_request(int); + int abort_all_request(int); + NCResult *get_result(void); + NCResult *get_result(int); + + int count_request_state(int) const; + int request_state(int) const; + +public: + NCTransation *get_transation_slot(void); + void transation_finished(NCTransation *); + NCResult *get_transation_result(NCTransation *); + int get_transation_state(NCTransation *); + + int server_count(void) const { return numServers; } + int request_count(void) const { return numRequests; } + int done_request_count(void) const { return doneRequests; } + +private: + int initFlag; + int maxServers; + int maxRequests; + int numServers; + int numRequests; + int maxRequestId; + int doneRequests; + ListObject freeList; + ListObject doneList; + NCServerInfo *serverList; + NCTransation *transList; + +public: + char *buf; +}; + +class NCServerInfo : private TimerObject +{ +public: + NCServerInfo(void); + ~NCServerInfo(void); + void Init(NCServer *, int, int); + + /* prepare wake timer_notify */ + void mark_as_ready() { attach_ready_timer(owner); } + virtual void timer_notify(void); + /* four reason server has more work to do */ + /* more transation attached */ + void more_request_and_ready(void) + { + reqWait++; + mark_as_ready(); + } + /* more close connection, should reconnecting */ + void more_closed_connection_and_ready(void) + { + connRemain++; + mark_as_ready(); + } + /* more idle connection available */ + void connection_idle_and_ready(void) { mark_as_ready(); } + /* more request can assign to idle pool */ + void request_done_and_ready(int oldstate); + + /* one request scheduled to SEND state */ + void request_scheduled(void) + { + reqRemain--; + reqSend++; + } + void request_sent(void) + { + reqSend--; + reqRecv++; + } + + /* queue transation to this server */ + void queue_request(NCTransation *req) + { + req->ListAddTail(&reqList); + more_request_and_ready(); + } + /* one connecting aborted */ + void connecting_failed(void) { connError++; } + void connecting_done(void) { connConnecting--; } + + /* abort all waiting transations */ + void abort_wait_queue(int err); + + int count_request_state(int type) const; + + /* get a waiting transation */ + NCTransation *get_request_from_queue(void) + { + NCTransation *trans = reqList.NextOwner(); + trans->list_del(); + reqWait--; + return trans; + } + +private: + int Connect(void); + ListObject reqList; + +public: // constant member declare as public + /* associated ServerPool */ + NCPool *owner; + /* basic server info */ + NCServer *info; + TimerList *timerList; + int mode; // 0--TCP 1--ASYNC 2--UDP + /* total connection */ + int connTotal; + +private: // transient member is private + /* transation in state WAIT */ + int reqWait; + /* transation in state SEND */ + int reqSend; + /* transation in state RECV */ + int reqRecv; + /* remain requests can assign to connection pool */ + int reqRemain; + + /* remain connections can connect */ + int connRemain; + /* number connections in connecting state */ + int connConnecting; + /* number of connect error this round */ + int connError; + /* busy connection count */ + inline int conn_working(void) const { return connTotal - connConnecting - connRemain; } + +public: //except idleList for code conventional + /* idle connection list */ + NCConnectionList idleList; + /* busy connection list SEND,RECV,LINGER */ + NCConnectionList busyList; +}; + +inline int NCConnection::is_dgram(void) const { return serverInfo->mode == 2; } +inline int NCConnection::is_async(void) const { return serverInfo->mode; } + +#endif diff --git a/src/search_local/index_storage/api/c_api/dtc_req.cc b/src/search_local/index_storage/api/c_api/dtc_req.cc new file mode 100644 index 0000000..fd10bbc --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_req.cc @@ -0,0 +1,1184 @@ +/* + * ===================================================================================== + * + * Filename: dtc_req.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dtc_api.h" +#include "dtc_int.h" + +// NCRequest's methods always assign and return +// cache the logical error code +#define return_err(x) return err = (x) +#define return_err_res(x, y, z) return new NCResult(x, y, z) +#define MAX_EXPIRETIME 30 * 24 * 3600 +#define CLIENT_CURVE 5 +#define AGENT_CURVE 8 + +NCRequest::NCRequest(NCServer *s, int op) +{ + compressFlag = 0; + if (op != DRequest::Monitor && (s && s->is_completed() == 0)) //MonitorRequest need not table check + s = NULL; + server = s; + if (server) + server->INC(); + tdef = NULL; + tablename = NULL; + keytype = 0; + + switch (op) + { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + if (server) + { + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + } + break; + //跨IDC分布支持 + case DRequest::SvrAdmin: + case DRequest::Invalidate: + if (server) + { + tdef = server->admin_tdef; + tablename = (char *)"@HOT_BACKUP"; + keytype = DField::Unsigned; + } + break; + case DRequest::Monitor: + break; + default: + op = DRequest::result_code; + } + cmd = op; + err = 0; + haskey = 0; + flags = 0; + key.u64 = 0; + key.bin.ptr = NULL; + key.bin.len = 0; + + limitStart = 0; + limitCount = 0; + adminCode = 0; + hotBackupID = 0; + master_hb_timestamp = 0; + slave_hb_timestamp = 0; + if (server) + kvl.keyinfo = &server->keyinfo; + gzip = NULL; +} + +NCRequest::~NCRequest(void) +{ + unset_key_value(); + unset_key(); + if (server) + DEC_DELETE(server); + if (gzip) + DELETE(gzip); +} + +// attach_server() error is transient, don't cache it +int NCRequest::attach_server(NCServer *s) +{ + // NO-OP + if (s == server) + return 0; + + switch (cmd) + { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + break; + default: + // don't allow change admin target server + return -EPERM; + } + + /* new server must be initialized */ + if (s == NULL || s->is_completed() == 0) + return -EC_NOT_INITIALIZED; + + /* precheck same table */ + if (server) + { + if (keytype != s->keytype) + return -EC_BAD_KEY_TYPE; + if (!server->keyinfo.Equal(s->keyinfo)) + return -EC_BAD_KEY_TYPE; + if (strcmp(tablename, s->tablename) != 0) + return -EC_BAD_TABLE_NAME; + if (tdef == NULL) + { + // no current tabledef + } + else if (tdef == server->tdef) + { + // same tabledef + } + else if (tdef->is_same_table(server->tdef)) + { + // hash equal + } + else + { + // force re-resolve fieldnames + ui.Unresolve(); + ci.Unresolve(); + fs.Unresolve(); + } + + // release old server + DEC_DELETE(server); + } + + // switch to new server + server = s; + server->INC(); + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + kvl.keyinfo = &server->keyinfo; + return 0; +} + +int NCRequest::Need(const char *n, int vid) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (cmd != DRequest::Get && cmd != DRequest::SvrAdmin) + return_err(-EC_BAD_OPERATOR); + int ret = fs.add_field(n, vid); + if (ret) + err = ret; + return ret; +} + +int NCRequest::add_condition(const char *n, uint8_t op, uint8_t t, const DTCValue &v) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (cmd == DRequest::Insert || cmd == DRequest::Replace) + return_err(-EC_BAD_OPERATOR); + switch (t) + { + case DField::Signed: + case DField::Unsigned: + if (op >= DField::TotalComparison) + return_err(-EC_BAD_OPERATOR); + break; + case DField::String: + case DField::Binary: + if (op != DField::EQ && op != DField::NE) + return_err(-EC_BAD_OPERATOR); + break; + default: + return_err(-EC_BAD_FIELD_TYPE); + } + int ret = ci.add_value(n, op, t, v); + if (ret) + err = ret; + return ret; +} + +int NCRequest::add_operation(const char *n, uint8_t op, uint8_t t, const DTCValue &v) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + switch (cmd) + { + case DRequest::Insert: + case DRequest::Replace: + case DRequest::SvrAdmin: + if (op != DField::Set) + return_err(-EC_BAD_OPERATOR); + break; + case DRequest::Update: + break; + default: + return_err(-EC_BAD_OPERATOR); + break; + } + + switch (t) + { + case DField::Signed: + case DField::Unsigned: + if (op >= DField::TotalOperation) + return_err(-EC_BAD_OPERATOR); + break; + + case DField::String: + case DField::Binary: + if (op == DField::Add || op == DField::OR) + return_err(-EC_BAD_OPERATOR); + break; + + case DField::Float: + if (op == DField::SetBits || op == DField::OR) + return_err(-EC_BAD_OPERATOR); + break; + default: + return_err(-EC_BAD_FIELD_TYPE); + } + int ret = ui.add_value(n, op, t, v); + if (ret) + err = ret; + return ret; +} + +int NCRequest::init_compress() +{ + int iret = 0; + if (server == NULL) + return -EC_NOT_INITIALIZED; + if (tdef == NULL) + { + iret = server->ping(); //ping and get tabledef + tdef = server->tdef; + } + if (iret) + return iret; + if (gzip == NULL) + NEW(DTCCompress, gzip); + if (gzip == NULL) + return -ENOMEM; + if (tdef == NULL) + return -EC_CHECKSUM_MISMATCH; + if (server->get_compress_level()) + { + gzip->set_compress_level(server->get_compress_level()); + } + return gzip->set_buffer_len(tdef->max_field_size()); +} + +int NCRequest::compress_set(const char *n, const char *v, int len) +{ + int iret = 0; + if (len < 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldlen is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (n == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldname is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (v == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldvalue is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (gzip == NULL) + { + iret = init_compress(); + if (iret) + return iret; + } + if (tdef->compress_field_id() <= 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:DTC must add a field for compress(field_type=2,field_size=8,default_value=compressflag)"); + return -EC_COMPRESS_ERROR; + } + + if (tdef->field_type(tdef->field_id(n)) != DField::Binary) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress just support binary field"); + return -EC_BAD_VALUE_TYPE; + } + + iret = gzip->compress(v, len); + if (iret) + { + if (iret == -111111) + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress buff is null,sth sucks happend"); + else + snprintf(_errmsg, sizeof(_errmsg), "compress error:zlib return code is %d.", iret); + return -EC_COMPRESS_ERROR; + } + iret = set_compress_flag(n); + if (iret) + return iret; + + return add_operation(n, DField::Set, DField::String, DTCValue::Make(gzip->get_buf(), gzip->get_len())); +} + +int NCRequest::compress_set_force(const char *n, const char *v, int len) +{ + int iret = 0; + if (len < 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldlen is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (n == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldname is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (v == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldvalue is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (gzip == NULL) + { + iret = init_compress(); + if (iret) + return iret; + } + + if (tdef->field_type(tdef->field_id(n)) != DField::Binary) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress just support binary field"); + return -EC_BAD_VALUE_TYPE; + } + + iret = gzip->compress(v, len); + if (iret) + { + if (iret == -111111) + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress buff is null,sth sucks happend"); + else + snprintf(_errmsg, sizeof(_errmsg), "compress error:zlib return code is %d.", iret); + return -EC_COMPRESS_ERROR; + } + if (iret) + return iret; + + return add_operation(n, DField::Set, DField::String, DTCValue::Make(gzip->get_buf(), gzip->get_len())); +} + +int NCRequest::add_value(const char *n, uint8_t t, const DTCValue &v) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (cmd != DRequest::Insert && cmd != DRequest::Replace) + return_err(-EC_BAD_COMMAND); + int ret = ui.add_value(n, DField::Set, t, v); + if (ret) + err = ret; + return ret; +} + +int NCRequest::unset_key(void) +{ + if (haskey) + { + if (server && server->keytype == DField::String) + { + + DELETE_ARRAY(key.bin.ptr); + key.Set(NULL, 0); + } + haskey = 0; + } + return 0; +} + +int NCRequest::unset_key_value(void) +{ + kvl.Unset(); + flags &= ~DRequest::Flag::MultiKeyValue; + return 0; +} + +int NCRequest::set_key(int64_t k) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (server->keytype != DField::Signed) + return_err(-EC_BAD_KEY_TYPE); + key = k; + haskey = 1; + unset_key_value(); + return 0; +} + +int NCRequest::set_key(const char *name, int l) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (server->keytype != DField::String) + return_err(-EC_BAD_KEY_TYPE); + char *a = new char[l]; + memcpy(a, name, l); + DELETE_ARRAY(key.bin.ptr); + haskey = 1; + key.Set(a, l); + unset_key_value(); + return 0; +} + +int NCRequest::add_key_value(const char *name, const DTCValue &v, uint8_t type) +{ + if (server == NULL) + return_err(-EC_NOT_INITIALIZED); + if (server->allow_batch_key() == 0) + return_err(-EC_NOT_INITIALIZED); + + int ret = kvl.add_value(name, v, type); + if (ret < 0) + return err = ret; + + flags |= DRequest::Flag::MultiKeyValue; + unset_key(); + return 0; +} + +int NCRequest::set_tab_def(void) +{ + int ret = 0; + if (server) + { + switch (cmd) + { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + ret = tdef != server->tdef; + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + break; + //跨IDC分布支持 + case DRequest::SvrAdmin: + case DRequest::Invalidate: + ret = tdef != server->admin_tdef; + tdef = server->admin_tdef; + tablename = (char *)"@HOT_BACKUP"; + keytype = DField::Unsigned; + break; + default: + break; + } + } + return ret; +} + +int NCRequest::Encode(const DTCValue *kptr, Packet *pkt) +{ + int err; + int force = set_tab_def(); + + if (tdef && ((err = ui.Resolve(tdef, force)) || + (err = ci.Resolve(tdef, force)) || + (err = fs.Resolve(tdef, force)))) + { + return err; + } + + if ((err = pkt->encode_request(*this, kptr)) != 0) + { + return err; + } + return 0; +} + +NCResult *NCRequest::execute_stream(const DTCValue *kptr) +{ + int resend = 1; + int nrecv = 0; + int nsent = 0; + + while (1) + { + int err; + + if (resend) + { + Packet pk; + + if ((err = Encode(kptr, &pk)) < 0) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + if ((err = server->Connect()) != 0) + { + if (err == -EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + return_err_res(err, "API::connecting", "client api connect server error"); + } + + nsent++; + if ((err = server->send_packet_stream(pk)) != 0) + { + if (server->autoReconnect && nsent <= 1 && (err == -ECONNRESET || err == -EPIPE)) + { + resend = 1; + continue; + } + return_err_res(err, "API::sending", "client api send packet error"); + } + } + + NCResult *res = new NCResult(tdef); + res->versionInfo.set_serial_nr(server->LastSerialNr()); + + err = server->decode_result_stream(*res); + + if (err < 0) + { + if (cmd != DRequest::Nop && err == -EAGAIN && nsent <= 1) + { + resend = 1; + delete res; + continue; + } + // network error always aborted + return res; + } + nrecv++; + + if (res->versionInfo.serial_nr() != server->LastSerialNr()) + { + log_debug("SN different, receive again. my SN: %lu, result SN: %lu", + (unsigned long)server->LastSerialNr(), + (unsigned long)res->versionInfo.serial_nr()); + // receive again + resend = 0; + } + else if (res->result_code() == -EC_CHECKSUM_MISMATCH && nrecv <= 1) + { + resend = 1; + // tabledef changed, resend + //agent restart reconnect + if (res->versionInfo.ReConnect()) + { + server->Close(); + } + } + else + { + // got valid result + if (res->result_code() >= 0 && cmd == DRequest::Get) + res->set_virtual_map(fs); + + uint64_t time = 0; + if (res->resultInfo.tag_present(7)) + { + char *t = res->resultInfo.time_info(); + if (t) + { + DTCTimeInfo *t_info = (DTCTimeInfo *)t; + time = (t_info->time) > 0 ? (t_info->time) : 0; + server->set_agent_time(time); + } + } + //agent restart reconnect + if (res->versionInfo.ReConnect()) + { + server->Close(); + } + return res; + } + delete res; // delete invalid result and loop + } + + // UNREACHABLE + return NULL; +} + +NCResult *NCRequest::execute_dgram(SocketAddress *peer, const DTCValue *kptr) +{ + int resend = 1; + int nrecv = 0; + int nsent = 0; + + while (1) + { + int err; + + if (resend) + { + Packet pk; + + if ((err = Encode(kptr, &pk)) < 0) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + nsent++; + if ((err = server->send_packet_dgram(peer, pk)) != 0) + { + if (peer == NULL && server->autoReconnect && nsent <= 1 && + (err == -ECONNRESET || err == -EPIPE || err == -ECONNREFUSED || err == -ENOTCONN)) + { + if ((err = server->Reconnect()) != 0) + { + if (err == -EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + return_err_res(err, "API::connecting", "client api connect server error"); + } + resend = 1; + continue; + } + return_err_res(err, "API::sending", "client api send packet error"); + } + } + + NCResult *res = new NCResult(tdef); + res->versionInfo.set_serial_nr(server->LastSerialNr()); + err = server->decode_result_dgram(peer, *res); + if (err < 0) + { + // network error encountered + return res; + } + nrecv++; + + if (res->versionInfo.serial_nr() != server->LastSerialNr()) + { + log_debug("SN different, receive again. my SN: %lu, result SN: %lu", + (unsigned long)server->LastSerialNr(), + (unsigned long)res->versionInfo.serial_nr()); + // receive again + resend = 0; + } + else if (res->result_code() == -EC_CHECKSUM_MISMATCH && nrecv <= 1) + { + resend = 1; + // tabledef changed, resend + } + else + { + // got valid result + if (res->result_code() >= 0 && cmd == DRequest::Get) + res->set_virtual_map(fs); + + uint64_t time = 0; + if (res->resultInfo.tag_present(7)) + { + char *t = res->resultInfo.time_info(); + if (t) + { + DTCTimeInfo *t_info = (DTCTimeInfo *)t; + time = (t_info->time) > 0 ? (t_info->time) : 0; + server->set_agent_time(time); + } + } + return res; + } + delete res; // delete invalid result and loop + } + + /* UNREACHABLE */ + return NULL; +} + +NCResult *NCRequest::execute_network(const DTCValue *kptr) +{ + NCResult *res = NULL; + if (!server->is_dgram()) + { + res = execute_stream(kptr); + } + else if (server->is_dgram()) + { + NCUdpPort *port = server->get_global_port(); + //get UDPPORT{SN,fd} from udp pool + if (port == NULL) + { + return_err_res(err, "API::execute", "get udp port error.udp port may exhaust"); + } + res = execute_dgram(&server->addr, kptr); + server->put_global_port(port); + } + else + { + int err; + if ((err = server->Connect()) != 0) + { + if (err == -EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + return_err_res(err, "API::connecting", "client api connect server error"); + } + + res = execute_dgram(NULL, kptr); + } + return res; +} + +NCResult *NCRequest::execute_internal(const DTCValue *kptr) +{ + if (tdef == NULL) + { + return_err_res(err, "API::encoding", "internal error: request has no tdef"); + } + + int force = set_tab_def(); + + int err; + if ( + (err = ui.Resolve(tdef, force)) || + (err = ci.Resolve(tdef, force)) || + (err = fs.Resolve(tdef, force))) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + NCResultInternal *res = server->execute_internal(*this, kptr); + + if (res == NULL) + { + return_err_res(err, "API::sending", "internal client api execute error"); + } + + if (res->result_code() >= 0 && cmd == DRequest::Get) + res->set_virtual_map(fs); + + return reinterpret_cast(res); +} + +NCResult *NCRequest::pre_check(const DTCValue *kptr) +{ + + if (cmd == DRequest::Monitor) + { + //Monitor Request need not check + cmd = DRequest::SvrAdmin; //MonitorRequest is a stub of RequestSvrAdmin.Agent should add MonitorRequest type later + return NULL; + } + if (err) + return_err_res(err, "API::encoding", "Init Operation Error"); + if (server == NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if (cmd == DRequest::result_code) + return_err_res(-EC_BAD_COMMAND, "API::encoding", "Unknown Request Type"); + if (server->badkey) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + if (server->badname) + return_err_res(-EC_BAD_TABLE_NAME, "API::encoding", "Table Name Mismatch"); + if (cmd != DRequest::Insert && cmd != DRequest::SvrAdmin && + !(flags & DRequest::Flag::MultiKeyValue) && kptr == NULL) + return_err_res(-EINVAL, "API::encoding", "Missing Key"); + if ((flags & DRequest::Flag::MultiKeyValue) && kvl.IsFlat() == 0) + return_err_res(-EINVAL, "API::encoding", "Missing Key Value"); + if (tdef == NULL) + { + int ret = 0; + uint64_t time_before = GET_TIMESTAMP(); + ret = server->ping(); //ping and get tabledef + uint64_t time_after = GET_TIMESTAMP(); + tdef = server->tdef; + + uint64_t timeInterval = time_after - time_before; + if (timeInterval < 0) + timeInterval = 0; + //log_info("timeInterval: %lu", timeInterval); + std::string accessKey = server->accessToken; + if (ret == -ETIMEDOUT) + { + server->dc->set_report_info(accessKey, CLIENT_CURVE, server->get_timeout()); + server->dc->set_report_info(accessKey, AGENT_CURVE, server->get_timeout()); + } + else + { + server->dc->set_report_info(accessKey, CLIENT_CURVE, timeInterval); + server->dc->set_report_info(accessKey, AGENT_CURVE, (server->get_agent_time() != 0) ? server->get_agent_time() : timeInterval); + } + log_info("NCRequest::pre_check,seq:%lu, clientTime:%lu, agentTime:%lu", server->LastSerialNr(), timeInterval, server->get_agent_time()); + // 清除 ping 之后获取的 agent-dtc 响应耗时 + server->set_agent_time(0); + } + return NULL; +} + +int NCRequest::set_compress_field_name() +{ + int iret = 0; + if (tdef && tdef->compress_field_id() > 0) //启用压缩必定有这个字段 + { + if (gzip == NULL) //读请求需要need字段 + { + if (cmd != DRequest::Get) + return 0; + iret = Need(tdef->field_name(tdef->compress_field_id()), 0); + if (iret) + { + snprintf(_errmsg, sizeof(_errmsg), "need CompressField error,errorcode is %d", iret); + return -1; + } + } + else //写请求 + { + if (cmd == DRequest::Insert || cmd == DRequest::Replace) + { + iret = add_operation(tdef->field_name(tdef->compress_field_id()), //name + DField::Set, DField::Signed, DTCValue::Make(compressFlag)); + } + else if (cmd == DRequest::Update) + { + iret = add_operation(tdef->field_name(tdef->compress_field_id()), //name + DField::OR, DField::Signed, DTCValue::Make(compressFlag)); + } + else + { + snprintf(_errmsg, sizeof(_errmsg), "Request type[%d] not support compressset", cmd); + return -1; + } + if (iret) + { + snprintf(_errmsg, sizeof(_errmsg), "set CompressField error,errorcode is %d", iret); + return -1; + } + } + } + return 0; +} +//if get,need compressfield,if write set compressfield; +NCResult *NCRequest::execute(const DTCValue *kptr) +{ + NCResult *ret = NULL; + if (kptr == NULL && haskey) + kptr = &key; + ret = pre_check(kptr); + if (ret == NULL) + { + if (set_compress_field_name()) + return_err_res(-EC_COMPRESS_ERROR, "API::execute", "set_compress_field_name error. please check request.error_message"); + ; + + uint64_t time_before = GET_TIMESTAMP(); + if (server->has_internal_executor()) + ret = execute_internal(kptr); + else + ret = execute_network(kptr); + uint64_t time_after = GET_TIMESTAMP(); + uint64_t timeInterval = time_after - time_before; + if (timeInterval < 0) + timeInterval = 0; + std::string accessKey = server->accessToken; + + if (ret != NULL && ret->resultInfo.result_code() == -ETIMEDOUT) + { + server->dc->set_report_info(accessKey, CLIENT_CURVE, server->get_timeout()); + server->dc->set_report_info(accessKey, AGENT_CURVE, server->get_timeout()); + } + else + { + server->dc->set_report_info(accessKey, CLIENT_CURVE, timeInterval); + server->dc->set_report_info(accessKey, AGENT_CURVE, (server->get_agent_time() != 0) ? server->get_agent_time() : timeInterval); + } + std::string stemp = accessKey.substr(0, 8); + uint32_t bid = 0; + sscanf(stemp.c_str(), "%u", &bid); + log_info("NCRequest::execute,seq:%lu, clientTime:%lu, agentTime:%lu, bid:%u", server->LastSerialNr(), timeInterval, server->get_agent_time(), bid); + // 清除 解包 之后获取的 agent-dtc 响应耗时 + server->set_agent_time(0); + } + else + log_info("NCRequest::execute,seq:%lu, pre_check return NULL", server->LastSerialNr()); + return ret; +} + +NCResult *NCRequest::execute(int64_t k) +{ + if (server == NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if (server->keytype != DField::Signed) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + DTCValue v(k); + return execute(&v); +} + +NCResult *NCRequest::execute(const char *k, int l) +{ + if (server == NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if (server->keytype != DField::String) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + DTCValue v(k, l); + return execute(&v); +} + +// packet encoding don't cache error code +int NCRequest::encode_buffer(char *&ptr, int &len, int64_t &magic, const DTCValue *kptr) +{ + if (kptr == NULL && haskey) + kptr = &key; + int err = 0; + NCResult *ret = pre_check(kptr); + if (ret != NULL) + { + err = ret->result_code(); + delete ret; + return err; + } + + set_tab_def(); + if (tdef && ((err = ui.Resolve(tdef, 0)) || + (err = ci.Resolve(tdef, 0)) || + (err = fs.Resolve(tdef, 0)))) + { + return err; + } + + if ((err = Packet::encode_simple_request(*this, kptr, ptr, len)) != 0) + { + return err; + } + + magic = server->LastSerialNr(); + return 0; +} + +int NCRequest::encode_buffer(char *&ptr, int &len, int64_t &magic, int64_t k) +{ + if (server == NULL) + return -EC_NOT_INITIALIZED; + if (server->keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + if (cmd == DRequest::SvrAdmin) + return -EC_BAD_COMMAND; + DTCValue v(k); + return encode_buffer(ptr, len, magic, &v); +} + +int NCRequest::encode_buffer(char *&ptr, int &len, int64_t &magic, const char *k, int l) +{ + if (server == NULL) + return -EC_NOT_INITIALIZED; + if (server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + DTCValue v(k, l); + return encode_buffer(ptr, len, magic, &v); +} +/* + * @descr: 无源模式设置超时时间 + * @param: key, dtc key value + * @param: t, 超时时间，相对时间 + */ +int NCRequest::set_expire_time(const char *key, int t) +{ + int ret = 0; + if (key == NULL) + { + log_error("invalid key value! key is null"); + return -EC_INVALID_KEY_VALUE; + } + if (t <= 0 || t > MAX_EXPIRETIME) + { + log_error("invalid expireTime! expireTime:%d", t); + return -EC_INVALID_EXPIRETIME; + } + if (server->tdef == NULL) + { + ret = server->ping(); + if (ret != 0) + { + log_error("set_expire_time ping fail! ret:%d", ret); + } + } + if (server->tdef == NULL) + { + log_error("set_expire_time get server tdef fail!"); + return -EC_EMPTY_TBDEF; + } + // field "_dtc_sys_expiretime" in array with index=1 + const char *fdname = server->tdef->field_name(1); + if (strcmp(fdname, "_dtc_sys_expiretime")) + { + log_error("bussiness error: invalid option without field '_dtc_sys_expiretime', set_expire_time fail!"); + return -EC_BUSINESS_WITHOUT_EXPIRETIME; + } + DTC::Server srv; + int kt = server->keytype; + switch (kt) + { + case DField::Signed: + case DField::Unsigned: + ret = srv.int_key(); + break; + case DField::String: + ret = srv.string_key(); + break; + case DField::Binary: + ret = srv.binary_key(); + break; + default: + break; + } + const char *address = server->get_address(); + const char *tbname = server->get_table_name(); + int timeout = server->get_timeout(); + ret = srv.set_address(address); + ret = srv.set_table_name(tbname); + srv.set_access_key(server->accessToken.c_str()); + srv.set_timeout(timeout); + DTC::UpdateRequest req(&srv); + switch (kt) + { + case DField::Signed: + case DField::Unsigned: + ret = req.set_key(atoi(key)); + break; + case DField::String: + case DField::Binary: + ret = req.set_key(key); + break; + default: + break; + } + + ret = req.Set("_dtc_sys_expiretime", t); + DTC::Result rst; + ret = req.execute(rst); + if (ret != 0) + { + log_error("set expireTime fail, errmsg:%s, errfrom:%s", rst.error_message(), rst.error_from()); + } + return ret; +} + +/* + * @descr: 无源模式获取超时时间 + * @param: key, dtc key value + */ +int NCRequest::get_expire_time(const char *key) +{ + int ret = 0; + if (key == NULL) + { + log_error("invalid key value! key is null"); + return -EC_INVALID_KEY_VALUE; + } + if (server->tdef == NULL) + { + ret = server->ping(); + if (ret != 0) + { + log_error("get_expire_time ping fail! ret:%d", ret); + } + } + if (server->tdef == NULL) + { + log_error("get_expire_time get server tdef fail!"); + return -EC_EMPTY_TBDEF; + } + // field "_dtc_sys_expiretime" in array with index=1 + const char *fdname = server->tdef->field_name(1); + if (strcmp(fdname, "_dtc_sys_expiretime")) + { + log_error("bussiness error: invalid option without field '_dtc_sys_expiretime', get_expire_time fail!"); + return -EC_BUSINESS_WITHOUT_EXPIRETIME; + } + DTC::Server srv; + int kt = server->keytype; + switch (kt) + { + case DField::Signed: + case DField::Unsigned: + ret = srv.int_key(); + break; + case DField::String: + ret = srv.string_key(); + break; + case DField::Binary: + ret = srv.binary_key(); + break; + default: + break; + } + const char *address = server->get_address(); + const char *tbname = server->get_table_name(); + int timeout = server->get_timeout(); + ret = srv.set_address(address); + ret = srv.set_table_name(tbname); + srv.set_access_key(server->accessToken.c_str()); + srv.set_timeout(timeout); + DTC::GetRequest req(&srv); + switch (kt) + { + case DField::Signed: + case DField::Unsigned: + ret = req.set_key(atoi(key)); + break; + case DField::String: + case DField::Binary: + ret = req.set_key(key); + break; + default: + break; + } + ret = req.Need("_dtc_sys_expiretime"); + DTC::Result rst; + ret = req.execute(rst); + if (ret != 0) + { + log_error("get expireTime fail, errmsg:%s, errfrom:%s", rst.error_message(), rst.error_from()); + return ret; + } + if (rst.num_rows() <= 0) + { + log_error("get expireTime fail, no data exist in dtc for key:%s", key); + return -EC_GET_EXPIRETIME_END_OF_RESULT; + } + ret = rst.fetch_row(); + if (ret < 0) + { + log_error("get expireTime fail, fetch_row error, errmsg:%s, errfrom:%s", rst.error_message(), rst.error_from()); + return ret; + } + int expiretime = 0; + expiretime = rst.int_value("_dtc_sys_expiretime"); + return expiretime; +} diff --git a/src/search_local/index_storage/api/c_api/dtc_srv.cc b/src/search_local/index_storage/api/c_api/dtc_srv.cc new file mode 100644 index 0000000..3ae6005 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_srv.cc @@ -0,0 +1,1056 @@ +/* + * ===================================================================================== + * + * Filename: dtc_srv.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include "dtc_int.h" +#include +#include + +using namespace std; + +const uint32_t ACCESS_KEY_LEN = 40; +#define CLIENT_CURVE 5 +#define AGENT_CURVE 8 + +const char *dtc_api_ver = "dtc-api-ver:4.3.4"; +const char *dtc_api_complite = "dtc-api-complite-date:"__DATE__ + " "__TIME__; + +int NCServer::_network_mode = 0; +char *NCServer::_server_address = NULL; +char *NCServer::_server_tablename = NULL; +extern "C" void set_network_mode(void) +{ + NCServer::_network_mode = 1; +} + +extern "C" void set_server_address(const char *address) +{ + NCServer::_server_address = strdup(address); +} + +extern "C" void set_server_tablename(const char *tablename) +{ + NCServer::_server_tablename = strdup(tablename); +} + +NCServer::NCServer(void) +{ + keytype = DField::None; + + tablename = NULL; + appname = NULL; + + autoUpdateTable = 1; + autoReconnect = 1; + tdef = NULL; + admin_tdef = NULL; + + unsigned int a = time(NULL); + a = rand_r(&a) + (long)this; + lastSN = rand_r(&a); + + completed = 0; + badkey = 0; + badname = 0; + autoping = 0; + errstr = NULL; + + timeout = 5000; + realtmo = 0; + + iservice = NULL; + executor = NULL; + netfd = -1; + lastAct = 0; + pingReq = NULL; + + ownerPool = NULL; + ownerId = 0; + dc = DataConnector::getInstance(); +} + +// copy most members, zero connection state +// copy address if use generic address +NCServer::NCServer(const NCServer &that) : addr(that.addr), + keyinfo(that.keyinfo) +{ +#define _MCOPY(x) this->x = that.x +#define _MZERO(x) this->x = 0 +#define _MNEG(x) this->x = -1 +#define _MDUP(x) this->x = that.x ? STRDUP(that.x) : NULL + + _MDUP(tablename); + _MDUP(appname); + _MCOPY(keytype); + _MCOPY(autoUpdateTable); + _MCOPY(autoReconnect); + + _MCOPY(completed); + _MCOPY(badkey); + _MCOPY(badname); + _MCOPY(autoping); + // errstr always compile time const string + _MCOPY(errstr); + + _MCOPY(tdef); + _MCOPY(admin_tdef); + if (tdef) + tdef->INC(); + if (admin_tdef) + admin_tdef->INC(); + + lastAct = time(NULL); + _MCOPY(timeout); + _MZERO(realtmo); // this is sockopt set to real netfd + _MCOPY(iservice); + _MCOPY(executor); + _MNEG(netfd); // real socket fd, cleared + _MZERO(pingReq); // cached ping request object + + unsigned int a = lastAct; + a = rand_r(&a) + (long)this; + lastSN = rand_r(&a); + + _MZERO(ownerPool); // serverpool linkage + _MZERO(ownerId); // serverpool linkage + +#undef _MCOPY +#undef _MZERO +#undef _MNEG +#undef _MDUP +} + +NCServer::~NCServer(void) +{ + DELETE(pingReq); + Close(); + FREE_IF(tablename); + FREE_IF(appname); + DEC_DELETE(tdef); + DEC_DELETE(admin_tdef); + DataConnector::getInstance()->send_data(); // 防止10s 间隔时间未到，程序先结束了少报一次 +} + +DataConnector *NCServer::dc = NULL; //DataConnector::getInstance(); + +void NCServer::clone_tab_def(const NCServer &source) +{ + DEC_DELETE(tdef); + tdef = source.tdef; + tdef->INC(); + if (tablename != NULL) + FREE(tablename); + tablename = STRDUP(source.tablename); + keytype = source.keytype; + completed = 1; +} + +int NCServer::set_address(const char *h, const char *p) +{ + if (h == NULL) + { + errstr = "No Host Specified"; + return -EC_BAD_HOST_STRING; + } + + char *oldhost = NULL; + if (addr.Name() != NULL) + { + // dup string put on stack + oldhost = strdupa(addr.Name()); + } + + const char *err = addr.set_address(h, p); + if (err) + { + this->errstr = err; + return -EC_BAD_HOST_STRING; + } + + // un-changed + if (oldhost != NULL && !strcmp(addr.Name(), oldhost)) + return 0; + + Close(); + + //set network model + executor = NULL; + if (!_network_mode && iservice && iservice->match_listening_ports(addr.Name(), NULL)) + { + executor = iservice->query_task_executor(); + DTCTableDefinition *t1 = iservice->query_table_definition(); + if (t1 != this->tdef) + { + DEC_DELETE(this->tdef); + this->tdef = t1; + } + DTCTableDefinition *t2 = iservice->query_admin_table_definition(); + if (t2 != this->tdef) + { + DEC_DELETE(this->admin_tdef); + this->admin_tdef = t2; + } + } + + return 0; +} + +int NCServer::int_key(void) +{ + if (keytype != DField::None && keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + keytype = DField::Signed; + if (tablename != NULL) + completed = 1; + return 0; +} + +int NCServer::string_key(void) +{ + if (keytype != DField::None && keytype != DField::String) + return -EC_BAD_KEY_TYPE; + keytype = DField::String; + if (tablename != NULL) + completed = 1; + return 0; +} + +int NCServer::add_key(const char *name, uint8_t type) +{ + int ret = keyinfo.add_key(name, type); + // TODO verify tabledef + if (ret == 0) + { + if (tablename != NULL) + completed = 1; + if (keytype == DField::None) + keytype = type; + } + return 0; +} + +int NCServer::set_table_name(const char *name) +{ + if (name == NULL) + return -EC_BAD_TABLE_NAME; + + if (tablename) + return mystrcmp(name, tablename, 256) == 0 ? 0 : -EC_BAD_TABLE_NAME; + + tablename = STRDUP(name); + + if (&NCServer::check_internal_service != 0 && !_network_mode) + { + check_internal_service(); + } + + if (keytype != DField::None) + completed = 1; + + return 0; +} + +int NCServer::field_type(const char *name) +{ + if (tdef == NULL) + ping(); + + if (tdef != NULL) + { + int id = tdef->field_id(name); + if (id >= 0) + return tdef->field_type(id); + } + + return DField::None; +} + +//fixed, sec --> msec +void NCServer::set_m_timeout(int n) +{ + timeout = n <= 0 ? 5000 : n; + update_timeout(); +} + +void NCServer::update_timeout(void) +{ + if (netfd >= 0 && realtmo != timeout) + { + struct timeval tv = {timeout / 1000, (timeout % 1000) * 1000}; + setsockopt(netfd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + if (timeout) + { + if (tv.tv_usec > 1000000) + { + tv.tv_usec += 100000; // add more 100ms + } + else + { + tv.tv_usec += 100; // add more 100us + } + if (tv.tv_usec >= 1000000) + { + tv.tv_usec -= 1000000; + tv.tv_sec += 1; + } + } + setsockopt(netfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + realtmo = timeout; + } +} + +void NCServer::Close(void) +{ + if (netfd >= 0) + { + close(netfd); + netfd = -1; + realtmo = 0; + } +} + +int NCServer::bind_temp_unix_socket(void) +{ + struct sockaddr_un tempaddr; + tempaddr.sun_family = AF_UNIX; + snprintf(tempaddr.sun_path, sizeof(tempaddr.sun_path), + "@dtcapi-%d-%d-%d", getpid(), netfd, (int)time(NULL)); + socklen_t len = SUN_LEN(&tempaddr); + tempaddr.sun_path[0] = 0; + return bind(netfd, (const sockaddr *)&tempaddr, len); +} + +int NCServer::Connect(void) +{ + if (ownerPool != NULL) + return -EC_PARALLEL_MODE; + if (executor != NULL) + return 0; + if (netfd >= 0) + return 0; + + int err = -EC_NOT_INITIALIZED; + + if (addr.socket_family() != 0) + { + netfd = addr.create_socket(); + if (netfd < 0) + { + log_error("create socket error: %d, %m", errno); + err = -errno; + } + else if (addr.socket_family() == AF_UNIX && is_dgram() && bind_temp_unix_socket() < 0) + { + log_error("bind unix socket error: %d, %m", errno); + err = -errno; + Close(); + } + else + { + update_timeout(); + if (addr.connect_socket(netfd) == 0) + { + return 0; + } + log_error("connect dtc server error: %d,%m", errno); + err = -errno; + if (err == -EINPROGRESS) + err = -ETIMEDOUT; + Close(); + } + } + return err; +} + +int NCServer::Reconnect(void) +{ + if (ownerPool != NULL) + return -EC_PARALLEL_MODE; + if (executor != NULL) + return 0; + if (netfd < 0) + return -ENOTCONN; + + if (addr.connect_socket(netfd) == 0) + { + return 0; + } + log_error("connect dtc server error: %d,%m", errno); + int err = -errno; + if (err == -EINPROGRESS) + err = -ETIMEDOUT; + Close(); + return err; +} + +int NCServer::send_packet_stream(Packet &pk) +{ + int err; +#ifndef SIOCOUTQ + if (1) //if(!is_dgram()) + { + char tmp[1]; + err = recv(netfd, tmp, sizeof(tmp), MSG_PEEK | MSG_DONTWAIT); + if (err == 0 || (err < 0 && errno != EAGAIN && errno != EWOULDBLOCK)) + { + if (!autoReconnect) + { + log_error("dtc server close connection: %d,%m", err); + return err; + } + else + { + log_debug("%s", "dtc server close connection, re-connect now."); + Close(); + err = Connect(); + if (err) + return err; + } + } + } +#endif + + while ((err = pk.Send(netfd)) == SendResultMoreData) + { + if (errno == EINPROGRESS) + { + log_error("socket[%d] send data timeout: %d,%m", netfd, errno); + // timeout + if (1) // if(!is_dgram()) + Close(); + return -ETIMEDOUT; + } + } + if (err != SendResultDone) + { + log_error("socket[%d] send data error: %d,%m", netfd, err); + err = -errno; + if (1) // if(!is_dgram()) + Close(); + return err; + } + return 0; +} + +// always return network error code +// EAGAIN must re-sent +int NCServer::decode_result_stream(NCResult &tk) +{ + if (netfd < 0) + { + tk.set_error(-ENOTCONN, "API::recving", "connection is closed"); + return -ENOTCONN; + } + + int err; + SimpleReceiver receiver(netfd); + + uint64_t beginTime = 0; + //log_debug("wait response time stamp = %lld", (long long)(beginTime = GET_TIMESTAMP())); + + while (1) + { + errno = 0; + err = tk.Decode(receiver); + + if (err == DecodeFatalError) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + + err = -errno; + if (err == 0) + err = -ECONNRESET; +#ifdef SIOCOUTQ + int value = 0; + ioctl(netfd, SIOCOUTQ, &value); + // unsent bytes + if (value > 0) + { + err = -EAGAIN; + tk.set_error(err, "API::sending", "client send packet error"); + Close(); + return err; + } +#endif + Close(); + // EAGAIN never return FatalError, should be DecodeWaitData + tk.set_error(err, "API::recving", "client recv packet error"); + return err; + } + + if (err == DecodeDataError) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + + // packet is valid + return 0; + } + + if (err == DecodeDone) + { + break; + } + + if (errno == EINPROGRESS || errno == EAGAIN) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + log_debug("use time %ldms", (long)((GET_TIMESTAMP() - beginTime) / 1000)); + + Close(); + tk.set_error(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + return -ETIMEDOUT; + } + } + + //log_debug("decode result use time: %ldms", (long)((GET_TIMESTAMP()-beginTime)/1000)); + + save_definition(&tk); + if (autoping) + { + time(&lastAct); + err = tk.versionInfo.keep_alive_timeout(); + if (err < 15) + err = 15; + lastAct += err - 1; + } + return 0; +} + +int NCServer::send_packet_dgram(SocketAddress *peer, Packet &pk) +{ + int err = 0; + + while ((err = pk.send_to(netfd, peer)) == SendResultMoreData) + { + if (errno == EINPROGRESS) + { + log_error("socket[%d] send data timeout: %d,%m", netfd, errno); + // timeout + return -ETIMEDOUT; + } + } + + if (err != SendResultDone) + { + log_error("socket[%d] send data error: %d,%m", netfd, err); + err = -errno; + return err; + } + return 0; +} + +static inline char *RecvFromFixedSize(int fd, int len, int &err, struct sockaddr *peeraddr, socklen_t *addrlen) +{ + int blen = len <= 0 ? 1 : len; + char *buf = (char *)MALLOC(blen); + blen = recvfrom(fd, buf, blen, 0, peeraddr, addrlen); + if (blen < 0) + { + err = errno; + free(buf); + buf = NULL; + } + else if (blen != len) + { + err = EIO; + free(buf); + buf = NULL; + } + else + { + err = 0; + } + + return buf; +} + +static char *RecvPacketPeek(int fd, int &len, int &err) +{ + // MSG_PEEK + MSG_TRUNC get next packet size + char dummy[1]; + len = recv(fd, dummy, 1, MSG_PEEK | MSG_TRUNC); + if (len < 0) + { + err = errno; + return NULL; + } + + return RecvFromFixedSize(fd, len, err, NULL, NULL); +} + +static char *RecvPacketPeekPeer(int fd, SocketAddress *peer, int &len, int &err) +{ + struct sockaddr peeraddr; + socklen_t sock_len; + + for (int n = 0; n < 10; n++) + { + char dummy[1]; + + sock_len = sizeof(peeraddr); + len = recvfrom(fd, dummy, 1, MSG_PEEK | MSG_TRUNC, &peeraddr, &sock_len); + if (len < 0) + { + err = errno; + return NULL; + } + + if (peer->Equal(&peeraddr, sock_len, SOCK_DGRAM)) + break; + } + + return RecvFromFixedSize(fd, len, err, &peeraddr, &sock_len); +} + +static char *RecvPacketPoll(int fd, int msec, int &len, int &err) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int n; + + while ((n = poll(&pfd, 1, msec)) < 0 && errno == EINTR) + /* LOOP IF EINTR */; + + if (n == 0) + { + err = EAGAIN; + return NULL; + } + if (n < 0) + { + err = errno; + return NULL; + } + + len = 0; + ioctl(fd, SIOCINQ, &len); + + return RecvFromFixedSize(fd, len, err, NULL, NULL); +} + +// always return network error code +// EAGAIN must re-sent +int NCServer::decode_result_dgram(SocketAddress *peer, NCResult &tk) +{ + if (netfd < 0) + { + tk.set_error(-ENOTCONN, "API::recving", "connection is closed"); + return -ENOTCONN; + } + + int saved_errno; // saved errno + int len; // recv len + char *buf; // packet buffer + uint64_t beginTime = 0; + + log_debug("wait response time stamp = %lld", (long long)(beginTime = GET_TIMESTAMP())); + + if (addr.socket_family() == AF_UNIX) + { + if (peer == NULL) + { + buf = RecvPacketPoll(netfd, timeout, len, saved_errno); + } + else + { + if (peer->connect_socket(netfd) < 0) + { + saved_errno = errno; + log_error("connect dtc server error: %d,%m", saved_errno); + if (saved_errno == EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + saved_errno = EC_SERVER_BUSY; + } + tk.set_error(-saved_errno, "API::connecting", "client api connect server error"); + return -saved_errno; + } + + buf = RecvPacketPoll(netfd, timeout, len, saved_errno); + + // Un-connect + struct sockaddr addr; + addr.sa_family = AF_UNSPEC; + connect(netfd, &addr, sizeof(addr)); + } + } + else + { + if (peer == NULL) + { + buf = RecvPacketPeek(netfd, len, saved_errno); + } + else + { + buf = RecvPacketPeekPeer(netfd, peer, len, saved_errno); + } + } + log_debug("receive result use time: %ldms", (long)((GET_TIMESTAMP() - beginTime) / 1000)); + + if (buf == NULL) + { + errno = saved_errno; + log_error("socket[%d] recv udp data error: %d,%m", netfd, saved_errno); + + if (saved_errno == EAGAIN) + { + // change errcode to TIMEDOUT; + saved_errno = ETIMEDOUT; + tk.set_error(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + } + else + { + if (saved_errno == 0) + saved_errno = ECONNRESET; + tk.set_error(-saved_errno, "API::recving", "connection reset by peer"); + } + return -saved_errno; + } + + int err = tk.Decode(buf, len, 1 /*eat buf*/); + + switch (err) + { + case DecodeFatalError: + log_error("socket[%d] decode udp data error: invalid packet content", netfd); + tk.set_error(-ECONNRESET, "API::recving", "invalid packet content"); + FREE_IF(buf); // buf not eaten + return -ECONNRESET; + + case DecodeWaitData: + default: + // UNREACHABLE, decode_packet never return these code + log_error("socket[%d] decode udp data error: %d, %m", netfd, errno); + tk.set_error(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + return -ETIMEDOUT; + + case DecodeDataError: + // packet maybe valid + break; + + case DecodeDone: + save_definition(&tk); + break; + } + + return 0; +} + +//Get udpport form udppool +NCUdpPort *NCServer::get_global_port() +{ + NCUdpPort *udpport = NCUdpPort::Get(addr.socket_family()); + + if (udpport) + { + netfd = udpport->fd; + lastSN = udpport->sn; + realtmo = udpport->timeout; + update_timeout(); + } + + return udpport; +} + +//Put the udpport to the udppool +void NCServer::put_global_port(NCUdpPort *udpport) +{ + if (udpport) + { + if (netfd > 0) + { + udpport->sn = lastSN; + udpport->timeout = realtmo; + netfd = -1; + udpport->Put(); + } + else + { + //if fd error, delete this udpport object + udpport->Eat(); + } + } +} + +void NCServer::save_definition(NCResult *tk) +{ + if (strcmp("*", tablename) != 0 && tk->result_code() == -EC_TABLE_MISMATCH) + { + badname = 1; + errstr = "Table Name Mismatch"; + return; + } + if (strcmp("*", tablename) != 0 && tk->result_code() == -EC_BAD_KEY_TYPE) + { + badkey = 1; + errstr = "Key Type Mismatch"; + return; + } + + DTCTableDefinition *t = tk->remote_table_definition(); + if (t == NULL) + return; + + if (t->is_admin_table()) + { + if (admin_tdef) + { + if (admin_tdef->is_same_table(t)) + return; + if (!autoUpdateTable) + { + badname = 1; + errstr = "Table Mismatch"; + tk->set_error(-EC_TABLE_MISMATCH, "API::executed", "admin_table Mismatch"); + return; + } + DEC_DELETE(admin_tdef); + } + admin_tdef = t; + admin_tdef->INC(); + } + else + { + if (tdef) + { + if (tdef->is_same_table(t)) + return; + if (!autoUpdateTable) + { + badname = 1; + errstr = "Table Mismatch"; + tk->set_error(-EC_TABLE_MISMATCH, "API::executed", "Table Mismatch"); + return; + } + DEC_DELETE(tdef); + } + tdef = t; + tdef->INC(); + + FREE(tablename); + tablename = STRDUP(tdef->table_name()); + keytype = tdef->key_type(); + + //bugfix， by ada + if (keytype == DField::Unsigned) + keytype = DField::Signed; + + if (keytype == DField::Binary) + keytype = DField::String; + } +} + +void NCServer::TryPing(void) +{ + if (autoping && netfd >= 0) + { + time_t now; + time(&now); + if (now >= lastAct) + ping(); + } +} + +int NCServer::ping(void) +{ + if (ownerPool != NULL) + return -EC_PARALLEL_MODE; + if (executor != NULL) + return 0; + NCRequest r(this, DRequest::Nop); + NCResult *t = r.execute_network(); + int ret = t->result_code(); + delete t; + return ret; +} + +NCResult *NCServer::decode_buffer(const char *buf, int len) +{ + NCResult *res = new NCResult(tdef); + + switch (len <= (int)sizeof(PacketHeader) ? DecodeFatalError : res->Decode(buf, len)) + { + default: + res->set_error(-EINVAL, "API::decoding", "invalid packet content"); + break; + case DecodeDataError: + case DecodeDone: + break; + } + return res; +} + +int NCServer::check_packet_size(const char *buf, int len) +{ + return NCResult::check_packet_size(buf, len); +} + +int NCServer::set_access_key(const char *token) +{ + int ret = 0; + std::string str; + if (token == NULL) + return -EC_BAD_ACCESS_KEY; + else + str = token; + if (str.length() != ACCESS_KEY_LEN) + { + log_error("Invalid accessKey!"); + accessToken = ""; + return -EC_BAD_ACCESS_KEY; + } + else + accessToken = str; + + /* 截取 AccessKey中前32位以后的bid， 以便上报至数据采集 dtchttpd */ + ret = dc->set_bussiness_id(accessToken); + return 0; +} + +DataConnector *DataConnector::pDataConnector = NULL; + +pthread_mutex_t wakeLock; +static void *Process(void *p) +{ + sigset_t sset; + sigfillset(&sset); + sigdelset(&sset, SIGSEGV); + sigdelset(&sset, SIGBUS); + sigdelset(&sset, SIGABRT); + sigdelset(&sset, SIGILL); + sigdelset(&sset, SIGCHLD); + sigdelset(&sset, SIGFPE); + pthread_sigmask(SIG_BLOCK, &sset, &sset); + + int ret = 0; + time_t next = 0; + struct timeval tv; + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + struct timespec ts; + ts.tv_sec = next; + ts.tv_nsec = 0; + + while (pthread_mutex_timedlock(&wakeLock, &ts) != 0) + { + gettimeofday(&tv, NULL); + if (tv.tv_sec >= next) + { + ret = DataConnector::getInstance()->send_data(); + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + } + ts.tv_sec = next; + ts.tv_nsec = 0; + } + pthread_mutex_unlock(&wakeLock); + return NULL; +} + +DataConnector::DataConnector() +{ +} + +DataConnector::~DataConnector() +{ +} + +int DataConnector::send_data() +{ + return 0; +} + +int DataConnector::set_report_info(const std::string str, const uint32_t curve, const uint64_t t) +{ + uint32_t bid = 0; + std::string stemp = str.substr(0, 8); + sscanf(stemp.c_str(), "%u", &bid); + struct bidCurve bc; + bc.bid = bid; + bc.curve = curve; + + if (mapBi.find(bc) != mapBi.end()) + { + do + { + ScopedLock lock(_lock); + mapBi[bc].TotalTime += t; + mapBi[bc].TotalRequests += 1; + } while (0); + } + return 0; +} + +void DataConnector::get_report_info(std::map &map) +{ + ScopedLock lock(_lock); + for (std::map::iterator it = mapBi.begin(); it != mapBi.end(); ++it) + { + struct businessStatistics bs; + map[it->first] = it->second; + /* mapBi 取出来之后，不管后面是否上报成功，该bid对应的数据都丢弃，重置为0 */ + mapBi[it->first] = bs; + } +} + +int DataConnector::set_bussiness_id(std::string str) +{ + ScopedLock lock(_lock); + if (mapBi.size() == 0) + { + pthread_mutex_init(&wakeLock, NULL); + if (pthread_mutex_trylock(&wakeLock) == 0) + { + int ret = pthread_create(&threadid, 0, Process, (void *)NULL); + if (ret != 0) + { + errno = ret; + return -1; + } + } + } + std::string stemp = str.substr(0, 8); + uint32_t bid = 0; + sscanf(stemp.c_str(), "%u", &bid); + + struct businessStatistics bs; + struct bidCurve bclient, bagent; + bclient.bid = bid; + bclient.curve = CLIENT_CURVE; + + bagent.bid = bid; + bagent.curve = AGENT_CURVE; + mapBi.insert(std::make_pair(bclient, bs)); + mapBi.insert(std::make_pair(bagent, bs)); + return 0; +} diff --git a/src/search_local/index_storage/api/c_api/dtc_wrap.cc b/src/search_local/index_storage/api/c_api/dtc_wrap.cc new file mode 100644 index 0000000..170d8b0 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_wrap.cc @@ -0,0 +1,1427 @@ +/* + * ===================================================================================== + * + * Filename: dtc_wrap.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +#include "dtc_api.h" +#include "dtc_int.h" +#include +#include +#include "compiler.h" + +using namespace DistributedTableCache; + +#define CAST(type, var) type *var = (type *)addr +#define CAST2(type, var, src) type *var = (type *)src->addr +#define CAST2Z(type, var, src) type *var = (type *)((src) ? (src)->addr : 0) +#define CAST3(type, var, src) type *var = (type *)src.addr + +//WRAPPER for log +__EXPORT +void DistributedTableCache::init_log(const char *app, const char *dir) +{ + ::_init_log_(app, dir); +} + +__EXPORT +void DistributedTableCache::set_log_level(int n) +{ + ::_set_log_level_(n); +} + +__EXPORT +int DistributedTableCache::set_key_value_max(unsigned int count) +{ + if (count >= 8 && count <= 10000) + { + NCKeyValueList::KeyValueMax = count; + } + else + { + return -EINVAL; + } + return 0; +}; + +// WRAPPER for Server +__EXPORT +Server::Server(void) +{ + NCServer *srv = new NCServer; + srv->INC(); + addr = srv; +} + +__EXPORT +Server::Server(const Server &that) +{ + CAST3(NCServer, old, that); + NCServer *srv = new NCServer(*old); + srv->INC(); + addr = srv; +} + +__EXPORT +Server::~Server(void) +{ + CAST(NCServer, srv); + DEC_DELETE(srv); +} + +__EXPORT +int Server::set_address(const char *host, const char *port) +{ + CAST(NCServer, srv); + return srv->set_address(host, port); +} + +__EXPORT +int Server::set_table_name(const char *name) +{ + CAST(NCServer, srv); + return srv->set_table_name(name); +} + +__EXPORT +void Server::set_compress_level(int level) +{ + CAST(NCServer, srv); + srv->set_compress_level(level); +} + +__EXPORT +const char *Server::get_address(void) const +{ + CAST(NCServer, srv); + return srv ? srv->get_address() : NULL; +} + +__EXPORT +const char *Server::get_table_name(void) const +{ + CAST(NCServer, srv); + return srv ? srv->get_table_name() : NULL; +} + +__EXPORT +const char *Server::get_server_address(void) const +{ + CAST(NCServer, srv); + return srv ? srv->get_server_address() : NULL; +} + +__EXPORT +const char *Server::get_server_table_name(void) const +{ + CAST(NCServer, srv); + return srv ? srv->get_server_table_name() : NULL; +} +__EXPORT +void Server::clone_tab_def(const Server &source) +{ + CAST(NCServer, srv); + CAST3(NCServer, ssrv, source); + srv->clone_tab_def(*ssrv); +} + +__EXPORT +int Server::int_key(void) +{ + CAST(NCServer, srv); + return srv->int_key(); +} + +__EXPORT +int Server::string_key(void) +{ + CAST(NCServer, srv); + return srv->string_key(); +} + +__EXPORT +int Server::binary_key(void) +{ + CAST(NCServer, srv); + return srv->string_key(); +} + +__EXPORT +int Server::add_key(const char *name, int type) +{ + CAST(NCServer, srv); + return srv->add_key(name, (uint8_t)type); +} + +__EXPORT +int Server::field_type(const char *name) +{ + CAST(NCServer, srv); + return srv->field_type(name); +} + +__EXPORT +const char *Server::error_message(void) const +{ + CAST(NCServer, srv); + return srv->error_message(); +} + +//fixed, sec ---> msec +__EXPORT +void Server::set_m_timeout(int n) +{ + CAST(NCServer, srv); + return srv->set_m_timeout(n); +} + +__EXPORT +void Server::set_timeout(int n) +{ + CAST(NCServer, srv); + return srv->set_m_timeout(n * 1000); +} + +__EXPORT +int Server::Connect(void) +{ + CAST(NCServer, srv); + return srv->Connect(); +} + +__EXPORT +void Server::Close(void) +{ + CAST(NCServer, srv); + return srv->Close(); +} + +__EXPORT +int Server::ping(void) +{ + CAST(NCServer, srv); + return srv->ping(); +} + +__EXPORT +void Server::auto_ping(void) +{ + CAST(NCServer, srv); + return srv->auto_ping(); +} + +__EXPORT +void Server::SetFD(int fd) +{ + CAST(NCServer, srv); + srv->SetFD(fd); +} + +__EXPORT +void Server::set_auto_update_tab(bool autoUpdate) +{ + CAST(NCServer, srv); + srv->set_auto_update_tab(autoUpdate); +} + +__EXPORT +void Server::set_auto_reconnect(int autoReconnect) +{ + CAST(NCServer, srv); + srv->set_auto_reconnect(autoReconnect); +} + +__EXPORT +void Server::set_access_key(const char *token) +{ + CAST(NCServer, srv); + srv->set_access_key(token); +} +// WRAPER for Request + +__EXPORT +Request::Request(Server *srv0, int op) +{ + CAST2Z(NCServer, srv, srv0); + NCRequest *req = new NCRequest(srv, op); + addr = (void *)req; +} + +Request::Request() +{ + NCRequest *req = new NCRequest(NULL, 0); + addr = (void *)req; +} + +__EXPORT +Request::~Request(void) +{ + CAST(NCRequest, req); + delete req; +} + +__EXPORT +int Request::attach_server(Server *srv0) +{ + CAST2Z(NCServer, srv, srv0); + CAST(NCRequest, r); + return r->attach_server(srv); +} + +__EXPORT +void Request::Reset(void) +{ + CAST(NCRequest, old); + if (old) + { + int op = old->cmd; + NCServer *srv = old->server; + srv->INC(); + old->~NCRequest(); + memset(old, 0, sizeof(NCRequest)); + new (addr) NCRequest(srv, op); + srv->DEC(); + } +} + +__EXPORT +void Request::Reset(int op) +{ + CAST(NCRequest, old); + + if (old) + { + NCServer *srv = old->server; + srv->INC(); + old->~NCRequest(); + memset(old, 0, sizeof(NCRequest)); + new (addr) NCRequest(srv, op); + srv->DEC(); + } +} + +__EXPORT +void Request::set_admin_code(int code) +{ + CAST(NCRequest, r); + r->set_admin_code(code); +} + +__EXPORT +void Request::set_hot_backup_id(long long v) +{ + CAST(NCRequest, r); + r->set_hot_backup_id(v); +} + +__EXPORT +void Request::set_master_hb_timestamp(long long t) +{ + CAST(NCRequest, r); + r->set_master_hb_timestamp(t); +} + +__EXPORT +void Request::set_slave_hb_timestamp(long long t) +{ + CAST(NCRequest, r); + r->set_slave_hb_timestamp(t); +} + +__EXPORT +void Request::no_cache(void) +{ + CAST(NCRequest, r); + return r->enable_no_cache(); +} + +__EXPORT +void Request::no_next_server(void) +{ + CAST(NCRequest, r); + return r->enable_no_next_server(); +} + +__EXPORT +int Request::Need(const char *name) +{ + CAST(NCRequest, r); + return r->Need(name, 0); +} + +__EXPORT +int Request::Need(const char *name, int vid) +{ + CAST(NCRequest, r); + return r->Need(name, vid); +} + +__EXPORT +int Request::field_type(const char *name) +{ + CAST(NCRequest, r); + return r->field_type(name); +} + +__EXPORT +void Request::limit(unsigned int st, unsigned int cnt) +{ + CAST(NCRequest, r); + return r->limit(st, cnt); +} + +__EXPORT +void Request::unset_key(void) +{ + CAST(NCRequest, r); + r->unset_key(); + r->unset_key_value(); +} + +__EXPORT +int Request::set_key(long long key) +{ + CAST(NCRequest, r); + return r->set_key((int64_t)key); +} + +__EXPORT +int Request::set_key(const char *key) +{ + CAST(NCRequest, r); + return r->set_key(key, strlen(key)); +} + +__EXPORT +int Request::set_key(const char *key, int len) +{ + CAST(NCRequest, r); + return r->set_key(key, len); +} + +__EXPORT +int Request::add_key_value(const char *name, long long v) +{ + CAST(NCRequest, r); + DTCValue key(v); + return r->add_key_value(name, key, KeyTypeInt); +} + +__EXPORT +int Request::add_key_value(const char *name, const char *v) +{ + CAST(NCRequest, r); + DTCValue key(v); + return r->add_key_value(name, key, KeyTypeString); +} + +__EXPORT +int Request::add_key_value(const char *name, const char *v, int len) +{ + CAST(NCRequest, r); + DTCValue key(v, len); + return r->add_key_value(name, key, KeyTypeString); +} + +__EXPORT +int Request::set_cache_id(long long id) +{ + CAST(NCRequest, r); + return r->set_cache_id(id); +} + +__EXPORT +Result *Request::execute(void) +{ + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->execute(); + return s; +} + +__EXPORT +Result *Request::execute(long long k) +{ + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->execute((int64_t)k); + return s; +} + +__EXPORT +Result *Request::execute(const char *k) +{ + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->execute(k, k ? strlen(k) : 0); + return s; +} + +__EXPORT +Result *Request::execute(const char *k, int l) +{ + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->execute(k, l); + return s; +} + +__EXPORT +int Request::execute(Result &s) +{ + CAST(NCRequest, r); + s.Reset(); + s.addr = r->execute(); + return s.result_code(); +} + +__EXPORT +int Request::execute(Result &s, long long k) +{ + CAST(NCRequest, r); + s.Reset(); + s.addr = r->execute((int64_t)k); + return s.result_code(); +} + +__EXPORT +int Request::execute(Result &s, const char *k) +{ + CAST(NCRequest, r); + s.Reset(); + s.addr = r->execute(k, k ? strlen(k) : 0); + return s.result_code(); +} + +__EXPORT +int Request::execute(Result &s, const char *k, int l) +{ + CAST(NCRequest, r); + s.Reset(); + s.addr = r->execute(k, l); + return s.result_code(); +} + +__EXPORT +const char *Request::error_message(void) const +{ + CAST(NCRequest, r); + if (r == NULL) + return NULL; + return r->error_message(); +} + +#define _DECLACT_(f0, t0, f1, f2, t1) \ + __EXPORT \ + int Request::f0(const char *n, t0 v) \ + { \ + CAST(NCRequest, r); \ + return r->f1(n, DField::f2, DField::t1, DTCValue::Make(v)); \ + } +#define _DECLACT2_(f0, t0, f1, f2, t1) \ + __EXPORT \ + int Request::f0(const char *n, t0 v, int l) \ + { \ + CAST(NCRequest, r); \ + return r->f1(n, DField::f2, DField::t1, DTCValue::Make(v, l)); \ + } + +_DECLACT_(EQ, long long, add_condition, EQ, Signed); +_DECLACT_(NE, long long, add_condition, NE, Signed); +_DECLACT_(GT, long long, add_condition, GT, Signed); +_DECLACT_(GE, long long, add_condition, GE, Signed); +_DECLACT_(LT, long long, add_condition, LT, Signed); +_DECLACT_(LE, long long, add_condition, LE, Signed); +_DECLACT_(EQ, const char *, add_condition, EQ, String); +_DECLACT_(NE, const char *, add_condition, NE, String); +_DECLACT2_(EQ, const char *, add_condition, EQ, String); +_DECLACT2_(NE, const char *, add_condition, NE, String); +_DECLACT_(Set, long long, add_operation, Set, Signed); +_DECLACT_(OR, long long, add_operation, OR, Signed); +_DECLACT_(Add, long long, add_operation, Add, Signed); + +__EXPORT +int Request::Sub(const char *n, long long v) +{ + CAST(NCRequest, r); + return r->add_operation(n, DField::Add, DField::Signed, DTCValue::Make(-(int64_t)v)); +} + +_DECLACT_(Set, double, add_operation, Set, Float); +_DECLACT_(Add, double, add_operation, Add, Float); + +__EXPORT +int Request::Sub(const char *n, double v) +{ + CAST(NCRequest, r); + return r->add_operation(n, DField::Add, DField::Signed, DTCValue::Make(-v)); +} + +_DECLACT_(Set, const char *, add_operation, Set, String); +_DECLACT2_(Set, const char *, add_operation, Set, String); +#undef _DECLACT_ +#undef _DECLACT2_ + +__EXPORT +int Request::compress_set(const char *n, const char *v, int len) +{ + CAST(NCRequest, r); + return r->compress_set(n, v, len); +} + +__EXPORT +int Request::compress_set_force(const char *n, const char *v, int len) +{ + CAST(NCRequest, r); + return r->compress_set_force(n, v, len); +} + +__EXPORT +int Request::set_multi_bits(const char *n, int o, int s, unsigned int v) +{ + if (s <= 0 || o >= (1 << 24) || o < 0) + return 0; + if (s >= 32) + s = 32; + + CAST(NCRequest, r); + // 1 byte 3 byte 4 byte + // s o v + uint64_t t = ((uint64_t)s << 56) + ((uint64_t)o << 32) + v; + return r->add_operation(n, DField::SetBits, DField::Unsigned, DTCValue::Make(t)); +} + +__EXPORT +int Request::set_expire_time(const char *key, int time) +{ + CAST(NCRequest, r); + return r->set_expire_time(key, time); +} + +__EXPORT +int Request::get_expire_time(const char *key) +{ + CAST(NCRequest, r); + return r->get_expire_time(key); +} + +// WRAPER for Result +__EXPORT +Result::Result(void) +{ + addr = NULL; +} + +__EXPORT +Result::~Result(void) +{ + Reset(); +} + +__EXPORT +void Result::set_error(int err, const char *via, const char *msg) +{ + Reset(); + addr = new NCResult(err, via, msg); +} + +__EXPORT +void Result::Reset(void) +{ + if (addr != NULL) + { + CAST(NCResult, task); + CAST(NCResultInternal, itask); + if (task->Role() == TaskRoleClient) + delete task; + else + delete itask; + } + addr = NULL; +} + +__EXPORT +int Result::fetch_row(void) +{ + CAST(NCResult, r); + if (r == NULL) + return -EC_NO_MORE_DATA; + if (r->result_code() < 0) + return r->result_code(); + if (r->result == NULL) + return -EC_NO_MORE_DATA; + return r->result->decode_row(); +} + +__EXPORT +int Result::rewind(void) +{ + CAST(NCResult, r); + if (r == NULL) + return -EC_NO_MORE_DATA; + if (r->result_code() < 0) + return r->result_code(); + if (r->result == NULL) + return -EC_NO_MORE_DATA; + return r->result->rewind(); +} + +__EXPORT +int Result::result_code(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->result_code(); +} + +__EXPORT +const char *Result::error_message(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return NULL; + int code = r->result_code(); + const char *msg = r->resultInfo.error_message(); + if (msg == NULL && code < 0) + { + char a[1]; + msg = strerror_r(-code, a, 1); + if (msg == a) + msg = NULL; + } + return msg; +} + +__EXPORT +const char *Result::error_from(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return NULL; + return r->resultInfo.error_from(); +} + +__EXPORT +long long Result::hot_backup_id() const +{ + CAST(NCResult, r); + return r->versionInfo.hot_backup_id(); +} + +__EXPORT +long long Result::master_hb_timestamp() const +{ + CAST(NCResult, r); + return r->versionInfo.master_hb_timestamp(); +} + +__EXPORT +long long Result::slave_hb_timestamp() const +{ + CAST(NCResult, r); + return r->versionInfo.slave_hb_timestamp(); +} + +__EXPORT +char *Result::server_info() const +{ + CAST(NCResult, r); + return r->resultInfo.server_info(); +} + +__EXPORT +long long Result::binlog_id() const +{ + DTCServerInfo *p = (DTCServerInfo *)server_info(); + return p ? p->binlog_id : 0; +} + +__EXPORT +long long Result::binlog_offset() const +{ + DTCServerInfo *p = (DTCServerInfo *)server_info(); + return p ? p->binlog_off : 0; +} +__EXPORT +long long Result::mem_size() const +{ + DTCServerInfo *p = (DTCServerInfo *)server_info(); + return p ? p->memsize : 0; +} +__EXPORT +long long Result::data_size() const +{ + DTCServerInfo *p = (DTCServerInfo *)server_info(); + return p ? p->datasize : 0; +} + +__EXPORT +int Result::num_rows(void) const +{ + CAST(NCResult, r); + if (r == NULL || r->result == NULL) + return 0; + return r->result->total_rows(); +} + +__EXPORT +int Result::total_rows(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->resultInfo.total_rows(); +} + +__EXPORT +long long Result::insert_id(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->resultInfo.insert_id(); +} + +__EXPORT +int Result::num_fields(void) const +{ + CAST(NCResult, r); + if (r == NULL || r->result == NULL) + return 0; + return r->result->num_fields(); +} + +__EXPORT +const char *Result::field_name(int n) const +{ + CAST(NCResult, r); + if (r == NULL || r->result == NULL) + return NULL; + if (n < 0 || n >= r->result->num_fields()) + return NULL; + return r->field_name(r->result->field_id(n)); +} + +__EXPORT +int Result::field_present(const char *name) const +{ + CAST(NCResult, r); + if (r == NULL || r->result == NULL) + return 0; + int id = r->field_id(name); + if (id < 0) + return 0; + return r->result->field_present(id); +} + +__EXPORT +int Result::field_type(int n) const +{ + CAST(NCResult, r); + if (r == NULL || r->result == NULL) + return FieldTypeNone; + if (n < 0 || n >= r->result->num_fields()) + return FieldTypeNone; + return r->field_type(r->result->field_id(n)); +} + +__EXPORT +int Result::affected_rows(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->resultInfo.affected_rows(); +} + +__EXPORT +long long Result::int_key(void) const +{ + CAST(NCResult, r); + const DTCValue *v = NULL; + if (r && r->flag_multi_key_result() && r->result) + { + v = r->result->field_value(0); + } + else if (r && r->result_key()) + { + v = r->result_key(); + } + if (v != NULL) + { + switch (r->field_type(0)) + { + case DField::Signed: + case DField::Unsigned: + return v->s64; + } + } + /* + if(r && r->result_key()) { + switch(r->field_type(0)) { + case DField::Signed: + case DField::Unsigned: + return r->result_key()->s64; + } + } +*/ + return 0; +} + +__EXPORT +const char *Result::string_key(void) const +{ + CAST(NCResult, r); + const DTCValue *v = NULL; + if (r && r->flag_multi_key_result() && r->result) + { + v = r->result->field_value(0); + } + else if (r && r->result_key()) + { + v = r->result_key(); + } + if (v != NULL) + { + switch (r->field_type(0)) + { + case DField::Binary: + case DField::String: + return v->bin.ptr; + } + } + + return NULL; +} + +__EXPORT +const char *Result::string_key(int *lp) const +{ + CAST(NCResult, r); + const DTCValue *v = NULL; + if (r && r->flag_multi_key_result() && r->result) + { + v = r->result->field_value(0); + } + else if (r && r->result_key()) + { + v = r->result_key(); + } + if (v != NULL) + { + switch (r->field_type(0)) + { + case DField::Binary: + case DField::String: + if (lp) + *lp = v->bin.len; + return v->bin.ptr; + } + } + + return NULL; +} + +const char *Result::string_key(int &l) const +{ + CAST(NCResult, r); + const DTCValue *v = NULL; + if (r && r->flag_multi_key_result() && r->result) + { + v = r->result->field_value(0); + } + else if (r && r->result_key()) + { + v = r->result_key(); + } + if (v != NULL) + { + switch (r->field_type(0)) + { + case DField::Binary: + case DField::String: + l = v->bin.len; + return v->bin.ptr; + } + } + + return NULL; +} + +__EXPORT +const char *Result::binary_key(void) const +{ + return string_key(); +} + +__EXPORT +const char *Result::binary_key(int *lp) const +{ + return string_key(lp); +} + +__EXPORT +const char *Result::binary_key(int &l) const +{ + return string_key(l); +} + +static inline int64_t GetIntValue(NCResult *r, int id) +{ + if (id >= 0) + { + const DTCValue *v; + if (id == 0 && !(r->result->field_present(0))) + v = r->result_key(); + else + v = r->result->field_value(id); + + if (v) + { + switch (r->field_type(id)) + { + case DField::Signed: + case DField::Unsigned: + return v->s64; + case DField::Float: + return llround(v->flt); + //return (int64_t)(v->flt); + case DField::String: + if (v->str.ptr) + return atoll(v->str.ptr); + } + } + } + return 0; +} + +static inline double GetFloatValue(NCResult *r, int id) +{ + if (id >= 0) + { + const DTCValue *v; + if (id == 0 && !(r->result->field_present(0))) + v = r->result_key(); + else + v = r->result->field_value(id); + + if (v) + { + switch (r->field_type(id)) + { + case DField::Signed: + return (double)v->s64; + case DField::Unsigned: + return (double)v->u64; + case DField::Float: + return v->flt; + case DField::String: + if (v->str.ptr) + return atof(v->str.ptr); + } + } + } + return NAN; +} + +static inline const char *GetStringValue(NCResult *r, int id, int *lenp) +{ + if (id >= 0) + { + const DTCValue *v; + if (id == 0 && !(r->result->field_present(0))) + v = r->result_key(); + else + v = r->result->field_value(id); + + if (v) + { + switch (r->field_type(id)) + { + case DField::String: + if (lenp) + *lenp = v->bin.len; + return v->str.ptr; + } + } + } + return NULL; +} + +static inline const char *GetBinaryValue(NCResult *r, int id, int *lenp) +{ + if (id >= 0) + { + const DTCValue *v; + if (id == 0 && !(r->result->field_present(0))) + v = r->result_key(); + else + v = r->result->field_value(id); + + if (v) + { + switch (r->field_type(id)) + { + case DField::String: + case DField::Binary: + if (lenp) + *lenp = v->bin.len; + return v->str.ptr; + } + } + } + return NULL; +} + +__EXPORT +long long Result::int_value(const char *name) const +{ + CAST(NCResult, r); + if (r && r->result) + return GetIntValue(r, r->field_id(name)); + return 0; +} + +__EXPORT +long long Result::int_value(int id) const +{ + CAST(NCResult, r); + if (r && r->result) + return GetIntValue(r, r->field_id_virtual(id)); + return 0; +} + +__EXPORT +double Result::float_value(const char *name) const +{ + CAST(NCResult, r); + if (r && r->result) + return GetFloatValue(r, r->field_id(name)); + return NAN; +} + +__EXPORT +double Result::float_value(int id) const +{ + CAST(NCResult, r); + if (r && r->result) + return GetFloatValue(r, r->field_id_virtual(id)); + return NAN; +} + +__EXPORT +const char *Result::string_value(const char *name, int *lenp) const +{ + if (lenp) + *lenp = 0; + CAST(NCResult, r); + if (r && r->result) + return GetStringValue(r, r->field_id(name), lenp); + return NULL; +} + +__EXPORT +const char *Result::string_value(int id, int *lenp) const +{ + if (lenp) + *lenp = 0; + CAST(NCResult, r); + if (r && r->result) + return GetStringValue(r, r->field_id_virtual(id), lenp); + return NULL; +} + +__EXPORT +const char *Result::string_value(const char *name, int &len) const +{ + return string_value(name, &len); +} + +__EXPORT +const char *Result::string_value(int id, int &len) const +{ + return string_value(id, &len); +} + +__EXPORT +const char *Result::string_value(const char *name) const +{ + return string_value(name, NULL); +} + +__EXPORT +const char *Result::string_value(int id) const +{ + return string_value(id, NULL); +} + +__EXPORT +const char *Result::binary_value(const char *name, int *lenp) const +{ + if (lenp) + *lenp = 0; + CAST(NCResult, r); + if (r && r->result) + return GetBinaryValue(r, r->field_id(name), lenp); + return NULL; +} + +//所以UnCompressBinaryValue接口的调用均会复制一份buffer，用户得到数据后需要手动释放 +__EXPORT +int Result::uncompress_binary_value(const char *name, char **buf, int *lenp) +{ + if (buf == NULL || lenp == NULL) + return -EC_UNCOMPRESS_ERROR; + + int iret = 0; + *lenp = 0; + *buf = NULL; + const char *buf_dup = NULL; + uint64_t compressflag = 0; + + CAST(NCResult, r); + if (r && r->result) + { + iret = r->init_compress(); + if (iret) + return iret; + if (name == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "field name is null"); + return -EC_UNCOMPRESS_ERROR; + } + int fieldId = r->field_id(name); + if (fieldId > 63) //fieldid must less than 64,because compressflag is 8 bytes uint + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "fieldid must less than 64,because compressflag is 8 bytes uint"); + return -EC_UNCOMPRESS_ERROR; + } + buf_dup = GetBinaryValue(r, fieldId, lenp); + if (buf_dup == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "binary_value of this field is NULL,please check fieldname and fieldtype is binary or not"); + return -EC_UNCOMPRESS_ERROR; + } + + if (r->compress_id() < 0) //没启用压缩 + { + *buf = (char *)MALLOC(*lenp); + if (*buf == NULL) + return -ENOMEM; + memcpy(*buf, buf_dup, *lenp); + return 0; + } + compressflag = GetIntValue(r, r->compress_id()); + //启用了压缩，但是该字段没有被压缩 + if (!(compressflag & (0x1 << fieldId))) + { + *buf = (char *)MALLOC(*lenp); + if (*buf == NULL) + return -ENOMEM; + memcpy(*buf, buf_dup, *lenp); + return 0; + } + + //需要解压 + iret = r->gzip->UnCompress(buf, lenp, buf_dup, *lenp); + if (iret == -111111) + return -EC_UNCOMPRESS_ERROR; + return iret; + } + return -EC_UNCOMPRESS_ERROR; +} + +//所以UnCompressBinaryValue接口的调用均会复制一份buffer，用户得到数据后需要手动释放 +//忽略compressflag +__EXPORT +int Result::uncompress_binary_value_force(const char *name, char **buf, int *lenp) +{ + if (buf == NULL || lenp == NULL) + return -EC_UNCOMPRESS_ERROR; + + int iret = 0; + *lenp = 0; + *buf = NULL; + const char *buf_dup = NULL; + + CAST(NCResult, r); + if (r && r->result) + { + if (name == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "field name is null"); + return -EC_UNCOMPRESS_ERROR; + } + iret = r->init_compress(); + if (iret) + return iret; + int fieldId = r->field_id(name); + buf_dup = GetBinaryValue(r, fieldId, lenp); + if (buf_dup == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "binary_value of this field is NULL,please check fieldname and fieldtype is binary or not"); + return -EC_UNCOMPRESS_ERROR; + } + //需要解压 + iret = r->gzip->UnCompress(buf, lenp, buf_dup, *lenp); + if (iret == -111111) + return -EC_UNCOMPRESS_ERROR; + return iret; + } + return -EC_UNCOMPRESS_ERROR; +} + +__EXPORT +const char *Result::uncompress_error_message() const +{ + CAST(NCResult, r); + if (r && r->gzip) + return r->gzip->error_message(); + return NULL; +} + +__EXPORT +const char *Result::binary_value(int id, int *lenp) const +{ + if (lenp) + *lenp = 0; + CAST(NCResult, r); + if (r && r->result) + return GetBinaryValue(r, r->field_id_virtual(id), lenp); + return NULL; +} + +__EXPORT +const char *Result::binary_value(const char *name, int &len) const +{ + return binary_value(name, &len); +} + +__EXPORT +const char *Result::binary_value(int id, int &len) const +{ + return binary_value(id, &len); +} + +__EXPORT +const char *Result::binary_value(const char *name) const +{ + return binary_value(name, NULL); +} + +__EXPORT +const char *Result::binary_value(int id) const +{ + return binary_value(id, NULL); +} + +__EXPORT +long long Result::Tag(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->get_api_tag(); +}; + +__EXPORT +void *Result::TagPtr(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return NULL; + return (void *)(long)r->get_api_tag(); +}; + +__EXPORT +long long Result::Magic(void) const +{ + CAST(NCResult, r); + if (r == NULL) + return 0; + return r->versionInfo.serial_nr(); +}; + +__EXPORT +long long Result::server_timestamp(void) const +{ + CAST(NCResult, r); + + if (r == NULL) + { + return 0; + } + + return r->resultInfo.Timestamp(); +}; + +__EXPORT +int Server::decode_packet(Result &s, const char *buf, int len) +{ + CAST(NCServer, srv); + s.Reset(); + s.addr = srv->decode_buffer(buf, len); + return s.result_code(); +} + +__EXPORT +int Server::check_packet_size(const char *buf, int len) +{ + return NCServer::check_packet_size(buf, len); +} + +__EXPORT +int Request::encode_packet(char *&ptr, int &len, long long &m) +{ + CAST(NCRequest, r); + int64_t mm; + int ret = r->encode_buffer(ptr, len, mm); + m = mm; + return ret; +} + +__EXPORT +int Request::encode_packet(char *&ptr, int &len, long long &m, long long k) +{ + CAST(NCRequest, r); + int64_t mm; + int ret = r->encode_buffer(ptr, len, mm, (int64_t)k); + m = mm; + return ret; +} + +__EXPORT +int Request::encode_packet(char *&ptr, int &len, long long &m, const char *k) +{ + CAST(NCRequest, r); + int64_t mm; + int ret = r->encode_buffer(ptr, len, mm, k, k ? strlen(k) : 0); + m = mm; + return ret; +} + +__EXPORT +int Request::encode_packet(char *&ptr, int &len, long long &m, const char *k, int l) +{ + CAST(NCRequest, r); + int64_t mm; + int ret = r->encode_buffer(ptr, len, mm, k, l); + m = mm; + return ret; +} diff --git a/src/search_local/index_storage/api/c_api/dtc_wrapp.cc b/src/search_local/index_storage/api/c_api/dtc_wrapp.cc new file mode 100644 index 0000000..bff6f13 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/dtc_wrapp.cc @@ -0,0 +1,250 @@ +/* + * ===================================================================================== + * + * Filename: dtc_wrapp.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +#include "dtc_api.h" +#include "dtc_pool.h" + +using namespace DistributedTableCache; + +#define CAST(type, var) type *var = (type *)addr +#define CAST2(type, var, src) type *var = (type *)src->addr +#define CAST3(type, var, src) type *var = (type *)src.addr + +// WRAPPER for Server +__EXPORT +ServerPool::ServerPool(int ms, int mr) +{ + NCPool *pl = new NCPool(ms, mr); + addr = pl; +} + +__EXPORT +ServerPool::~ServerPool(void) +{ + CAST(NCPool, pl); + delete pl; +} + +__EXPORT +int ServerPool::add_server(Server *srv, int mreq, int mconn) +{ + CAST(NCPool, pl); + CAST2(NCServer, s, srv); + + return pl->add_server(s, mreq, mconn); +} + +__EXPORT +int ServerPool::add_request(Request *req, long long tag) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if (r->server == NULL) + return -EC_NOT_INITIALIZED; + + return pl->add_request(r, tag, NULL); +} + +__EXPORT +int ServerPool::add_request(Request *req, void *tag) +{ + return add_request(req, (long long)(long)tag); +} + +__EXPORT +int ServerPool::add_request(Request *req, long long tag, long long k) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if (r->server == NULL) + return -EC_NOT_INITIALIZED; + if (r->server->keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k); + return pl->add_request(r, tag, &v); +} + +__EXPORT +int ServerPool::add_request(Request *req, void *tag, long long k) +{ + return add_request(req, (long long)(long)tag, k); +} + +__EXPORT +int ServerPool::add_request(Request *req, long long tag, const char *k) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if (r->server == NULL) + return -EC_NOT_INITIALIZED; + if (r->server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k); + return pl->add_request(r, tag, &v); +} + +__EXPORT +int ServerPool::add_request(Request *req, void *tag, const char *k) +{ + return add_request(req, (long long)(long)tag, k); +} + +__EXPORT +int ServerPool::add_request(Request *req, long long tag, const char *k, int l) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if (r->server == NULL) + return -EC_NOT_INITIALIZED; + if (r->server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k, l); + return pl->add_request(r, tag, &v); +} + +__EXPORT +int ServerPool::add_request(Request *req, void *tag, const char *k, int l) +{ + return add_request(req, (long long)(long)tag, k, l); +} + +__EXPORT +int ServerPool::execute(int msec) +{ + CAST(NCPool, pl); + return pl->execute(msec); +} + +__EXPORT +int ServerPool::execute_all(int msec) +{ + CAST(NCPool, pl); + return pl->execute_all(msec); +} + +__EXPORT +int ServerPool::cancel_request(int id) +{ + CAST(NCPool, pl); + return pl->cancel_request(id); +} + +__EXPORT +int ServerPool::cancel_all_request(int type) +{ + CAST(NCPool, pl); + return pl->cancel_all_request(type); +} + +__EXPORT +int ServerPool::abort_request(int id) +{ + CAST(NCPool, pl); + return pl->abort_request(id); +} + +__EXPORT +int ServerPool::abort_all_request(int type) +{ + CAST(NCPool, pl); + return pl->abort_all_request(type); +} + +__EXPORT +Result *ServerPool::get_result(void) +{ + return get_result(0); +} + +__EXPORT +Result *ServerPool::get_result(int id) +{ + CAST(NCPool, pl); + NCResult *a = pl->get_result(id); + if ((long)a < 4096 && (long)a > -4095) + return NULL; + Result *s = new Result(); + s->addr = (void *)a; + return s; +} + +__EXPORT +int ServerPool::get_result(Result &s) +{ + return get_result(s, 0); +} + +__EXPORT +int ServerPool::get_result(Result &s, int id) +{ + CAST(NCPool, pl); + s.Reset(); + NCResult *a = pl->get_result(id); + long iv = (long)a; + if (iv < 4096 && iv > -4095) + { + s.addr = (void *)(new NCResult(iv, "API::executing", iv > 0 ? "Request not completed" : id ? "Invalid request" : "No more result")); + return iv; + } + s.addr = (void *)a; + return 0; +} + +__EXPORT +int ServerPool::server_count(void) const +{ + CAST(NCPool, pl); + return pl->server_count(); +} + +__EXPORT +int ServerPool::request_count(int type) const +{ + CAST(NCPool, pl); + if (type == DONE) + return pl->done_request_count(); + if (type == (WAIT | SEND | RECV | DONE)) + return pl->request_count(); + return pl->count_request_state(type); +} + +__EXPORT +int ServerPool::request_state(int reqId) const +{ + CAST(NCPool, pl); + return pl->request_state(reqId); +} + +__EXPORT +int ServerPool::get_epoll_fd(int size) +{ + CAST(NCPool, pl); + return pl->get_epoll_fd(size); +} + +const int ALL_STATE = WAIT | SEND | RECV | DONE; diff --git a/src/search_local/index_storage/api/c_api/key_list.cc b/src/search_local/index_storage/api/c_api/key_list.cc new file mode 100644 index 0000000..c5c1430 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/key_list.cc @@ -0,0 +1,154 @@ +/* + * ===================================================================================== + * + * Filename: key_list.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "mem_check.h" +#include "protocol.h" +#include "key_list.h" +#include "buffer_error.h" + +int NCKeyValueList::KeyValueMax = 32; + +int NCKeyInfo::KeyIndex(const char *n) const +{ + namemap_t::const_iterator i = keyMap.find(n); + return i == keyMap.end() ? -1 : i->second; +} + +int NCKeyInfo::add_key(const char *name, int type) +{ + switch (type) + { + case DField::Signed: + case DField::String: + break; + default: + return -EC_BAD_KEY_TYPE; + } + if (KeyIndex(name) >= 0) + { + //return -EC_DUPLICATE_FIELD; + + // ignore duplicate key field name + // because NCKeyInfo may be initialized by check_internal_service() + // add again must be allowed for code compatibility + return 0; + } + int cnt = key_fields(); + if (cnt >= (int)(sizeof(keyType) / sizeof(keyType[0]))) + return -EC_BAD_MULTIKEY; + keyName[cnt] = name; + keyMap[name] = cnt; + keyType[cnt] = type; + keyCount++; + return 0; +} + +int NCKeyInfo::Equal(const NCKeyInfo &other) const +{ + int n = key_fields(); + // key field count mis-match + if (other.key_fields() != n) + return 0; + // key type mis-match + if (memcmp(keyType, other.keyType, n) != 0) + return 0; + for (int i = 0; i < n; i++) + { + // key name mis-match + if (strcasecmp(keyName[i], other.keyName[i]) != 0) + return 0; + } + return 1; +} + +void NCKeyValueList::Unset(void) +{ + if (kcount) + { + const int kn = key_fields(); + for (int i = 0; i < kcount; i++) + { + for (int j = 0; j < kn; j++) + if (key_type(j) == DField::String || key_type(j) == DField::Binary) + FREE(Value(i, j).bin.ptr); + } + + FREE_CLEAR(val); + memset(fcount, 0, sizeof(fcount)); + kcount = 0; + } +} + +int NCKeyValueList::add_value(const char *name, const DTCValue &v, int type) +{ + const int kn = key_fields(); + + int col = keyinfo->KeyIndex(name); + if (col < 0 || col >= kn) + return -EC_BAD_KEY_NAME; + + switch (key_type(col)) + { + case DField::Signed: + case DField::Unsigned: + if (type != DField::Signed && type != DField::Unsigned) + return -EC_BAD_VALUE_TYPE; + break; + case DField::String: + case DField::Binary: + if (type != DField::String && type != DField::Binary) + return -EC_BAD_VALUE_TYPE; + if (v.bin.len > 255) + return -EC_KEY_OVERFLOW; + break; + default: + return -EC_BAD_KEY_TYPE; + } + + int row = fcount[col]; + if (row >= KeyValueMax) + return -EC_TOO_MANY_KEY_VALUE; // key值太多 + if (row >= kcount) + { + if (REALLOC(val, (kcount + 1) * kn * sizeof(DTCValue)) == NULL) + throw std::bad_alloc(); + memset(&Value(row, 0), 0, kn * sizeof(DTCValue)); + kcount++; + } + DTCValue &slot = Value(row, col); + switch (key_type(col)) + { + case DField::Signed: + case DField::Unsigned: + slot = v; + break; + + case DField::String: + case DField::Binary: + slot.bin.len = v.bin.len; + slot.bin.ptr = (char *)MALLOC(v.bin.len + 1); + if (slot.bin.ptr == NULL) + throw std::bad_alloc(); + memcpy(slot.bin.ptr, v.bin.ptr, v.bin.len); + slot.bin.ptr[v.bin.len] = '\0'; + break; + } + fcount[col]++; + return 0; +} diff --git a/src/search_local/index_storage/api/c_api/key_list.h b/src/search_local/index_storage/api/c_api/key_list.h new file mode 100644 index 0000000..7fccedf --- /dev/null +++ b/src/search_local/index_storage/api/c_api/key_list.h @@ -0,0 +1,117 @@ +/* + * ===================================================================================== + * + * Filename: key_list.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CHC_KEYLIST_H +#define __CHC_KEYLIST_H + +#include +#include + +#include + +class NCKeyInfo +{ +private: + struct nocase + { + bool operator()(const char *const &a, const char *const &b) const + { + return strcasecmp(a, b) < 0; + } + }; + typedef std::map namemap_t; + +private: + namemap_t keyMap; + const char *keyName[8]; + uint8_t keyType[8]; + int keyCount; + +public: + // zero is KeyField::None + void Clear(void) + { + keyCount = 0; + memset(keyType, 0, sizeof(keyType)); + memset(keyName, 0, sizeof(keyName)); + keyMap.clear(); + } + NCKeyInfo() + { + keyCount = 0; + memset(keyType, 0, sizeof(keyType)); + memset(keyName, 0, sizeof(keyName)); + } + NCKeyInfo(const NCKeyInfo &that) + { + keyCount = that.keyCount; + memcpy(keyType, that.keyType, sizeof(keyType)); + memcpy(keyName, that.keyName, sizeof(keyName)); + for (int i = 0; i < keyCount; i++) + keyMap[keyName[i]] = i; + } + ~NCKeyInfo() {} + + int add_key(const char *name, int type); + int Equal(const NCKeyInfo &other) const; + int KeyIndex(const char *n) const; + const char *key_name(int n) const { return keyName[n]; } + int key_type(int id) const { return keyType[id]; } + int key_fields(void) const { return keyCount; } +}; + +class NCKeyValueList +{ +public: + static int KeyValueMax; + +public: + NCKeyInfo *keyinfo; + DTCValue *val; + int fcount[8]; + int kcount; + +public: + NCKeyValueList(void) : keyinfo(NULL), val(NULL), kcount(0) + { + memset(fcount, 0, sizeof(fcount)); + } + ~NCKeyValueList() + { + FREE_CLEAR(val); + } + int key_fields(void) const { return keyinfo->key_fields(); } + int key_type(int id) const { return keyinfo->key_type(id); } + int key_count(void) const { return kcount; } + const char *key_name(int id) const { return keyinfo->key_name(id); } + + void Unset(void); + int add_value(const char *, const DTCValue &, int); + DTCValue &Value(int row, int col) { return val[row * keyinfo->key_fields() + col]; } + const DTCValue &Value(int row, int col) const { return val[row * keyinfo->key_fields() + col]; } + DTCValue &operator()(int row, int col) { return Value(row, col); } + const DTCValue &operator()(int row, int col) const { return Value(row, col); } + int IsFlat(void) const + { + for (int i = 1; i < keyinfo->key_fields(); i++) + if (fcount[0] != fcount[i]) + return 0; + return 1; + } +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api/log_client.cc b/src/search_local/index_storage/api/c_api/log_client.cc new file mode 100644 index 0000000..841a9ab --- /dev/null +++ b/src/search_local/index_storage/api/c_api/log_client.cc @@ -0,0 +1,26 @@ +/* + * ===================================================================================== + * + * Filename: log_client.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "dtc_api.h" +#include + +__EXPORT +void DistributedTableCache::write_log(int level, + const char *file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf, 5, 6))) + __attribute__((__alias__("_write_log_"))); diff --git a/src/search_local/index_storage/api/c_api/readme.txt b/src/search_local/index_storage/api/c_api/readme.txt new file mode 100644 index 0000000..62b9dcc --- /dev/null +++ b/src/search_local/index_storage/api/c_api/readme.txt @@ -0,0 +1,3 @@ +注意事项： +1.auto_dtcc++.sh 该脚本是需要root权限去执行的，它的作用是将dtc sdk 的动态库 +放置在 /usr/lib 目录下，并且执行命令 ldconfig 使配置生效 \ No newline at end of file diff --git a/src/search_local/index_storage/api/c_api/somain.c b/src/search_local/index_storage/api/c_api/somain.c new file mode 100644 index 0000000..3de03cc --- /dev/null +++ b/src/search_local/index_storage/api/c_api/somain.c @@ -0,0 +1,49 @@ +/* + * ===================================================================================== + * + * Filename: somain.c + * + * Description: entry function. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#undef _GNU_SOURCE +#define _GNU_SOURCE +#include +#include +#include +#include + +const char __invoke_dynamic_linker__[] + __attribute__((section(".interp"))) + __HIDDEN = +#if __x86_64__ + "/lib64/ld-linux-x86-64.so.2" +#else + "/lib/ld-linux.so.2" +#endif + ; + +__HIDDEN +void _so_start(char *arg1, ...) +{ +#define BANNER "DTC client API v" DTC_VERSION_DETAIL "\n" \ + " - TCP connection supported\n" \ + " - UDP connection supported\n" \ + " - UNIX stream connection supported\n" \ + " - embeded threading connection supported\n" \ + " - protocol packet encode/decode interface supported\n" \ + " - async execute (except embeded threading) supported \n" + + int unused; + unused = write(1, BANNER, sizeof(BANNER) - 1); + _exit(0); +} diff --git a/src/search_local/index_storage/api/c_api/udp_pool.cc b/src/search_local/index_storage/api/c_api/udp_pool.cc new file mode 100644 index 0000000..b137593 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/udp_pool.cc @@ -0,0 +1,257 @@ +/* + * ===================================================================================== + * + * Filename: udp_pool.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "udp_pool.h" +#include "log.h" + +static unsigned int bindip = 0xFFFFFFFF; + +unsigned int GetBindIp(void) +{ + const char *name = getenv("DTCAPI_UDP_INTERFACE"); + + if (name == NULL || name[0] == 0 || strcmp(name, "*") == 0) + { + return 0; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + struct ifconf ifc; + struct ifreq *ifr = NULL; + int n = 0; + int i; + + if (fd < 0) + return 0; + + ifc.ifc_len = 0; + ifc.ifc_req = NULL; + + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + { + ifr = (struct ifreq *) alloca(ifc.ifc_len > 128 ? ifc.ifc_len : 128); + ifc.ifc_req = ifr; + + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + n = ifc.ifc_len / sizeof(struct ifreq); + } + + close(fd); + + for (i = 0; i < n; i++) + { + if (strncmp(ifr[i].ifr_name, name, sizeof(ifr[i].ifr_name)) != 0) + continue; + + if (ifr[i].ifr_addr.sa_family == AF_INET) + return ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr; + } + + return 0; +} + +static int CreatePortIpv4(void) +{ + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (bindip == 0xFFFFFFFF) + bindip = GetBindIp(); + + if (bindip != 0) + { + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = bindip; + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) + { + close(fd); + return -1; + } + } + + return fd; +} + +static int CreatePortIpv6(void) +{ + int fd = socket(AF_INET6, SOCK_DGRAM, 0); + + return fd; +} + +static int CreatePortUnix(void) +{ + int fd = socket(AF_UNIX, SOCK_DGRAM, 0); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), + "@dtcapi-global-%d-%d-%d", getpid(), fd, (int)time(NULL)); + socklen_t len = SUN_LEN(&addr); + addr.sun_path[0] = 0; + if (bind(fd, (const struct sockaddr *)&addr, len) < 0) + { + close(fd); + return -1; + } + + return fd; +} + + +struct NCUdpPortList { +public: + int stopped; + int family; + int (*newport)(void); + pthread_mutex_t lock; + NCUdpPort *list; +public: + NCUdpPort *Get(void); + void Put(NCUdpPort *); +public: + ~NCUdpPortList(void); +}; + +// this destructor only called when unloading libdtc.so +NCUdpPortList::~NCUdpPortList(void) +{ + if (pthread_mutex_lock(&lock) == 0) + { + stopped = 1; + while (list != NULL) + { + NCUdpPort *port = list; + list = port->next; + delete port; + } + pthread_mutex_unlock(&lock); + } +} + +NCUdpPort *NCUdpPortList::Get(void) +{ + NCUdpPort *port = NULL; + + if (pthread_mutex_lock(&lock) == 0) + { + if (list != NULL) + { + port = list; + list = port->next; + } + pthread_mutex_unlock(&lock); + } + else + { + log_error("api mutex_lock error,may have fd leak"); + } + + if (port != NULL) + { + if (getpid() == port->pid) + { + port->sn++; + } + else + { + delete port; + port = NULL; + } + } + + if (port == NULL) + { + int fd = newport(); + + if (fd > 0) + { + port = new NCUdpPort; + port->fd = fd; + unsigned int seed = fd + (long)port + (long) & port + (long)pthread_self() + (long)port; + port->sn = rand_r(&seed); + port->pid = getpid(); + port->timeout = -1; + port->family = family; + } + } + + return port; +} + +void NCUdpPortList::Put(NCUdpPort *port) +{ + if (this != NULL && pthread_mutex_lock(&lock) == 0) + { + if(stopped) { + // always delete port after unloading process + port->Eat(); + } else { + port->next = list; + list = port; + } + pthread_mutex_unlock(&lock); + } + else + { + port->Eat(); + } +} + +static NCUdpPortList ipv4List = { 0, AF_INET, CreatePortIpv4, PTHREAD_MUTEX_INITIALIZER, NULL }; +static NCUdpPortList ipv6List = { 0, AF_INET6, CreatePortIpv6, PTHREAD_MUTEX_INITIALIZER, NULL }; +static NCUdpPortList unixList = { 0, AF_UNIX, CreatePortUnix, PTHREAD_MUTEX_INITIALIZER, NULL }; + +struct NCUdpPortList *GetPortList(int family) { + switch(family) { + case AF_INET: + return &ipv4List; + case AF_INET6: + return &ipv6List; + case AF_UNIX: + return &unixList; + } + return NULL; +} + +NCUdpPort *NCUdpPort::Get(int family) +{ + NCUdpPortList *portList = GetPortList(family); + if(portList == NULL) { + return NULL; + } + + return portList->Get(); +} + +void NCUdpPort::Put() +{ + NCUdpPortList *portList = GetPortList(family); + portList->Put(this); +} diff --git a/src/search_local/index_storage/api/c_api/udp_pool.h b/src/search_local/index_storage/api/c_api/udp_pool.h new file mode 100644 index 0000000..17c42e3 --- /dev/null +++ b/src/search_local/index_storage/api/c_api/udp_pool.h @@ -0,0 +1,76 @@ +/* + * ===================================================================================== + * + * Filename: udp_pool.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +class NCPort +{ +public: + uint64_t sn; + int fd; + int timeout; //the timeout this socket is. + +public: + NCPort() + { + sn = 0; + fd = -1; + timeout = -1; + } + + NCPort(const NCPort &that) + { + sn = that.sn; + fd = that.fd; + timeout = that.timeout; + } + + ~NCPort() + { + if (fd >= 0) + { + close(fd); + } + } +}; + +class NCUdpPortList; +class NCUdpPort : public NCPort +{ +public: + int family; + pid_t pid; + +private: + NCUdpPort *next; + +private: + NCUdpPort() + { + pid = -1; + }; + ~NCUdpPort(){}; + +public: + friend class NCUdpPortList; + static NCUdpPort *Get(int family); // get from cache + void Put(); // put back cache + void Eat() + { + delete this; + } // eat and delete +}; diff --git a/src/search_local/index_storage/api/c_api_cc/Makefile b/src/search_local/index_storage/api/c_api_cc/Makefile new file mode 100644 index 0000000..bc67639 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/Makefile @@ -0,0 +1,42 @@ +LIB_PATH = ../../../.. + +include ../../Make.conf + +GIT_VERSION=$(shell git log . | head -n 4 | grep "commit" | cut -d " " -f 2 | cut -c 1-7) +SVN_REVISION = $(shell test -d .svn && (svn info | grep "Last Changed Rev" | cut -d " " -f 4)) +ifeq "$(SVN_REVISION)a" "a" + SVN_REVISION = "(unknown)" +endif + +VPATH = ../../common +################compile############# +target = libdtc.a container_api.pic.o version.pic.o somain.pic.o libdtc.so libdtc.pic.a + +# CLIENTAPI macro use for scope test only +CFLAGS += -DCLIENTAPI +CFLAGS += -pthread -I../../common -I../../stat -I../../../../3rdlib/CA_API/ $(ZINC) +LIBS = ${Z_LIB} -ldl -lpthread $(CA_API_LIB) + +filelist := dtcreq dtcsvr dtcpool dtcwrap dtcwrapp dtcqossvr \ + poller timerlist keylist table_def \ + log_client memcheck md5 value \ + section decode encode field_api \ + packet_base packet_client \ + task_base task_const sockaddr udppool compress buffer thread + +#liddtc.so: container_api.pic.o version.pic.o + +# for auto ln -sf +#libdtc_objs := $(patsubst %,%.o,$(filelist)) +#libdtc.so: LDFLAGS += -Wl,--version-script,dtcapi.lst -e _so_start container_api.pic.o version.pic.o somain.pic.o +#libdtc_filename := libdtc-gcc-$(GCCVER)-r$(GIT_VERSION).so +#libdtc_soname := libdtc.so.1 + +###############install############## +target_install = libdtc.a libdtc.pic.a libdtc.so dtcapi.h +install_dir = ../../../bin +%.a: install_dir = ../../../lib +%.h: install_dir = ../../../include + +include ../../Make.rules + diff --git a/src/search_local/index_storage/api/c_api_cc/container_api.cc b/src/search_local/index_storage/api/c_api_cc/container_api.cc new file mode 100644 index 0000000..e8e4cce --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/container_api.cc @@ -0,0 +1,101 @@ + +#include +#include + +#include "container.h" +#include "version.h" +#include "dtcint.h" + +typedef IInternalService *(*QueryInternalServiceFunctionType)(const char *name, const char *instance); + +IInternalService *QueryInternalService(const char *name, const char *instance) +{ + QueryInternalServiceFunctionType entry = NULL; + entry = (QueryInternalServiceFunctionType)dlsym(RTLD_DEFAULT, "_QueryInternalService"); + if(entry == NULL) + return NULL; + return entry(name, instance); +} + +static inline int fieldtype2keytype(int t) +{ + switch(t) { + case DField::Signed: + case DField::Unsigned: + return DField::Signed; + + case DField::String: + case DField::Binary: + return DField::String; + } + + return DField::None; +} + +void NCServer::CheckInternalService(void) +{ + if(NCResultInternal::VerifyClass()==0) + return; + + IInternalService *inst = QueryInternalService("dtcd", this->tablename); + + // not inside dtcd or tablename not found + if(inst == NULL) + return; + + // version mismatch, internal service is unusable + const char *version = inst->query_version_string(); + if(version==NULL || strcasecmp(version_detail, version) != 0) + return; + + // cast to DTC service + IDTCService *inst1 = static_cast(inst); + + DTCTableDefinition *tdef = inst1->query_table_definition(); + + // verify tablename etc + if(tdef->is_same_table(tablename)==0) + return; + + // verify and save key type + int kt = fieldtype2keytype(tdef->key_type()); + if(kt == DField::None) + // bad key type + return; + + if(keytype == DField::None) { + keytype = kt; + } else if(keytype != kt) { + badkey = 1; + errstr = "Key Type Mismatch"; + return; + } + + if(keyinfo.KeyFields()!=0) { + // FIXME: checking keyinfo + + // ZAP key info, use ordered name from server + keyinfo.Clear(); + } + // add NCKeyInfo + for(int i=0; ikey_fields(); i++) { + kt = fieldtype2keytype(tdef->field_type(i)); + if(kt == DField::None) + // bad key type + return; + keyinfo.AddKey(tdef->field_name(i), kt); + } + + // OK, save it. + // old tdef always NULL, because tablename didn't set, Server didn't complete + this->tdef = tdef; + this->admin_tdef = inst1->query_admin_table_definition(); + // useless here, internal DTCTableDefinition don't maintent a usage count + tdef->INC(); + this->iservice = inst1; + this->completed = 1; + if(GetAddress() && iservice->match_listening_ports(GetAddress(), NULL)) { + executor = iservice->query_task_executor(); + } +} + diff --git a/src/search_local/index_storage/api/c_api_cc/dtcapi.h b/src/search_local/index_storage/api/c_api_cc/dtcapi.h new file mode 100644 index 0000000..ca981ac --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcapi.h @@ -0,0 +1,1021 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include +#include +#include +#include + +#define IP_LENGHT 16 + +namespace DTC { + + typedef struct route_node + { + int bid; + int port; + int status; + int weight; + char ip[IP_LENGHT]; + }ROUTE_NODE; + + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int ChangeNodeAddress = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + const int ClearCache = 21; + const int ColExpandStatus = 24; + const int ColExpand = 25; + const int ColExpandDone = 26; + const int ColExpandKey = 27; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) +#ifndef WIN32 + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); +#endif + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + friend class QOSServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + //for compress + void SetCompressLevel(int); + //get address and tablename set by user + const char * GetAddress(void) const; + const char * GetTableName(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char * GetServerAddress(void) const; + const char * GetServerTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + + void SetAccessKey(const char *token); + + public: + void IncErrCount(); + uint64_t GetErrCount(); + void ClearErrCount(); + + void IncRemoveCount(); + int GetRemoveCount(); + void ClearRemoveCount(); + + void IncTotalReq(); + uint64_t GetTotalReq(); + void ClearTotalReq(); + + void AddTotalElaps(uint64_t iElaps); + uint64_t GetTotalElaps(); + void ClearTotalElaps(); + }; + + class DTCQosServer; + + class DTCServers + { + public: + DTCServers(); + ~DTCServers(); + + private: + DTCServers(const DTCServers&); + DTCServers& operator=(const DTCServers&); + + public: + int SetRouteList(std::vector& ip_list); + void SetIDCNo(int IDCNo); + Server* GetServer(); + + int SetAccessKey(const char *token); + int SetTableName(const char *tableName); + /* + Signed=1, // Signed Integer + String=4, // String, case insensitive, null ended + Binary=5, // opaque binary data + */ + int SetKeyType(int type); + void SetAgentTime(int t); + void SetMTimeout(int n); + std::string GetErrorMsg(); + + private: + int IsServerHasExisted(ROUTE_NODE& ip); + int ConstructServers(); + int ConstructServers2(std::vector& IPList); + void DisorderList(int *tmpBuckets, int size); + int ConstructBalanceBuckets(); + int RefreshBalanceBuckets(uint64_t ); + void RemoveServerFromBuckets(uint64_t ); + Server* GetOneServerFromBuckets(); + void SetErrorMsg(int err, std::string from, std::string msg); + + private: + int m_TimeOut; + int m_AgentTime; + int m_KeyType; + char* m_TableName; + std::string m_AccessToken; + std::string m_ErrMsg; + + private: + bool m_SetRoute; + bool m_ConstructedBySetIPs; + int m_Bid; + int m_IDCNo; + int m_BucketsPos; + int m_BalanceBucketSize; + uint64_t m_BidVersion; + uint64_t m_LastGetCaTime; + uint64_t m_RefreshBucketsTime; + uint64_t m_RemoveBucketsTime; + int* m_LoadBalanceBuckets; + DTCQosServer* m_QOSSevers; + std::vector m_IPList; + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int CompressSet(const char *, const char *, int); + //just compress and set. Don't need compressflag + int CompressSetForce(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *ErrorMessage(void) const; + + //无源模式超时时间 add by xuxinxin, 2014/12/09 + int SetExpireTime(const char* key, int time); + int GetExpireTime(const char* key); + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest: public Request { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor){} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + long long MemSize() const; + long long DataSize() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + int UnCompressBinaryValue(const char *name,char **buf,int *lenp); + //Uncompress Binary Value without check compressflag + int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp); + const char * UnCompressErrorMessage() const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + + EC_TASK_TIMEOUT, + + EC_BUSINESS_WITHOUT_EXPIRETIME, //62 + EC_EMPTY_TBDEF, //63 + EC_INVALID_KEY_VALUE, //64 + EC_INVALID_EXPIRETIME, //65 + + EC_GET_EXPIRETIME_END_OF_RESULT, //66 + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; + + enum{ + ER_SET_IPLIST_NULL = 20001, + ER_SET_INSTANCE_PROPERTIES_ERR, + ER_KEY_TYPE, + ER_CC_VERSION_ERR, + ER_ROUTE_INFO_NULL, + ER_BID_VERSION_ERR, + ER_PORT_OUT_RANGE, + ER_STATUS_ERROR_VALUE, + ER_WEIGHT_ERROR_VALUE, + ER_IP_ERROR_VALUE, + }; + +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/dtcapi.lst b/src/search_local/index_storage/api/c_api_cc/dtcapi.lst new file mode 100644 index 0000000..119b77f --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcapi.lst @@ -0,0 +1,12 @@ +{ + global: + *DTC*; + __invoke_dynamic_linker__; + set_network_mode; + set_server_address; + set_server_tablename; + + local: + *; +}; + diff --git a/src/search_local/index_storage/api/c_api_cc/dtcint.h b/src/search_local/index_storage/api/c_api_cc/dtcint.h new file mode 100644 index 0000000..a44f71a --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcint.h @@ -0,0 +1,617 @@ +#ifndef __CHC_CLI_H +#define __CHC_CLI_H + +#include +#include +#include +#include +#include +#include +#include "keylist.h" +#include +#include +#include +#include "udppool.h" +#include "compress.h" +#include "poll_thread.h" +#include "lock.h" +#include + +/* + * Goal: + * single transation (*) + * batch transation (*) + * async transation + */ + + +#define HUNDRED 100 +#define THOUSAND 1000 +#define TENTHOUSAND 10000 +#define HUNDREDTHOUSAND 100000 +#define MILLION 1000000 + + +enum E_REPORT_TYPE +{ + RT_MIN, + RT_SHARDING, + RT_ALL, + RT_MAX +}; + + +//tp99 统计区间，单位us +static const uint32_t kpi_sample_count[] = +{ + 200, 500, 1000, + 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, + 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000, 100000, + 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000, 1000000, + 2000000, 500000, 10000000 +}; + + +class DTCTask; +class CPacket; + +class NCResult; +class NCResultInternal; +class NCPool; +struct NCTransation; +class IDTCTaskExecutor; +class IDTCService; + +class NCBase { +private: + AtomicU32 _count; +public: + NCBase(void) {} + ~NCBase(void) {} + int INC(void) { return ++_count; } + int DEC(void) { return --_count; } + int CNT(void) { return _count.get(); }; +}; + +class NCRequest; +class DataConnector; +class NCServer: public NCBase { +public: // global server state + NCServer(); + NCServer(const NCServer &); + ~NCServer(void); + + // base settings + SocketAddress addr; + char *tablename; + char *appname; + + static DataConnector *dc; + + //for compress + void SetCompressLevel(int level){compressLevel=level;} + int GetCompressLevel(void){return compressLevel;} + + //dtc set _server_address and _server_tablename for plugin + static char * _server_address; + static char * _server_tablename; + + int keytype; + int autoUpdateTable; + int autoReconnect; + static int _network_mode; + NCKeyInfo keyinfo; + + void CloneTabDef(const NCServer& source); + int SetAddress(const char *h, const char *p=NULL); + int SetTableName(const char *); + const char * GetAddress(void) const { return addr.Name(); }//this addres is set by user + const char * GetServerAddress(void) const { return _server_address; }//this address is set by dtc + const char * GetServerTableName(void) const { return _server_tablename;} + int IsDgram(void) const { return addr.socket_type()==SOCK_DGRAM; } + int IsUDP(void) const { return addr.socket_type()==SOCK_DGRAM && addr.socket_family()==AF_INET; } + const char * GetTableName(void) const { return tablename; } + int IntKey(void); + int StringKey(void); + int FieldType(const char*); + int IsCompleted(void) const { return completed; } + int AddKey(const char* name, uint8_t type); + int KeyFieldCnt(void) const { return keyinfo.KeyFields() ? : keytype != DField::None ? 1 : 0; } + int AllowBatchKey(void) const { return keyinfo.KeyFields(); } + int SimpleBatchKey(void) const { return keyinfo.KeyFields()==1 && keytype == keyinfo.KeyType(0); } + + void SetAutoUpdateTab(bool autoUpdate){ autoUpdateTable = autoUpdate?1:0; } + void SetAutoReconnect(int reconnect){ autoReconnect = reconnect; } + + // error state + unsigned completed:1; + unsigned badkey:1; + unsigned badname:1; + unsigned autoping:1; + const char *errstr; + + const char *ErrorMessage(void) const { return errstr; } + + // table definition + DTCTableDefinition *tdef; + DTCTableDefinition *admin_tdef; + + void SaveDefinition(NCResult *); + DTCTableDefinition* GetTabDef(int cmd) const; + + /*date:2014/06/04, author:xuxinxin*/ + std::string accessToken; + int SetAccessKey(const char *token); +private: // serialNr manupulation + uint64_t lastSN; +public: + uint64_t NextSerialNr(void) { ++lastSN; if(lastSN==0) lastSN++; return lastSN; } + uint64_t LastSerialNr(void) { return lastSN; } + +private: // timeout settings + int timeout; + int realtmo; +public: + void SetMTimeout(int n); + int GetTimeout(void) const { return timeout; } +private: + int compressLevel; + +private: // sync execution + IDTCService *iservice; + IDTCTaskExecutor *executor; + int netfd; + time_t lastAct; + NCRequest *pingReq; + +private: + uint64_t agentTime; + +public: + void SetAgentTime(int t){agentTime=t;} + uint64_t GetAgentTime(void){return agentTime;} + +public: + int Connect(void); + int Reconnect(void); + void Close(void); + void SetFD(int fd) { Close(); netfd = fd; UpdateTimeout(); } + // stream, connected + int SendPacketStream(CPacket &); + int DecodeResultStream(NCResult &); + // dgram, connected or connectless + int SendPacketDgram(SocketAddress *peer, CPacket &); + int DecodeResultDgram(SocketAddress *peer, NCResult &); + // connectless + NCUdpPort *GetGlobalPort(void); + void PutGlobalPort(NCUdpPort *); + + void TryPing(void); + int Ping(void); + void AutoPing(void) { if(!IsDgram()) autoping = 1; } + NCResultInternal *ExecuteInternal(NCRequest &rq, const DTCValue *kptr) { return executor->task_execute(rq, kptr); } + int HasInternalExecutor(void) const { return executor != 0; } +private: + int BindTempUnixSocket(void); + void UpdateTimeout(void); + void UpdateTimeoutAnyway(void); + // this method is weak, and don't exist in libdtc.a + __attribute__((__weak__)) void CheckInternalService(void); + +public: // transation manager, impl at dtcpool.cc + int AsyncConnect(int &); + NCPool *ownerPool; + int ownerId; + void SetOwner(NCPool *, int); + + NCResult *DecodeBuffer(const char *, int); + static int CheckPacketSize(const char *, int); + +//add by neolv to QOS +private: + //原有的不变增加三个参数用于做QOS + uint64_t m_iErrCount; + //用于统计请求总数， 请求总数只对一个周期有效 + uint64_t m_iTotalCount; + //用于统计请求总耗时， 请求总数只对一个周期有效 + uint64_t m_iTotalElaps; + //被摘次数 + int m_RemoveCount; +public: + void IncErrCount() + { + ++m_iErrCount; + } + uint64_t GetErrCount() + { + return m_iErrCount; + } + void ClearErrCount() + { + m_iErrCount = 0; + } + + void IncRemoveCount() + { + ++m_RemoveCount; + } + int GetRemoveCount() + { + return m_RemoveCount; + } + void ClearRemoveCount() + { + m_RemoveCount = 0; + } + + void IncTotalReq() + { + ++m_iTotalCount; + } + uint64_t GetTotalReq() + { + return m_iTotalCount; + } + void ClearTotalReq() + { + m_iTotalCount = 0; + } + + void AddTotalElaps(uint64_t iElaps) + { + m_iTotalElaps += iElaps; + } + uint64_t GetTotalElaps() + { + return m_iTotalElaps; + } + void ClearTotalElaps() + { + m_iTotalElaps = 0; + } + +}; + + +class NCRequest { +public: + NCServer *server; + uint8_t cmd; + uint8_t haskey; + uint8_t flags; + int err; + DTCValue key; + NCKeyValueList kvl; + FieldValueByName ui; + FieldValueByName ci; + FieldSetByName fs; + + DTCTableDefinition *tdef; + char *tablename; + int keytype; + + unsigned int limitStart; + unsigned int limitCount; + int adminCode; + + uint64_t hotBackupID; + uint64_t MasterHBTimestamp; + uint64_t SlaveHBTimestamp; + +public: + NCRequest(NCServer *, int op); + ~NCRequest(void); + int AttachServer(NCServer *); + + void EnableNoCache(void) { flags |= DRequest::Flag::no_cache; } + void EnableNoNextServer(void) { flags |= DRequest::Flag::no_next_server; } + void EnableNoResult(void) { flags |= DRequest::Flag::NoResult; } + int AddCondition(const char *n, uint8_t op, uint8_t t, const DTCValue &v); + int AddOperation(const char *n, uint8_t op, uint8_t t, const DTCValue &v); + int CompressSet(const char *n,const char * v,int len); + int CompressSetForce(const char *n,const char * v,int len); + int AddValue(const char *n, uint8_t t, const DTCValue &v); + int Need(const char *n, int); + void Limit(unsigned int st, unsigned int cnt) { + if(cnt==0) st = 0; + limitStart = st; + limitCount = cnt; + } + + int SetKey(int64_t); + int SetKey(const char *, int); + int UnsetKey(void); + int UnsetKeyValue(void); + int FieldType(const char* name){ return server?server->FieldType(name):DField::None; } + int AddKeyValue(const char* name, const DTCValue &v, uint8_t type); + int SetCacheID(int dummy) { return err = -EINVAL; } + void SetAdminCode(int code){ adminCode = code; } + void SetHotBackupID(uint64_t v){ hotBackupID=v; } + void SetMasterHBTimestamp(uint64_t t){ MasterHBTimestamp=t; } + void SetSlaveHBTimestamp(uint64_t t){ SlaveHBTimestamp=t; } + + // never return NULL + NCResult *Execute(const DTCValue *key=NULL); + NCResult *ExecuteStream(const DTCValue *key=NULL); + NCResult *ExecuteDgram(SocketAddress *peer, const DTCValue *key = NULL); + NCResult *ExecuteNetwork(const DTCValue *key=NULL); + NCResult *ExecuteInternal(const DTCValue *key=NULL); + NCResult *Execute(int64_t); + NCResult *Execute(const char *, int); + NCResult *PreCheck(const DTCValue *key); // return error result, NULL==success + int SetCompressFieldName(void);//Need compress flag for read,or set compressFlag for write + int Encode(const DTCValue *key, CPacket *); + // return 1 if tdef changed... + int SetTabDef(void); + + int EncodeBuffer(char *&ptr, int&len, int64_t &magic, const DTCValue *key=NULL); + int EncodeBuffer(char *&ptr, int&len, int64_t &magic, int64_t); + int EncodeBuffer(char *&ptr, int&len, int64_t &magic, const char *, int); + const char* ErrorMessage(void) const + { + return _errmsg; + } + + int SetExpireTime(const char* key, int t); + int GetExpireTime(const char* key); + +private: + int CheckKey(const DTCValue *kptr); + int setCompressFlag(const char * name) + { + if (tdef==NULL) + return -EC_NOT_INITIALIZED; + if (tdef->field_id(name) >= 64) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:field id must less than 64"); + return -EC_COMPRESS_ERROR; + } + compressFlag|=(1<field_id(name)); + return 0; + } + uint64_t compressFlag;//field flag + DTCCompress *gzip; + int initCompress(void); + char _errmsg[1024]; +}; + +class NCResultLocal { + public: + const uint8_t *vidmap; + long long apiTag; + int maxvid; + DTCCompress *gzip; + + public: + NCResultLocal(DTCTableDefinition* tdef) : + vidmap(NULL), + apiTag(0), + maxvid(0), + gzip (NULL), + _tdef(tdef), + compressid(-1) + { + } + + virtual ~NCResultLocal(void) { + FREE_CLEAR(vidmap); + DELETE (gzip); + } + + virtual int FieldIdVirtual(int id) const { + return id > 0 && id <= maxvid ? vidmap[id-1] : 0; + } + + virtual void SetApiTag(long long t) { apiTag = t; } + virtual long long GetApiTag(void) const { return apiTag; } + + void SetVirtualMap(FieldSetByName &fs) + { + if(fs.max_virtual_id()){ + fs.Resolve(_tdef, 0); + vidmap = fs.virtual_map(); + maxvid = fs.max_virtual_id(); + } + } + + virtual int initCompress() + { + if (NULL == _tdef) + { + return -EC_CHECKSUM_MISMATCH; + } + + int iret = 0; + compressid = _tdef->compress_field_id(); + if (compressid<0) return 0; + if (gzip==NULL) + NEW(DTCCompress,gzip); + if (gzip==NULL) + return -ENOMEM; + iret = gzip->set_buffer_len(_tdef->max_field_size()); + if (iret) return iret; + return 0; + } + + virtual const int CompressId (void)const {return compressid;} + + private: + DTCTableDefinition* _tdef; + uint64_t compressid; + +}; + +class NCResult: public NCResultLocal, public DTCTask +{ + public: + NCResult(DTCTableDefinition *tdef) : NCResultLocal(tdef), DTCTask(tdef, TaskRoleClient, 0) { + if(tdef) tdef->INC(); + mark_allow_remote_table(); + } + + NCResult(int err, const char *from, const char *msg) : NCResultLocal(NULL), DTCTask(NULL, TaskRoleClient, 1) { + resultInfo.set_error_dup(err, from, msg); + } + virtual ~NCResult() { + DTCTableDefinition *tdef = table_definition(); + DEC_DELETE(tdef); + } +}; + +class NCResultInternal: public NCResultLocal, public TaskRequest +{ + public: + NCResultInternal(DTCTableDefinition* tdef=NULL) : NCResultLocal (tdef) + { + } + + virtual ~NCResultInternal() + { + } + + static inline int VerifyClass(void) + { + NCResultInternal *ir = 0; + NCResult *er = reinterpret_cast(ir); + + NCResultLocal *il = (NCResultLocal *) ir; + NCResultLocal *el = (NCResultLocal *) er; + + DTCTask *it = (DTCTask *) ir; + DTCTask *et = (DTCTask *) er; + + long dl = reinterpret_cast(il) - reinterpret_cast(el); + long dt = reinterpret_cast(it) - reinterpret_cast(et); + return dl==0 && dt==0; + } +}; + +/*date:2014/06/09, author:xuxinxin 模调上报 */ +class DataConnector +{ + struct businessStatistics + { + uint64_t TotalTime; // 10s内请求总耗时 + uint32_t TotalRequests; // 10s内请求总次数 + + public: + businessStatistics(){ TotalTime = 0; TotalRequests = 0; } + }; + + struct bidCurve{ + uint32_t bid; + uint32_t curve; + bool operator < (const bidCurve &that) const{ + int sum1 = bid * 10 + curve; + int sum2 = that.bid * 10 + that.curve; + return sum1 < sum2; + } + }; + + struct top_percentile_statistics + { + uint32_t uiBid; + uint32_t uiAgentIP; + uint16_t uiAgentPort; + uint32_t uiTotalRequests; //10s内请求总次数 + uint64_t uiTotalTime; //10s内请求总耗时 + uint32_t uiFailCount; //10s内错误次数 + uint64_t uiMaxTime; //10s内的最大执行时间 + uint64_t uiMinTime; //10s内的最小执行时间 + uint32_t statArr[sizeof(kpi_sample_count) / sizeof(kpi_sample_count[0])]; //统计值 + + public: + top_percentile_statistics() + { + uiBid = 0; + uiAgentIP = 0; + uiAgentPort = 0; + uiTotalRequests = 0; + uiTotalTime = 0; + uiFailCount = 0; + uiMaxTime = 0; + uiMinTime = 0; + memset(statArr, 0, sizeof(statArr)); + } + top_percentile_statistics(const top_percentile_statistics &that) + { + this->uiBid = that.uiBid; + this->uiAgentIP = that.uiAgentIP; + this->uiAgentPort = that.uiAgentPort; + this->uiTotalRequests = that.uiTotalRequests; + this->uiTotalTime = that.uiTotalTime; + this->uiFailCount = that.uiFailCount; + this->uiMaxTime = that.uiMaxTime; + this->uiMinTime = that.uiMinTime; + memcpy(this->statArr, that.statArr, sizeof(this->statArr)); + } + top_percentile_statistics &operator =(const top_percentile_statistics &that) + { + this->uiBid = that.uiBid; + this->uiAgentIP = that.uiAgentIP; + this->uiAgentPort = that.uiAgentPort; + this->uiTotalRequests = that.uiTotalRequests; + this->uiTotalTime = that.uiTotalTime; + this->uiFailCount = that.uiFailCount; + this->uiMaxTime = that.uiMaxTime; + this->uiMinTime = that.uiMinTime; + memcpy(this->statArr, that.statArr, sizeof(this->statArr)); + + return *this; + } + }; + + private: + std::map mapBi; + Mutex _lock; // 读写 TotalTime、TotalRequests时，加锁，防止脏数据 + std::map mapTPStat; + Mutex m_tp_lock; //读写 tp99 数据时，加锁，防止脏数据 + static DataConnector *pDataConnector; + DataConnector(); + ~DataConnector(); + pthread_t threadid; + + public: + static DataConnector* getInstance() + { + if( pDataConnector == NULL) + pDataConnector = new DataConnector(); + return pDataConnector; + }; + + public: + int SendData(); + int SetReportInfo(const std::string str, const uint32_t curve, const uint64_t t); + void GetReportInfo(std::map &mapData); + int SetBussinessId(std::string str); + + int SendTopPercentileData(); + int SetTopPercentileData(const std::string strAccessKey, const std::string strAgentAddr, const uint64_t elapse, const int status); + void GetTopPercentileData(std::map &mapStat); + int SetTopPercentileConfig(std::string strAccessKey, const std::string strAgentAddr); + +private: + int ParseAddr(const std::string strAddr, uint64_t *uipIP = NULL, uint64_t *uipPort = NULL); +}; + + +class CTopPercentileSection +{ +public: + static int16_t GetTPSection(uint64_t elapse); + +private: + //执行时间小于1000us的 + static int16_t GetLTThousand(uint64_t elapse); + //执行时间大于等于1000us 小于 10000us的 + static int16_t GetGTEThousandLTTenThousand(uint64_t elapse); + //执行时间大于等于10000us 小于 100000us的 + static int16_t GetGTETenThousandLTHundredThousand(uint64_t elapse); + //执行时间大于等于100000us 小于 1000000us的 + static int16_t GetGTEHundredThousandLTMillion(uint64_t elapse); + //执行时间大于等于1000000us的 + static int16_t GetGTEMillion(uint64_t elapse); +}; + + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/dtcpool.cc b/src/search_local/index_storage/api/c_api_cc/dtcpool.cc new file mode 100644 index 0000000..5008155 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcpool.cc @@ -0,0 +1,1009 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include +#include +#include +#include +#include "myepoll.h" + +#include "dtcpool.h" + +// must be bitwise +enum { + RS_IDLE = 0, + RS_WAIT = 1, + RS_SEND = 2, + RS_RECV = 4, + RS_DONE = 8, +}; + +enum { + SS_CONNECTED = 0, + SS_LINGER = 1, + SS_CLOSED = 1, + SS_CONNECTING = 2, +}; + +#define MAXREQID 2147483646L + +NCPool::NCPool(int ms, int mr) + : + CPollerUnit(1024+16) +{ + int i; + + if(ms > (1<<20)) + ms = 1<<20; + if(mr > (1<<20)) + mr = 1<<20; + + maxServers = ms; + maxRequests = mr; + maxRequestId = MAXREQID / maxRequests; + + serverList = new NCServerInfo[maxServers]; + // needn't zero init + numServers = 0; + + transList = new NCTransation[maxRequests]; + for(i=0; iownerPool != NULL) + return -EINVAL; + + if(numServers >= maxServers) + return -E2BIG; + + if(server->IsDgram()) + { + if(mConn == 0) + mConn = 1; + else if(mConn != 1) + return -EINVAL; + } else { + if(mConn == 0) + mConn = mReq; + else if(mConn > mReq) + return -EINVAL; + } + + server->SetOwner(this, numServers); + serverList[numServers].Init(server, mReq, mConn); + numServers++; + return 0; +} + +/* switch basic server to async mode */ +void NCServer::SetOwner(NCPool *owner, int id) +{ + Close(); + ownerId = id;; + ownerPool = owner; + INC(); +} + +/* async connect, high level */ +int NCServer::AsyncConnect(int &netfd) +{ + int err = -EC_NOT_INITIALIZED; + netfd = -1; + + if(addr.SocketFamily() != 0) { + netfd = addr.CreateSocket(); + if(netfd < 0) { + err = -errno; + } else if(addr.SocketFamily()==AF_UNIX && IsDgram() && BindTempUnixSocket() < 0) { + err = -errno; + close(netfd); + netfd = -1; + } else { + fcntl(netfd, F_SETFL, O_RDWR|O_NONBLOCK); + if(addr.ConnectSocket(netfd)==0) + return 0; + err = -errno; + if(err!=-EINPROGRESS) + { + close(netfd); + netfd = -1; + } + } + } + return err; +} + +NCConnection::NCConnection(NCPool *owner, NCServerInfo *si) + : + CPollerObject(owner), + timer(this) +{ + serverInfo = si; + sreq = NULL; + state = SS_CLOSED; + result = NULL; +} + +NCConnection::~NCConnection(void) +{ + if(state==SS_CONNECTING) + /* decrease connecting count */ + serverInfo->ConnectingDone(); + /* remove partial result */ + DELETE(result); + /* remove from all connection queue */ + ListDel(); + /* abort all associated requests */ + AbortRequests(-EC_REQUEST_ABORTED); + serverInfo->MoreClosedConnectionAndReady(); +} + +/* connection is idle again */ +void NCConnection::SwitchToIdle(void) +{ + if(state==SS_CONNECTING) + /* decrease connecting count */ + serverInfo->ConnectingDone(); + ListMove(&serverInfo->idleList); + serverInfo->ConnectionIdleAndReady(); + state = SS_CONNECTED; + DisableOutput(); + EnableInput(); + timer.DisableTimer(); + /* ApplyEvnets by caller */ +} + +/* prepare connecting state, wait for EPOLLOUT */ +void NCConnection::SwitchToConnecting(void) +{ + state = SS_CONNECTING; + timer.AttachTimer(serverInfo->timerList); + ListMove(&serverInfo->busyList); + DisableInput(); + EnableOutput(); + // no apply events, following AttachPoller will do it */ +} + +/* try async connecting */ +int NCConnection::Connect(void) +{ + int err = serverInfo->info->AsyncConnect(netfd); + if(err == 0) { + SwitchToIdle(); + /* treat epollsize overflow as fd overflow */ + if(AttachPoller() < 0) + return -EMFILE; + /* AttachPoller will ApplyEvents automatically */ + } else if(err == -EINPROGRESS) { + SwitchToConnecting(); + /* treat epollsize overflow as fd overflow */ + if(AttachPoller() < 0) + return -EMFILE; + /* AttachPoller will ApplyEvents automatically */ + } + + return err; +} + +/* Link connection & transation */ +void NCConnection::ProcessRequest(NCTransation *r) +{ + /* linkage between connection & transation */ + sreq = r; + sreq->AttachConnection(this); + + /* adjust server connection statistics */ + ListMoveTail(&serverInfo->busyList); + serverInfo->RequestScheduled(); + + /* initial timing and flushing */ + timer.AttachTimer(serverInfo->timerList); + SendRequest(); +} + +/* abort all requests associated transation */ +void NCConnection::AbortRequests(int err) +{ + if(sreq) + { + sreq->Abort(err); + sreq = NULL; + } + while(reqList.ListEmpty()==0) + { + NCTransation *req = reqList.NextOwner(); + req->Abort(err); + } +} + +/* abort sending transation, linger connection for RECV request */ +void NCConnection::AbortSendSide(int err) +{ + if(sreq) + { + sreq->Abort(err); + sreq = NULL; + } + if(!IsAsync() || reqList.ListEmpty()) + Close(err); + else + { + // async connection, lingering + ListMove(&serverInfo->busyList); + state = SS_LINGER; + timer.AttachTimer(serverInfo->timerList); + } +} + +/* close a connection */ +void NCConnection::Close(int err) +{ + AbortRequests(err); + if(IsDgram()==0) + delete this; + /* UDP has not connection close */ +} + +/* a valid result received */ +void NCConnection::DoneResult(void) +{ + if(result) + { + /* searching SN */ + CListObject *reqPtr = reqList.ListNext(); + while(reqPtr != &reqList) + { + NCTransation *req = reqPtr->ListOwner(); + if(req->MatchSN(result)) + { + /* SN matched, transation is dont */ + req->Done(result); + result = NULL; + break; + } + reqPtr = reqPtr->ListNext(); + } + DELETE(result); + } + if(IsAsync()==0) { + /* + * SYNC server, switch to idle state, + * ASYNC server always idle, no switch needed + */ + SwitchToIdle(); + ApplyEvents(); + } else if(state == SS_LINGER) { + /* close LINGER connection if all result done */ + if(reqList.ListEmpty()) + delete this; + } + /* + // disabled because async server should trigger by req->Done + else + serverInfo->MarkAsReady(); + */ +} + +/* hangup by recv zero bytes */ +int NCConnection::CheckHangup(void) +{ + if(IsAsync()) + return 0; + char buf[1]; + int n = recv(netfd, buf, sizeof(buf), MSG_DONTWAIT|MSG_PEEK); + return n >= 0; +} + +/* + * sending result, may called by other component + * apply events is required + */ +void NCConnection::SendRequest(void) +{ + int ret = sreq->Send(netfd); + timer.DisableTimer(); + switch (ret) + { + case SendResultMoreData: + /* more data to send, enable EPOLLOUT and timer */ + EnableOutput(); + ApplyEvents(); + timer.AttachTimer(serverInfo->timerList); + break; + + case SendResultDone: + /* send OK, disable output and enable receiving */ + DisableOutput(); + EnableInput(); + sreq->SendOK(&reqList); + sreq = NULL; + serverInfo->RequestSent(); + /* fire up receiving timer */ + timer.AttachTimer(serverInfo->timerList); + if(IsAsync()) + { + ListMoveTail(&serverInfo->idleList); + serverInfo->ConnectionIdleAndReady(); + } +#if 0 + else { + /* fire up receiving logic */ + RecvResult(); + } +#endif + ApplyEvents(); + break; + + default: + AbortSendSide(-ECONNRESET); + break; + } +} + +int NCConnection::RecvResult(void) +{ + int ret; + if(result==NULL) + { + result = new NCResult(serverInfo->info->tdef); + receiver.attach(netfd); + receiver.erase(); + } + if(IsDgram()) + { + if(serverInfo->owner->buf==NULL) + serverInfo->owner->buf = new char[65536]; + ret = recv(netfd, serverInfo->owner->buf, 65536, 0); + if(ret <= 0) + ret = DecodeFatalError; + else + ret = result->Decode(serverInfo->owner->buf, ret); + } else + ret = result->Decode(receiver); + timer.DisableTimer(); + switch (ret) + { + default: + case DecodeFatalError: + Close(-ECONNRESET); + return -1; // connection bad + break; + + case DecodeDataError: + DoneResult(); + // more result maybe available + ret = 1; + break; + + case DecodeIdle: + case DecodeWaitData: + // partial or no result available yet + ret = 0; + break; + + case DecodeDone: + serverInfo->info->SaveDefinition(result); + DoneResult(); + // more result maybe available + ret = 1; + break; + } + if(sreq || !reqList.ListEmpty()) + timer.AttachTimer(serverInfo->timerList); + return ret; +} + +void NCConnection::InputNotify(void) +{ + switch(state) { + case SS_CONNECTED: + while(1) { + if(RecvResult() <= 0) + break; + } + break; + case SS_LINGER: + while(reqList.ListEmpty()==0) { + if(RecvResult() <= 0) + break; + } + default: + break; + } +} + +void NCConnection::OutputNotify(void) +{ + switch(state) + { + case SS_CONNECTING: + SwitchToIdle(); + break; + case SS_CONNECTED: + SendRequest(); + break; + default: + DisableOutput(); + break; + } +} + +void NCConnection::HangupNotify(void) +{ + if(state==SS_CONNECTING) { + serverInfo->ConnectingFailed(); + } + AbortRequests(-ECONNRESET); + delete this; +} + +void NCConnection::TimerNotify(void) +{ + if(sreq || !reqList.ListEmpty()) + { + Close(-ETIMEDOUT); + } +} + +NCServerInfo::NCServerInfo(void) +{ + info = NULL; +} + +NCServerInfo::~NCServerInfo(void) +{ + while(!idleList.ListEmpty()) + { + NCConnection *conn = idleList.NextOwner(); + delete conn; + } + + while(!busyList.ListEmpty()) + { + NCConnection *conn = busyList.NextOwner(); + delete conn; + } + + while(!reqList.ListEmpty()) + { + NCTransation *trans = reqList.NextOwner(); + trans->Abort(-EC_REQUEST_ABORTED); + } + + if(info && info->DEC()==0) + delete info; +} + +void NCServerInfo::Init(NCServer *server, int maxReq, int maxConn) +{ + info = server; + reqList.ResetList(); + idleList.ResetList(); + busyList.ResetList(); + + reqWait = 0; + reqSend = 0; + reqRecv = 0; + reqRemain = maxReq; + + connRemain = maxConn; + connTotal = maxConn; + connConnecting = 0; + connError = 0; + owner = server->ownerPool; + int to = server->GetTimeout(); + if(to <= 0) to = 3600 * 1000; // 1 hour + timerList = owner->GetTimerListByMSeconds( to ); + + mode = server->IsDgram() ? 2 : maxReq==maxConn ? 0 : 1; +} + +int NCServerInfo::Connect(void) +{ + NCConnection *conn = new NCConnection(owner, this); + connRemain--; + int err = conn->Connect(); + if(err==0) { + /* pass */; + } else if(err==-EINPROGRESS) { + connConnecting++; + err = 0; /* NOT a ERROR */ + } else { + connRemain++; + delete conn; + connError++; + } + return err; +} + +void NCServerInfo::AbortWaitQueue(int err) +{ + while(!reqList.ListEmpty()) + { + NCTransation *trans = reqList.NextOwner(); + trans->Abort(err); + } +} + +void NCServerInfo::TimerNotify() +{ + while(reqRemain>0 && !reqList.ListEmpty()) + { + if(!idleList.ListEmpty()) + { + // has something to do + NCConnection *conn = idleList.NextOwner(); + + conn->ProcessRequest(GetRequestFromQueue()); + } else { + // NO connection available + if(connRemain == 0) + break; + // need more connection to process + if(connConnecting >= reqWait) + break; + + int err; + // connect error, abort processing + if((err=Connect()) != 0) + { + if(connRemain>=connTotal) + AbortWaitQueue(err); + break; + } + } + } + /* no more work, clear bogus ready timer */ + CTimerObject::DisableTimer(); + /* reset connect error count */ + connError = 0; +} + +void NCServerInfo::RequestDoneAndReady(int state) { + switch(state) { + case RS_WAIT: + reqWait--; // Abort a queued not really ready + break; + case RS_SEND: + reqSend--; + reqRemain++; + MarkAsReady(); + break; + case RS_RECV: + reqRecv--; + reqRemain++; + MarkAsReady(); + break; + } +} + +NCTransation::NCTransation(void) +{ +} + +NCTransation::~NCTransation(void) +{ + DELETE(packet); + DELETE(result); +} + +/* clear transation state */ +void NCTransation::Clear(void) +{ + // detach from list, mostly freeList + CListObject::ResetList(); + + // increase reqId + genId++; + + // reset local state + state = RS_IDLE; + reqTag = 0;; + server = NULL; + packet = NULL; + result = NULL; +} + +/* get transation result */ +NCResult * NCTransation::GetResult(void) { + NCResult *ret = result; + ret->SetApiTag(reqTag); + DELETE(packet); + Clear(); + return ret; +} + +/* attach new request to transation slot */ +int NCTransation::AttachRequest(NCServerInfo *info, long long tag, NCRequest *req, DTCValue *key) { + state = RS_WAIT; + reqTag = tag; + server = info; + packet = new CPacket; + int ret = req->Encode(key, packet); + if(ret < 0) { + /* encode error */ + result = new NCResult(ret, "API::encoding", "client encode packet error"); + } else { + SN = server->info->LastSerialNr(); + } + return ret; +} + +extern inline void NCTransation::AttachConnection(NCConnection *c) { + state = RS_SEND; + conn = c; +} + +extern inline void NCTransation::SendOK(CListObject *queue) { + state = RS_RECV; + ListAddTail(queue); +} + +extern inline void NCTransation::RecvOK(CListObject *queue) { + state = RS_DONE; + ListAddTail(queue); +} + +/* abort current transation, zero means succ */ +void NCTransation::Abort(int err) +{ + if(server == NULL) // empty slot or done + return; + + //Don't reverse abort connection, just clear linkage + //NCConnection *c = conn; + //DELETE(c); + conn = NULL; + + NCPool *owner = server->owner; + ListDel(); + + server->RequestDoneAndReady(state); + server = NULL; + + if(err) + { + result = new NCResult(err, "API::aborted", "client abort request"); + } + + owner->TransationFinished(this); +} + +NCResult *NCPool::GetTransationResult(NCTransation *req) +{ + NCResult *ret = req->GetResult(); + doneRequests--; + numRequests--; + req->ListAdd(&freeList); + return ret; +} + +/* get transation slot for new request */ +NCTransation * NCPool::GetTransationSlot(void) +{ + NCTransation *trans = freeList.NextOwner(); + trans->Clear(); + trans->RoundGenId(maxRequestId); + numRequests ++; + return trans; +} + +void NCPool::TransationFinished(NCTransation *trans) +{ + trans->RecvOK(&doneList); + doneRequests++; +} + +int NCPool::AddRequest(NCRequest *req, long long tag, DTCValue *key) +{ + int ret; + if(numRequests >= maxRequests) + { + return -E2BIG; + } + + if(req->server == NULL) + { + return -EC_NOT_INITIALIZED; + } + + NCServer *server = req->server; + if(server->ownerPool == NULL) + { + ret = AddServer(server, 1); + if(ret < 0) + return ret; + } + + if(server->ownerPool != this) + { + return -EINVAL; + } + + if(key==NULL && req->haskey) key = &req->key; + + NCTransation *trans = GetTransationSlot(); + NCServerInfo *info = &serverList[server->ownerId]; + + if(trans->AttachRequest(info, tag, req, key) < 0) + { + // attach error, result already prepared */ + TransationFinished(trans); + } else { + info->QueueRequest(trans); + } + + return (trans-transList) + trans->GenId() * maxRequests + 1; +} + +void NCPool::ExecuteOneLoop(int timeout) +{ + WaitPollerEvents(timeout); + uint64_t now = GET_TIMESTAMP(); + ProcessPollerEvents(); + CheckExpired(now); + CheckReady(); +} + +int NCPool::Execute(int timeout) +{ + if(timeout > 3600000) timeout = 3600000; + InitializePollerUnit(); + ExecuteOneLoop(0); + if(doneRequests > 0) + return doneRequests; + int64_t till = GET_TIMESTAMP() + timeout * TIMESTAMP_PRECISION / 1000; + while(doneRequests <= 0) + { + int64_t exp = till - GET_TIMESTAMP(); + if(exp < 0) + break; + int msec; +#if TIMESTAMP_PRECISION > 1000 + msec = exp / (TIMESTAMP_PRECISION/1000); +#else + msec = exp * 1000/TIMESTAMP_PRECISION; +#endif + ExecuteOneLoop(ExpireMicroSeconds(msec, 1)); + } + return doneRequests; +} + +int NCPool::ExecuteAll(int timeout) +{ + if(timeout > 3600000) timeout = 3600000; + InitializePollerUnit(); + ExecuteOneLoop(0); + if(numRequests <= doneRequests) + return doneRequests; + int64_t till = GET_TIMESTAMP() + timeout * TIMESTAMP_PRECISION / 1000; + while(numRequests > doneRequests) + { + int64_t exp = till - GET_TIMESTAMP(); + if(exp < 0) + break; + int msec; +#if TIMESTAMP_PRECISION > 1000 + msec = exp / (TIMESTAMP_PRECISION/1000); +#else + msec = exp * 1000/TIMESTAMP_PRECISION; +#endif + ExecuteOneLoop(ExpireMicroSeconds(msec, 1)); + } + return doneRequests; +} + +int NCPool::InitializePollerUnit(void) +{ + if(initFlag==-1) + { + initFlag = CPollerUnit::InitializePollerUnit() >= 0; + } + return initFlag; +} + +int NCPool::GetEpollFD(int mp) +{ + if(initFlag == -1 && mp > CPollerUnit::GetMaxPollers()) + CPollerUnit::SetMaxPollers(mp); + InitializePollerUnit(); + return CPollerUnit::GetFD(); +} + +NCTransation * NCPool::Id2Req(int reqId) const +{ + if(reqId <= 0 || reqId > MAXREQID) + return NULL; + reqId--; + NCTransation *req = &transList[reqId % maxRequests]; + if(reqId / maxRequests != req->GenId()) + return NULL; + return req; +} + + +int NCPool::AbortRequest(NCTransation *req) +{ + if(req->conn) + { + /* send or recv state */ + if(req->server->mode==0) + { // SYNC, always close connecction + req->conn->Close(-EC_REQUEST_ABORTED); + } else if(req->server->mode==1) + { // ASYNC, linger connection for other result, abort send side + if(req->State()==RS_SEND) + req->conn->AbortSendSide(-EC_REQUEST_ABORTED); + else + req->Abort(-EC_REQUEST_ABORTED); + } else { // UDP, abort request only + req->Abort(-EC_REQUEST_ABORTED); + } + return 1; + } + else if(req->server) + { + /* waiting state, abort request only */ + req->Abort(-EC_REQUEST_ABORTED); + return 1; + } + return 0; +} + +int NCPool::CancelRequest(int reqId) +{ + NCTransation *req = Id2Req(reqId); + if(req==NULL) + return -EINVAL; + + AbortRequest(req); + NCResult *res = GetTransationResult(req); + DELETE(res); + return 1; +} + +int NCPool::CancelAllRequest(int type) +{ + int n = 0; + if((type & ~0xf) != 0) + return -EINVAL; + for(int i=0; iState())) + { + AbortRequest(req); + NCResult *res = GetTransationResult(req); + DELETE(res); + n++; + } + } + return n; +} + +int NCPool::AbortRequest(int reqId) +{ + NCTransation *req = Id2Req(reqId); + if(req==NULL) + return -EINVAL; + + return AbortRequest(req); +} + +int NCPool::AbortAllRequest(int type) +{ + int n = 0; + if((type & ~0xf) != 0) + return -EINVAL; + type &= ~RS_DONE; + for(int i=0; iState())) + { + AbortRequest(req); + n++; + } + } + return n; +} + +NCResult *NCPool::GetResult(int reqId) +{ + NCTransation *req; + if(reqId==0) + { + if(doneList.ListEmpty()) + return (NCResult *)-EAGAIN; + req = doneList.NextOwner(); + } else { + req = Id2Req(reqId); + if(req==NULL) return (NCResult *)-EINVAL; + + if(req->State() != RS_DONE) + return (NCResult *)req->State(); + } + return GetTransationResult(req); +} + +int NCServerInfo::CountRequestState(int type) const +{ + int n = 0; + if((type & RS_WAIT)) + n += reqWait; + if((type & RS_SEND)) + n += reqSend; + if((type & RS_RECV)) + n += reqRecv; + return n; +} + +int NCPool::CountRequestState(int type) const +{ + int n = 0; +#if 0 + for(int i=0; iState(); +} + diff --git a/src/search_local/index_storage/api/c_api_cc/dtcpool.h b/src/search_local/index_storage/api/c_api_cc/dtcpool.h new file mode 100644 index 0000000..3a0bd92 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcpool.h @@ -0,0 +1,275 @@ +#ifndef __CHC_CLI_POOL_H +#define __CHC_CLI_POOL_H + +#include + +#include "list.h" +#include "poller.h" +#include "timerlist.h" +#include "dtcint.h" +#include "cache_error.h" + +class NCServerInfo; +class NCPool; +class NCConnection; + +// transation is a internal async request +class NCTransation : + public CListObject +{ +public: + // constructor/destructor equiv + NCTransation(void); + ~NCTransation(void); + /* clear transation state */ + void Clear(void); + /* abort current transation, zero means succ */ + void Abort(int); + /* transation succ with result */ + void Done(NCResult *res) { result = res; Abort(0); } + /* get transation result */ + NCResult * GetResult(void); + /* adjust generation id */ + void RoundGenId(int m) { if(genId >= m) genId = 0; } + + /* state & info management */ + int State(void) const { return state; } + int GenId(void) const { return genId; } + int MatchSN(NCResult *res) const { return SN == res->versionInfo.SerialNr(); } + /* send packet management */ + int Send(int fd) { return packet->Send(fd); } + int AttachRequest(NCServerInfo *s, long long tag, NCRequest *req, DTCValue *key); + void AttachConnection(NCConnection *c); + void SendOK(CListObject*); + void RecvOK(CListObject*); + +public: // constant member declare as static + // owner info, associated server + NCServerInfo *server; + // attached connection, SEND, RECV + NCConnection *conn; + +private:// transient members is private + // current transation state, WAIT, SEND, RECV, DONE + int state; + // internal transation generation id + int genId; + // associated request tag + long long reqTag; + // associated request SN, state SEND, RECV + uint64_t SN; + + // sending packet + CPacket *packet; + + // execute result + NCResult *result; +}; + +class NCConnection : + public CListObject, + public CPollerObject +{ +public: + NCConnection(NCPool *, NCServerInfo *); + ~NCConnection(void); + + int IsDgram(void) const; + int IsAsync(void) const; + /* starting state machine, by NCServerInfo::Connect */ + int Connect(void); + /* attach transation to connection */ + void ProcessRequest(NCTransation *); + /* flush send channel */ + void SendRequest(void); + /* flush recv channel */ + int RecvResult(void); + /* check connection hangup, recv zero bytes */ + int CheckHangup(void); + + /* abort all associated request */ + void AbortRequests(int err); + /* close connection */ + void Close(int err); + /* abort current sending request, linger recv channel */ + void AbortSendSide(int err); + /* a valid result received */ + void DoneResult(void); + /* connection is idle, usable for transation */ + void SwitchToIdle(void); + /* connection is async connecting */ + void SwitchToConnecting(void); + + virtual void InputNotify(void); + virtual void OutputNotify(void); + virtual void HangupNotify(void); + virtual void TimerNotify(void); + +private: + /* associated server */ + NCServerInfo *serverInfo; + /* queued transation in RECV state */ + CListObject reqList; + /* sending transation */ + NCTransation *sreq; + /* decoding/decoded result */ + NCResult *result; + /* connection state */ + int state; + + int NETFD(void) const { return netfd; } + +private: + CTimerMember timer; + CSimpleReceiver receiver; +}; + +typedef CListObject NCConnectionList; + +class NCPool : + public CPollerUnit, + public CTimerUnit +{ +public: + NCPool(int maxServers, int maxRequests); + ~NCPool(); + + int InitializePollerUnit(void); + + int GetEpollFD(int maxpoller); + int AddServer(NCServer *srv, int maxReq=1, int maxConn=0); + int AddRequest(NCRequest *req, long long tag, DTCValue *key=0); + + void ExecuteOneLoop(int timeout); + int Execute(int timeout); + int ExecuteAll(int timeout); + + NCTransation *Id2Req(int) const; + int CancelRequest(int); + int CancelAllRequest(int); + int AbortRequest(NCTransation *); + int AbortRequest(int); + int AbortAllRequest(int); + NCResult *GetResult(void); + NCResult *GetResult(int); + + int CountRequestState(int) const; + int RequestState(int) const; +public: + NCTransation * GetTransationSlot(void); + void TransationFinished(NCTransation *); + NCResult * GetTransationResult(NCTransation *); + int GetTransationState(NCTransation *); + + int ServerCount(void) const { return numServers; } + int RequestCount(void) const { return numRequests; } + int DoneRequestCount(void) const { return doneRequests; } +private: + int initFlag; + int maxServers; + int maxRequests; + int numServers; + int numRequests; + int maxRequestId; + int doneRequests; + CListObject freeList; + CListObject doneList; + NCServerInfo *serverList; + NCTransation *transList; +public: + char *buf; +}; + +class NCServerInfo : + private CTimerObject +{ +public: + NCServerInfo(void); + ~NCServerInfo(void); + void Init(NCServer *, int, int); + + /* prepare wake TimerNotify */ + void MarkAsReady() { AttachReadyTimer(owner); } + virtual void TimerNotify(void); + /* four reason server has more work to do */ + /* more transation attached */ + void MoreRequestAndReady(void) { reqWait++; MarkAsReady(); } + /* more close connection, should reconnecting */ + void MoreClosedConnectionAndReady(void) { connRemain++; MarkAsReady(); } + /* more idle connection available */ + void ConnectionIdleAndReady(void) { MarkAsReady(); } + /* more request can assign to idle pool */ + void RequestDoneAndReady(int oldstate); + + /* one request scheduled to SEND state */ + void RequestScheduled(void) { reqRemain--; reqSend++; } + void RequestSent(void) { reqSend--; reqRecv++; } + + /* queue transation to this server */ + void QueueRequest(NCTransation *req) + { + req->ListAddTail(&reqList); + MoreRequestAndReady(); + } + /* one connecting aborted */ + void ConnectingFailed(void) { connError++; } + void ConnectingDone(void) { connConnecting--; } + + /* abort all waiting transations */ + void AbortWaitQueue(int err); + + int CountRequestState(int type) const; + + /* get a waiting transation */ + NCTransation * GetRequestFromQueue(void) + { + NCTransation *trans = reqList.NextOwner(); + trans->ListDel(); + reqWait--; + return trans; + } + +private: + int Connect(void); + CListObject reqList; + +public: // constant member declare as public + /* associated ServerPool */ + NCPool *owner; + /* basic server info */ + NCServer *info; + CTimerList *timerList; + int mode; // 0--TCP 1--ASYNC 2--UDP + /* total connection */ + int connTotal; + +private:// transient member is private + + /* transation in state WAIT */ + int reqWait; + /* transation in state SEND */ + int reqSend; + /* transation in state RECV */ + int reqRecv; + /* remain requests can assign to connection pool */ + int reqRemain; + + /* remain connections can connect */ + int connRemain; + /* number connections in connecting state */ + int connConnecting; + /* number of connect error this round */ + int connError; + /* busy connection count */ + inline int connWorking(void) const { return connTotal - connConnecting - connRemain; } +public: //except idleList for code conventional + /* idle connection list */ + NCConnectionList idleList; + /* busy connection list SEND,RECV,LINGER */ + NCConnectionList busyList; +}; + +inline int NCConnection::IsDgram(void) const { return serverInfo->mode==2; } +inline int NCConnection::IsAsync(void) const { return serverInfo->mode; } + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/dtcqossvr.cc b/src/search_local/index_storage/api/c_api_cc/dtcqossvr.cc new file mode 100644 index 0000000..85a53bc --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcqossvr.cc @@ -0,0 +1,834 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "protocol.h" + +#include "dtcapi.h" +#include "dtcint.h" +#include "dtcqossvr.h" + +using namespace DTC; + +/* linux-2.6.38.8/include/linux/compiler.h */ +//# define likely(x) __builtin_expect(!!(x), 1) +//# define unlikely(x) __builtin_expect(!!(x), 0) + +#define random(x) (rand()%x) + +const uint32_t ACCESS_KEY_LEN = 40; + +DTCQosServer::DTCQosServer():m_Status(0), m_Weight(0), m_RemoveTimeStamp(0), + m_LastRemoveTime(0), m_MaxTimeStamp(0), m_Server(NULL) +{ +} + +DTCQosServer::~DTCQosServer() +{ + if (this->m_Server != NULL){ + delete this->m_Server; + this->m_Server = NULL; + } +} + +DTCServers::DTCServers() + :m_TimeOut(50), m_AgentTime(0), m_KeyType(0), m_TableName(NULL), + m_SetRoute(false), m_ConstructedBySetIPs(false), m_Bid(0), + m_IDCNo(0), m_BucketsPos(0), m_BalanceBucketSize(0), m_BidVersion(0), + m_LastGetCaTime(0), m_RefreshBucketsTime(0), m_RemoveBucketsTime(0), + m_LoadBalanceBuckets(NULL), m_QOSSevers(NULL) +{ +} + +DTCServers::~DTCServers() +{ + if(this->m_TableName) + { + free(this->m_TableName); + this->m_TableName = NULL; + } + + if(this->m_LoadBalanceBuckets) + { + free(this->m_LoadBalanceBuckets); + this->m_LoadBalanceBuckets = NULL; + } + + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } +} + +void PrintIPNode(ROUTE_NODE RouteNode) +{ + printf("\n"); + printf("\t bid = %d \n", RouteNode.bid); + printf("\t ip = %s \n", RouteNode.ip); + printf("\t port = %d \n", RouteNode.port); + printf("\t weight = %d \n", RouteNode.weight); + printf("\t status = %d \n", RouteNode.status); + printf("\n"); +} + +void DTCServers::SetErrorMsg(int err, std::string from, std::string msg) +{ + char buffer[1000]; + memset(buffer, 0, 1000); + sprintf(buffer, "ERROR CODE %d, FROM %s ERRMSG %s", err, from.c_str(), msg.c_str()); + std::string ErrMsg(buffer); + this->m_ErrMsg.clear(); + this->m_ErrMsg = ErrMsg; +} + +std::string DTCServers::GetErrorMsg() +{ + return this->m_ErrMsg; +} + +int InvalidIpaddr(char *str) +{ + if(str == NULL || *str == '\0') + return 1; + + union + { + struct sockaddr addr; + struct sockaddr_in6 addr6; + struct sockaddr_in addr4; + } a; + memset(&a, 0, sizeof(a)); + if(1 == inet_pton(AF_INET, str, &a.addr4.sin_addr)) + return 0; + else if(1 == inet_pton(AF_INET6, str, &a.addr6.sin6_addr)) + return 0; + return 1; +} + +int DTCServers::SetRouteList(std::vector& IPList) +{ + if(IPList.empty()) + { + return -ER_SET_IPLIST_NULL; + } + + std::vector().swap(this->m_IPList); + std::vector::iterator it = IPList.begin(); + for(; it != IPList.end(); ++it) + { + ROUTE_NODE ip; + ip.bid = it->bid; + if(it->port > 0 && it->port <= 65535){ + ip.port = it->port; + }else{ + SetErrorMsg(ER_PORT_OUT_RANGE, "DTCServers::SetRouteList", "port is out of range!"); + return -ER_PORT_OUT_RANGE; + } + if(it->status == 0 || it->status == 1){ + ip.status = it->status; + }else{ + SetErrorMsg(ER_STATUS_ERROR_VALUE, "DTCServers::SetRouteList", "status is error value!"); + return -ER_STATUS_ERROR_VALUE; + } + if(it->weight > 0){ + ip.weight = it->weight; + }else{ + SetErrorMsg(ER_WEIGHT_ERROR_VALUE, "DTCServers::SetRouteList", "weight is error value!"); + return -ER_WEIGHT_ERROR_VALUE; + } + if(InvalidIpaddr(it->ip) == 0){ + memcpy(ip.ip, it->ip, IP_LEN); + }else{ + SetErrorMsg(ER_IP_ERROR_VALUE, "DTCServers::SetAccessKey", "ip is error value!"); + return -ER_IP_ERROR_VALUE; + } +#ifdef DEBUG_INFO + PrintIPNode(ip); +#endif + this->m_IPList.push_back(ip); + } + this->m_SetRoute = true; + + return 0; +} + +void DTCServers::SetIDCNo(int IDCNo) +{ + this->m_IDCNo = IDCNo; +} + +int DTCServers::SetAccessKey(const char *token) +{ + std::string str; + if(token == NULL) + return -EC_BAD_ACCESS_KEY; + else + str = token; + if(str.length() != ACCESS_KEY_LEN) + { + log_error("Invalid accessKey!"); + this->m_AccessToken = ""; + SetErrorMsg(EC_BAD_ACCESS_KEY, "DTCServers::SetAccessKey", "Invalid accessKey!"); + return -EC_BAD_ACCESS_KEY; + } + else + this->m_AccessToken = str; + + std::string stemp = str.substr(0, 8); + sscanf(stemp.c_str(), "%d", &(this->m_Bid)); + return 0; +} + +int DTCServers::SetTableName(const char *tableName) +{ + if(tableName==NULL) return -DTC::EC_BAD_TABLE_NAME; + + if(this->m_TableName) + return mystrcmp(tableName, this->m_TableName, 256)==0 ? 0 : -DTC::EC_BAD_TABLE_NAME; + + this->m_TableName = STRDUP(tableName); + + return 0; +} + +int DTCServers::SetKeyType(int type) +{ + switch(type) + { + case DField::Signed: + case DField::Unsigned: + case DField::Float: + case DField::String: + case DField::Binary: + this->m_KeyType = type; + break; + default: + return -DTC::EC_BAD_KEY_TYPE; + break; + } + return 0; +} + +/* +int DTCServers::AddKey(const char* name, uint8_t type) +{ +} +*/ + +void DTCServers::SetAgentTime(int t) +{ + this->m_AgentTime = t; +} + +void DTCServers::SetMTimeout(int n) +{ + this->m_TimeOut = n<=0 ? 5000 : n; +} + +int DTCServers::ConstructServers() +{ + if(this->m_IPList.empty()) + { + log_error("ip list is empty!"); + SetErrorMsg(ER_SET_IPLIST_NULL, "DTCServers::ConstructServers", "ip list is empty!"); + return -ER_SET_IPLIST_NULL; + } + + if(this->m_AccessToken.empty() || (this->m_TableName == NULL) || (this->m_KeyType == 0)) + { + log_error("m_AccessToken m_TableName or m_KeyType is unset"); + SetErrorMsg(ER_SET_INSTANCE_PROPERTIES_ERR, "DTCServers::ConstructServers", "m_AccessToken m_TableName or m_KeyType is unset!"); + return -ER_SET_INSTANCE_PROPERTIES_ERR; + } + + int i = 0; + int ret = 0; + char tmpPort[7]; + memset(tmpPort, 0, sizeof(tmpPort)); + + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + int IPCount = this->m_IPList.size(); + this->m_QOSSevers = new DTCQosServer[IPCount]; + for( ; i < IPCount; ++i) + { + this->m_QOSSevers[i].CreateDTCServer(); + Server *server = this->m_QOSSevers[i].GetDTCServer(); + sprintf(tmpPort, "%d", this->m_IPList[i].port); + ret = server->SetAddress(this->m_IPList[i].ip, tmpPort); + if(ret < 0) + { + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + + return -ret; + } + + this->m_QOSSevers[i].setWeight(this->m_IPList[i].weight); + this->m_QOSSevers[i].SetStatus(this->m_IPList[i].status); + ret = server->SetTableName(this->m_TableName); + if(ret < 0) + { + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + return -ret; + } + + server->SetAccessKey(this->m_AccessToken.c_str()); + + switch(this->m_KeyType) + { + case DField::Signed: + case DField::Unsigned: + server->IntKey(); + break; + case DField::String: + server->StringKey(); + break; + case DField::Binary: + server->StringKey(); + break; + case DField::Float: + default: + { + log_error("key type is wrong!"); + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + SetErrorMsg(ER_KEY_TYPE, "DTCServers::ConstructServers", "key type is wrong!"); + return -ER_KEY_TYPE; + } + break; + } + + if(this->m_TimeOut > 0) + { + server->SetMTimeout(this->m_TimeOut); + } + } + return 0; +} + +int DTCServers::IsServerHasExisted(ROUTE_NODE& ip) +{ + int i; + for (i = 0; i < this->m_IPList.size(); i++) + { + if((ip.port == this->m_IPList[i].port) && (strncmp(ip.ip, this->m_IPList[i].ip, IP_LENGHT) == 0)) + return i; + } + return -1; +} + +int DTCServers::ConstructServers2(std::vector& IPList) +{ + if(IPList.empty()) + { + log_error("ip list is empty!"); + SetErrorMsg(ER_SET_IPLIST_NULL, "DTCServers::ConstructServers2", "ip list is empty!"); + return -ER_SET_IPLIST_NULL; + } + + if(this->m_AccessToken.empty() || (this->m_TableName == NULL) || (this->m_KeyType == 0)) + { + log_error("m_AccessToken m_TableName or m_KeyType is unset"); + SetErrorMsg(ER_SET_INSTANCE_PROPERTIES_ERR, "DTCServers::ConstructServers2", "m_AccessToken m_TableName or m_KeyType is unset!"); + return -ER_SET_INSTANCE_PROPERTIES_ERR; + } + + int i = 0; + int ret = 0; + char tmpPort[7]; + memset(tmpPort, 0, sizeof(tmpPort)); + + int IPCount = IPList.size(); + DTCQosServer* tmpQosServer = new DTCQosServer[IPCount]; + for ( ; i < IPCount; i++) + { + int idx = IsServerHasExisted(IPList[i]); + if (idx >= 0){ + tmpQosServer[i].setWeight(IPList[i].weight); + tmpQosServer[i].SetStatus(IPList[i].status); + tmpQosServer[i].SetRemoveTimeStamp(this->m_QOSSevers[idx].GetRemoveTimeStamp()); + tmpQosServer[i].SetLastRemoveTime(this->m_QOSSevers[idx].GetLastRemoveTime()); + tmpQosServer[i].SetMaxTimeStamp(this->m_QOSSevers[idx].GetMaxTimeStamp()); + tmpQosServer[i].SetDTCServer(this->m_QOSSevers[idx].GetDTCServer()); + this->m_QOSSevers[idx].ResetDTCServer(); + } else { + tmpQosServer[i].CreateDTCServer(); + Server *server = tmpQosServer[i].GetDTCServer(); + sprintf(tmpPort, "%d", IPList[i].port); + ret = server->SetAddress(IPList[i].ip, tmpPort); + if(ret < 0) + { + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + delete[] tmpQosServer; + return -ret; + } + + tmpQosServer[i].setWeight(IPList[i].weight); + tmpQosServer[i].SetStatus(IPList[i].status); + ret = server->SetTableName(this->m_TableName); + if(ret < 0) + { + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + delete[] tmpQosServer; + return -ret; + } + server->SetAccessKey(this->m_AccessToken.c_str()); + + switch(this->m_KeyType) + { + case DField::Signed: + case DField::Unsigned: + server->IntKey(); + break; + case DField::String: + server->StringKey(); + break; + case DField::Binary: + server->StringKey(); + break; + case DField::Float: + default: + { + log_error("key type is wrong!"); + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + delete[] tmpQosServer; + SetErrorMsg(ER_KEY_TYPE, "DTCServers::ConstructServers2", "key type is wrong!"); + return -ER_KEY_TYPE; + } + break; + } + + if(this->m_TimeOut > 0) + { + server->SetMTimeout(this->m_TimeOut); + } + } + } + + std::vector().swap(this->m_IPList); + if(this->m_QOSSevers) + { + delete[] this->m_QOSSevers; + this->m_QOSSevers = NULL; + } + this->m_QOSSevers = tmpQosServer; + tmpQosServer = NULL; + + return 0; +} + +void init_random() +{ + uint64_t ticks; + struct timeval tv; + int fd; + gettimeofday(&tv, NULL); + ticks = tv.tv_sec + tv.tv_usec; + fd = open("/dev/urandom", O_RDONLY); + if (fd > 0) + { + uint64_t r; + int i; + for (i = 0; i <100 ; i++) + { + read(fd, &r, sizeof(r)); + ticks += r; + } + close(fd); + } + srand(ticks); +} + +unsigned int new_rand() +{ + int fd; + unsigned int n = 0; + fd = open("/dev/urandom", O_RDONLY); + if(fd > 0) + { + read(fd, &n, sizeof(n)); + } + close (fd); + return n; +} + +void DTCServers::DisorderList(int *tmpBuckets, int size) +{ + int randCount = 0;// 索引 + unsigned int position = 0;// 位置 + int k = 0; +// srand((int)time(0)); + do{ + int r = size - randCount; +// position = random(r); + init_random(); + unsigned int tmp = new_rand(); + position = tmp%r; + this->m_LoadBalanceBuckets[k++] = tmpBuckets[position]; + randCount++; + // 将最后一位数值赋值给已经被使用的position + tmpBuckets[position] = tmpBuckets[r - 1]; + }while(randCount < size); + return ; +} + +int DTCServers::ConstructBalanceBuckets() +{ + if(!this->m_QOSSevers || this->m_IPList.empty()) + { + log_error("QOSSevers is null or ip count <0 "); + SetErrorMsg(ER_ROUTE_INFO_NULL, "DTCServers::ConstructBalanceBuckets", "QOSSevers is null or ip count <0!"); + return -ER_ROUTE_INFO_NULL; + } + + int i = 0; + int totalCount = 0; + int IPCount = this->m_IPList.size(); + for( ; i < IPCount; ++i) + { + totalCount += this->m_QOSSevers[i].m_Weight; + } + FREE_IF(this->m_LoadBalanceBuckets); + this->m_LoadBalanceBuckets = (int *)malloc(sizeof(int) * totalCount); + this->m_BalanceBucketSize = totalCount; + int* tmpBuckets = (int *)malloc(sizeof(int) * totalCount); + int j = 0; + int pos = 0; + for( ; j < IPCount; ++j) + { + int k = 0; + for( ; k < this->m_QOSSevers[j].m_Weight; ++k) + { + tmpBuckets[pos] = j; + pos ++; + } + } + this->m_BucketsPos = 0; + DisorderList(tmpBuckets, totalCount); +// FREE_IF(tmpBuckets); + FREE_CLEAR(tmpBuckets); + return 0; +} + +void DTCServers::RemoveServerFromBuckets(uint64_t now) +{ + int i = 0; + int IPCount = this->m_IPList.size(); + for( ; i < IPCount; ++i) + { + if(0 == this->m_QOSSevers[i].GetStatus()) + { + continue; + } + uint64_t errorCount = this->m_QOSSevers[i].GetDTCServer()->GetErrCount(); + if(errorCount >= DEFAULT_REMOVE_ERROR_COUNT) + { + this->m_QOSSevers[i].GetDTCServer()->IncRemoveCount(); + uint64_t removeTime = this->m_QOSSevers[i].GetDTCServer()->GetRemoveCount() * DEFAULT_ROUTE_INTERVAL_TIME; + if(removeTime >= DEFAULT_MAX_REMOVE_THRESHOLD_TIME) + { + removeTime = DEFAULT_MAX_REMOVE_THRESHOLD_TIME; + } + this->m_QOSSevers[i].SetRemoveTimeStamp(removeTime); + this->m_QOSSevers[i].SetLastRemoveTime(now); + this->m_QOSSevers[i].SetStatus(0); + log_debug("bid=[%d], remove time=[%lu], now=[%lu], address=[%s], pos=[%d]!", \ + this->m_Bid, removeTime, now, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), i); +#ifdef DEBUG_INFO + printf("remove bid=[%d],now=[%lu],remvoe timeStamp=[%lu],remove count=[%d],address=[%s]!\n", \ + this->m_Bid, now, removeTime, this->m_QOSSevers[i].GetDTCServer()->GetRemoveCount(), \ + this->m_QOSSevers[i].GetDTCServer()->GetAddress()); +#endif + } + } +} + +int DTCServers::RefreshBalanceBuckets(uint64_t now) +{ + int i = 0; + int IPCount = this->m_IPList.size(); + for( ; i < IPCount; ++i) + { + if(this->m_QOSSevers[i].GetStatus() == 0) + { + if(this->m_QOSSevers[i].GetLastRemoveTime() != 0) + { + if(now - this->m_QOSSevers[i].GetLastRemoveTime() > this->m_QOSSevers[i].GetRemoveTimeStamp()) + { + this->m_QOSSevers[i].SetStatus(1); + this->m_QOSSevers[i].SetRemoveTimeStamp(0); + this->m_QOSSevers[i].SetLastRemoveTime(0); + this->m_QOSSevers[i].GetDTCServer()->ClearErrCount(); + log_debug("bid=[%d], address=[%s], pos=[%d]!", \ + this->m_Bid, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), i); +#ifdef DEBUG_INFO + printf("refresh bid=[%d],address=[%s],now=[%lu]!\n", \ + this->m_Bid, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), now); +#endif + } + } + } + } + + return 0; +} + +DTC::Server* DTCServers::GetOneServerFromBuckets() +{ + int serverPos = 0; + int lastBucketsPos = this->m_BucketsPos; + do{ + ++this->m_BucketsPos; + if(this->m_BucketsPos >= this->m_BalanceBucketSize) + this->m_BucketsPos = 0; + + if(unlikely(!this->m_LoadBalanceBuckets)) + return NULL; + serverPos = this->m_LoadBalanceBuckets[this->m_BucketsPos]; + if(this->m_QOSSevers[serverPos].GetStatus()) + { +#ifdef DEBUG_INFO + printf("get server address=[%s]\n", this->m_QOSSevers[serverPos].GetDTCServer()->GetAddress()); +#endif + return this->m_QOSSevers[serverPos].GetDTCServer(); + }else{ + //整租服务皆不可用，返回默认位置的server + if(lastBucketsPos == this->m_BucketsPos) + { + int tmpPos = this->m_LoadBalanceBuckets[DEFAULT_SERVER_POS]; + return this->m_QOSSevers[tmpPos].GetDTCServer(); + } + } + }while(1); +} + +DTC::Server* DTCServers::GetServer() +{ + int ret = 0; + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t now = tv.tv_sec; + if(this->m_SetRoute && (!this->m_IPList.empty()) && (!this->m_ConstructedBySetIPs)) + { +#ifdef DEBUG_INFO + printf("construct server by set route\n"); +#endif + ret = ConstructServers(); + if(ret < 0) + { + log_error("construct servers by set route failed! error code=[%d]", ret); + SetErrorMsg(ret, "DTCServers::GetServer", "construct servers by set route failed!"); + return NULL; + } + ret = ConstructBalanceBuckets(); + if(ret < 0) + { + log_error("construct balance bucket failed! error code=[%d]", ret); + SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed"); + return NULL; + } + this->m_ConstructedBySetIPs = true; +#ifdef DEBUG_INFO + printf("get one server from set route\n"); +#endif + goto GetOneServer; + } + + if(this->m_SetRoute && (!this->m_IPList.empty()) && this->m_ConstructedBySetIPs) + { +#ifdef DEBUG_INFO + printf("get one server from set route\n"); +#endif + goto GetOneServer; + } + + if(0 == this->m_BidVersion) + { + uint64_t BidVersion = 0; + int ret = get_version(&BidVersion); + if(BidVersion <= 0) + { + log_info("get version from CC error!"); +// SetErrorMsg(ER_BID_VERSION_ERR, "DTCServers::GetServer", "get version from CC error!"); + SetErrorMsg(ret, "DTCServers::GetServer", "get version from CC error!"); + return NULL; + } + IP_ROUTE IPRoute; + memset(&IPRoute, 0, sizeof(IP_ROUTE)); + ret = get_ip_route(this->m_Bid, &IPRoute); + if(ret < 0 || IPRoute.ip_num <= 0 || IPRoute.ip_list == NULL) + { + log_error("get ip list by bid failed! ip list is null!"); + SetErrorMsg(0, "DTCServers::GetServer", "get ip list by bid failed! ip list is null!"); + free_ip_route(&IPRoute); + return NULL; + } + + int i = 0; + std::vector().swap(this->m_IPList); + for(; i < IPRoute.ip_num; ++i) + { + ROUTE_NODE RouteNode; + RouteNode.bid = IPRoute.ip_list[i].bid; + memcpy(RouteNode.ip, IPRoute.ip_list[i].ip, IP_LEN); + RouteNode.port = IPRoute.ip_list[i].port; + RouteNode.status = IPRoute.ip_list[i].status; + RouteNode.weight = IPRoute.ip_list[i].weight; +#ifdef DEBUG_INFO + PrintIPNode(RouteNode); +#endif + this->m_IPList.push_back(RouteNode); + } + free_ip_route(&IPRoute); + +#ifdef DEBUG_INFO + printf("construct server by cc\n"); +#endif + ret = ConstructServers(); + if(ret < 0) + { + log_error("construct servers failed!"); + SetErrorMsg(ret, "DTCServers::GetServer", "construct servers failed!!"); + return NULL; + } + //TODO cc version管理 + + ret = ConstructBalanceBuckets(); + if(ret < 0) + { + log_error("construct balance bucket failed! error code=[%d]", ret); + SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed!"); + return NULL; + } +#ifdef DEBUG_INFO + printf("get one server from cc\n"); +#endif + this->m_LastGetCaTime = now; + this->m_BidVersion = BidVersion; + goto GetOneServer; + } + + if(now - this->m_LastGetCaTime >= DEFAULT_ROUTE_EXPIRE_TIME) + { + uint64_t BidVersion = 0; + get_version(&BidVersion); + this->m_LastGetCaTime = now; + if(this->m_BidVersion >= BidVersion) + { + log_info("the version is lastest!"); +#ifdef DEBUG_INFO + printf("get one server from cc\n"); +#endif + goto GetOneServer; + } + IP_ROUTE IPRoute; + memset(&IPRoute, 0, sizeof(IP_ROUTE)); + ret = get_ip_route(this->m_Bid, &IPRoute); + if(ret < 0 || IPRoute.ip_num <= 0 || IPRoute.ip_list == NULL) + { + log_error("get ip list by bid failed! ip list is null!"); + SetErrorMsg(0, "DTCServers::GetServer", "get ip list by bid failed! ip list is null!"); + free_ip_route(&IPRoute); + return NULL; + } + + int i = 0; + std::vector IPList; + + for(; i < IPRoute.ip_num; ++i) + { + ROUTE_NODE RouteNode; + RouteNode.bid = IPRoute.ip_list[i].bid; + memcpy(RouteNode.ip, IPRoute.ip_list[i].ip, IP_LEN); + RouteNode.port = IPRoute.ip_list[i].port; + RouteNode.status = IPRoute.ip_list[i].status; + RouteNode.weight = IPRoute.ip_list[i].weight; +#ifdef DEBUG_INFO + PrintIPNode(RouteNode); +#endif + IPList.push_back(RouteNode); + } + free_ip_route(&IPRoute); + +#ifdef DEBUG_INFO + printf("construct server by cc\n"); +#endif + ret = ConstructServers2(IPList); + if(ret < 0) + { + std::vector().swap(this->m_IPList); + this->m_IPList = IPList; + log_error("construct servers failed!"); + SetErrorMsg(ret, "DTCServers::GetServer", "construct servers failed!"); + return NULL; + } + this->m_IPList = IPList; + ret = ConstructBalanceBuckets(); + if(ret < 0) + { + log_error("construct balance bucket failed! error code=[%d]", ret); + SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed!"); + return NULL; + } + this->m_BidVersion = BidVersion; +#ifdef DEBUG_INFO + printf("get one server from cc\n"); +#endif + goto GetOneServer; + }else{ +#ifdef DEBUG_INFO + printf("get one server from cc\n"); +#endif + goto GetOneServer; + } + log_error("Exception process!"); + SetErrorMsg(0, "DTCServers::GetServer", "Exception process!"); + return NULL; + +GetOneServer: +/* if(now - this->m_RefreshBucketsTime >= DEFAULT_REFRESH_BUCKETS_TIME) + { + RefreshBalanceBuckets(now); + this->m_RefreshBucketsTime = now; + } + if(now - this->m_RemoveBucketsTime >= DEFAULT_REMOVE_BUCKETS_TIME) + { + RemoveServerFromBuckets(now); + this->m_RemoveBucketsTime = now; + } +*/ + RefreshBalanceBuckets(now); + RemoveServerFromBuckets(now); + return GetOneServerFromBuckets(); +} diff --git a/src/search_local/index_storage/api/c_api_cc/dtcqossvr.h b/src/search_local/index_storage/api/c_api_cc/dtcqossvr.h new file mode 100644 index 0000000..2346b59 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcqossvr.h @@ -0,0 +1,108 @@ +#ifndef __CHC_CLI_SERVERS_H +#define __CHC_CLI_SERVERS_H + +#include "dtcint.h" + +extern "C" +{ +#include "app_client_set.h" +} + +#define DEFAULT_ROUTE_EXPIRE_TIME 120 +#define DEFAULT_MAX_REMOVE_THRESHOLD_TIME 1800 +#define DEFAULT_REMOVE_ERROR_COUNT 3 +#define DEFAULT_ROUTE_INTERVAL_TIME 5 +#define DEFAULT_SERVER_POS 0 + +namespace DTC { + + class DTCQosServer + { + public: + DTCQosServer(); + ~DTCQosServer(void); + private: + DTCQosServer(const DTCQosServer& qosServer); + DTCQosServer& operator=(const DTCQosServer& qosServer); + + public: + int GetStatus() + { + return this->m_Status; + } + void SetStatus(int iFlag) + { + this->m_Status = iFlag; + } + + int GetWeight() + { + return this->m_Weight; + } + void setWeight(int iFlag) + { + this->m_Weight = iFlag; + } + + uint64_t GetRemoveTimeStamp() + { + return this->m_RemoveTimeStamp; + } + void SetRemoveTimeStamp(uint64_t iFlag) + { + this->m_RemoveTimeStamp = iFlag; + } + + uint64_t GetLastRemoveTime() + { + return this->m_LastRemoveTime; + } + void SetLastRemoveTime(uint64_t iFlag) + { + this->m_LastRemoveTime = iFlag; + } + + uint64_t GetMaxTimeStamp() + { + return this->m_MaxTimeStamp; + } + void SetMaxTimeStamp(uint64_t iFlag) + { + this->m_MaxTimeStamp = iFlag; + } + + Server* GetDTCServer() + { + return this->m_Server; + } + + void SetDTCServer(Server* ser) + { + this->m_Server = ser; + } + + void ResetDTCServer() + { + this->m_Server = NULL; + } + + void CreateDTCServer() + { + this->m_Server = new Server(); + } + + public: + friend class DTCServers; + + private: + int m_Status; + int m_Weight; + uint64_t m_RemoveTimeStamp;//需要被摘除的时间 + uint64_t m_LastRemoveTime; //上次被移除的时间点 + uint64_t m_MaxTimeStamp;//最大指数阀值 + Server *m_Server; + }; + +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/dtcreq.cc b/src/search_local/index_storage/api/c_api_cc/dtcreq.cc new file mode 100644 index 0000000..4ef56a1 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcreq.cc @@ -0,0 +1,1120 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dtcapi.h" +#include "dtcint.h" + +// NCRequest's methods always assign and return +// cache the logical error code +#define return_err(x) return err = (x) +#define return_err_res(x,y,z) return new NCResult(x,y,z) +#define MAX_EXPIRETIME 30*24*3600 +#define CLIENT_CURVE 5 +#define AGENT_CURVE 8 + + +NCRequest::NCRequest(NCServer *s, int op) +{ + compressFlag = 0; + if( op != DRequest::Monitor && (s && s->IsCompleted()==0) )//MonitorRequest need not table check + s = NULL; + server = s; + if(server) server->INC(); + tdef = NULL; + tablename = NULL; + keytype = 0; + + switch(op) { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + if(server){ + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + } + break; + //跨IDC分布支持 + case DRequest::SvrAdmin: + case DRequest::Invalidate: + if(server){ + tdef = server->admin_tdef; + tablename =(char *) "@HOT_BACKUP"; + keytype = DField::Unsigned; + } + break; + case DRequest::Monitor: + break; + default: + op = DRequest::ResultCode; + } + cmd = op; + err = 0; + haskey = 0; + flags = 0; + key.u64 = 0; + key.bin.ptr = NULL; + key.bin.len = 0; + + limitStart = 0; + limitCount = 0; + adminCode = 0; + hotBackupID = 0; + MasterHBTimestamp = 0; + SlaveHBTimestamp = 0; + if(server) + kvl.keyinfo = &server->keyinfo; + gzip = NULL; +} + +NCRequest::~NCRequest(void){ + UnsetKeyValue(); + UnsetKey(); + if(server) DEC_DELETE(server); + if(gzip) DELETE(gzip); +} + +// AttachServer() error is transient, don't cache it +int NCRequest::AttachServer(NCServer *s) +{ + // NO-OP + if(s==server) + return 0; + + switch(cmd) { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + break; + default: + // don't allow change admin target server + return -EPERM; + } + + /* new server must be initialized */ + if(s==NULL || s->IsCompleted()==0) + return -EC_NOT_INITIALIZED; + + /* precheck same table */ + if(server) { + if(keytype != s->keytype) + return -EC_BAD_KEY_TYPE; + if(!server->keyinfo.Equal(s->keyinfo)) + return -EC_BAD_KEY_TYPE; + if(strcmp(tablename, s->tablename) != 0) + return -EC_BAD_TABLE_NAME; + if(tdef==NULL) { + // no current tabledef + } else if(tdef == server->tdef) { + // same tabledef + } else if(tdef->IsSameTable(server->tdef)) { + // hash equal + } else { + // force re-resolve fieldnames + ui.Unresolve(); + ci.Unresolve(); + fs.Unresolve(); + } + + // release old server + DEC_DELETE(server); + } + + // switch to new server + server = s; + server->INC(); + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + kvl.keyinfo = &server->keyinfo; + return 0; +} + +int NCRequest::Need(const char *n, int vid) +{ + if(server==NULL) + return_err(-EC_NOT_INITIALIZED); + if(cmd!=DRequest::Get && cmd!=DRequest::SvrAdmin) + return_err(-EC_BAD_OPERATOR); + int ret = fs.AddField(n, vid); + if(ret) err = ret; + return ret; +} + +int NCRequest::AddCondition(const char *n, uint8_t op, uint8_t t, const DTCValue &v) +{ + if(server==NULL) + return_err(-EC_NOT_INITIALIZED); + if(cmd==DRequest::Insert || cmd==DRequest::Replace) + return_err(-EC_BAD_OPERATOR); + switch(t) { + case DField::Signed: + case DField::Unsigned: + if(op >= DField::TotalComparison) + return_err(-EC_BAD_OPERATOR); + break; + case DField::String: + case DField::Binary: + if(op != DField::EQ && op != DField::NE) + return_err(-EC_BAD_OPERATOR); + break; + default: + return_err(-EC_BAD_FIELD_TYPE); + } + int ret = ci.AddValue(n, op, t, v); + if(ret) err = ret; + return ret; +} + +int NCRequest::AddOperation(const char *n, uint8_t op, uint8_t t, const DTCValue &v) +{ + if(server==NULL) return_err(-EC_NOT_INITIALIZED); + switch(cmd) + { + case DRequest::Insert: + case DRequest::Replace: + case DRequest::SvrAdmin: + if(op != DField::Set) + return_err(-EC_BAD_OPERATOR); + break; + case DRequest::Update: + break; + default: + return_err(-EC_BAD_OPERATOR); + break; + } + + switch(t) { + case DField::Signed: + case DField::Unsigned: + if(op >= DField::TotalOperation) + return_err(-EC_BAD_OPERATOR); + break; + + case DField::String: + case DField::Binary: + if(op == DField::Add||op == DField::OR) return_err(-EC_BAD_OPERATOR); + break; + + case DField::Float: + if(op == DField::SetBits || op == DField::OR) + return_err(-EC_BAD_OPERATOR); + break; + default: + return_err(-EC_BAD_FIELD_TYPE); + } + int ret = ui.AddValue(n, op, t, v); + if(ret) err = ret; + return ret; +} + +int NCRequest::initCompress() +{ + int iret = 0; + if (server==NULL) + return -EC_NOT_INITIALIZED; + if (tdef==NULL) + { + iret = server->Ping();//ping and get tabledef + tdef = server->tdef; + } + if (iret) + return iret; + if (gzip==NULL) + NEW(DTCCompress,gzip); + if (gzip==NULL) + return -ENOMEM; + if (tdef==NULL) + return -EC_CHECKSUM_MISMATCH; + if (server->GetCompressLevel()) + { + gzip->SetCompressLevel(server->GetCompressLevel()); + } + return gzip->SetBufferLen(tdef->MaxFieldSize()); +} + +int NCRequest::CompressSet(const char *n,const char * v,int len) +{ + int iret = 0; + if (len < 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldlen is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (n == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldname is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (v == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldvalue is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (gzip==NULL) + { + iret = initCompress(); + if (iret) return iret; + } + if (tdef->CompressFieldId() <= 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:DTC must add a field for compress(FieldType=2,FieldSize=8,DefaultValue=compressflag)"); + return -EC_COMPRESS_ERROR; + } + + if (tdef->FieldType(tdef->FieldId(n)) != DField::Binary) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress just support binary field"); + return - EC_BAD_VALUE_TYPE; + } + + iret = gzip->Compress(v,len); + if (iret) + { + if (iret==-111111) + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress buff is null,sth sucks happend"); + else + snprintf(_errmsg, sizeof(_errmsg), "compress error:zlib return code is %d.",iret); + return -EC_COMPRESS_ERROR ; + } + iret = setCompressFlag(n); + if (iret) return iret; + + return AddOperation(n,DField::Set,DField::String,DTCValue::Make(gzip->GetBuf(),gzip->GetLen())); +} + +int NCRequest::CompressSetForce(const char *n,const char * v,int len) +{ + int iret = 0; + if (len < 0) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldlen is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (n == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldname is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (v == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:fieldvalue is invalid"); + return -EC_COMPRESS_ERROR; + } + + if (gzip==NULL) + { + iret = initCompress(); + if (iret) return iret; + } + + if (tdef->FieldType(tdef->FieldId(n)) != DField::Binary) + { + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress just support binary field"); + return - EC_BAD_VALUE_TYPE; + } + + iret = gzip->Compress(v,len); + if (iret) + { + if (iret==-111111) + snprintf(_errmsg, sizeof(_errmsg), "compress error:compress buff is null,sth sucks happend"); + else + snprintf(_errmsg, sizeof(_errmsg), "compress error:zlib return code is %d.",iret); + return -EC_COMPRESS_ERROR ; + } + if (iret) return iret; + + return AddOperation(n,DField::Set,DField::String,DTCValue::Make(gzip->GetBuf(),gzip->GetLen())); +} + +int NCRequest::AddValue(const char *n, uint8_t t, const DTCValue &v) +{ + if(server==NULL) return_err(-EC_NOT_INITIALIZED); + if(cmd!=DRequest::Insert && cmd!=DRequest::Replace) return_err(-EC_BAD_COMMAND); + int ret = ui.AddValue(n, DField::Set, t, v); + if(ret) err = ret; + return ret; +} + +int NCRequest::UnsetKey(void) { + if(haskey) { + if(server && server->keytype == DField::String) + { + + DELETE_ARRAY(key.bin.ptr); + key.Set(NULL, 0); + } + haskey = 0; + } + return 0; +} + +int NCRequest::UnsetKeyValue(void) { + kvl.Unset(); + flags &= ~DRequest::Flag::MultiKeyValue; + return 0; +} + +int NCRequest::SetKey(int64_t k) { + if(server==NULL) + return_err(-EC_NOT_INITIALIZED); + if(server->keytype != DField::Signed) + return_err(-EC_BAD_KEY_TYPE); + key = k; + haskey = 1; + UnsetKeyValue(); + return 0; +} + +int NCRequest::SetKey(const char *name, int l) { + if(server==NULL) + return_err(-EC_NOT_INITIALIZED); + if(server->keytype != DField::String) + return_err(-EC_BAD_KEY_TYPE); + char *a = new char[l]; + memcpy(a, name, l); + DELETE_ARRAY(key.bin.ptr); + haskey = 1; + key.Set(a, l); + UnsetKeyValue(); + return 0; +} + +int NCRequest::AddKeyValue(const char* name, const DTCValue &v, uint8_t type) +{ + if(server==NULL) + return_err(-EC_NOT_INITIALIZED); + if(server->AllowBatchKey()==0) + return_err(-EC_NOT_INITIALIZED); + + int ret = kvl.AddValue(name, v, type); + if(ret < 0) + return err = ret; + + flags |= DRequest::Flag::MultiKeyValue; + UnsetKey(); + return 0; +} + +int NCRequest::SetTabDef(void) +{ + int ret = 0; + if(server){ + switch(cmd) { + case DRequest::Nop: + case DRequest::Get: + case DRequest::Purge: + case DRequest::Flush: + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + ret = tdef != server->tdef; + tdef = server->tdef; + tablename = server->tablename; + keytype = server->keytype; + break; + //跨IDC分布支持 + case DRequest::SvrAdmin: + case DRequest::Invalidate: + ret = tdef != server->admin_tdef; + tdef = server->admin_tdef; + tablename = (char *)"@HOT_BACKUP"; + keytype = DField::Unsigned; + break; + default: + break; + } + } + return ret; +} + +int NCRequest::CheckKey(const DTCValue *kptr) +{ + int keyType = tdef->KeyType(); + int keySize = tdef->KeySize(); + if(kptr){//多key查询时 kptr为空 + if(keyType == 1 || keyType == 2){ + return CheckIntValue( *kptr, keyType, keySize); + }else if(keyType == 4 || keyType == 5){ + if(keySize < kptr->str.len) + return -EC_BAD_FIELD_SIZE_ON_CHECKKEY; + }else{ + return -EC_EXCEPTION_ON_CHECKKEY; + } + } + return 0; +} + +int NCRequest::Encode(const DTCValue *kptr, CPacket *pkt) +{ + int err; + int force = SetTabDef(); + + if(tdef &&( (err = CheckKey(kptr)) || + (err = ui.Resolve(tdef, force)) || + (err = ci.Resolve(tdef, force)) || + (err = fs.Resolve(tdef, force)) + )) + { + return err; + } + + if((err = pkt->EncodeRequest(*this, kptr)) != 0) { + return err; + } + return 0; +} + +NCResult *NCRequest::ExecuteStream(const DTCValue *kptr) +{ + int resend = 1; + int nrecv = 0; + int nsent = 0; + + while(1) + { + int err; + + if(resend) + { + CPacket pk; + + if((err=Encode(kptr, &pk)) < 0) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + if((err = server->Connect()) != 0) + { + server->IncErrCount(); + if(err==-EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + log_error("dtc connect error %d\n", err); + return_err_res(err, "API::connecting", "client api connect server error"); + } + + nsent++; + if((err = server->SendPacketStream(pk)) != 0) + { + if(server->autoReconnect && nsent <= 1 && (err==-ECONNRESET || err==-EPIPE)) + { + resend = 1; + continue; + } + return_err_res(err, "API::sending", "client api send packet error"); + } + } + + NCResult *res = new NCResult(tdef); + res->versionInfo.SetSerialNr(server->LastSerialNr()); + + err = server->DecodeResultStream(*res); + + if(err < 0) + { + if(cmd!=DRequest::Nop && err==-EAGAIN && nsent<=1) + { + resend = 1; + delete res; + continue; + } + // network error always aborted + return res; + } + nrecv++; + + if (res->versionInfo.SerialNr() != server->LastSerialNr()) + { + log_debug("SN different, receive again. my SN: %lu, result SN: %lu", + (unsigned long)server->LastSerialNr(), + (unsigned long)res->versionInfo.SerialNr()); + // receive again + resend = 0; + } else if (res->ResultCode() == -EC_CHECKSUM_MISMATCH && nrecv <= 1) + { + resend = 1; + // tabledef changed, resend + //agent restart reconnect + if(res->versionInfo.ReConnect()){ + server->Close(); + } + } + else + { + // got valid result + if (res->ResultCode() >= 0 && cmd == DRequest::Get) + res->SetVirtualMap(fs); + + if(0 != server->GetRemoveCount()) + server->ClearRemoveCount(); + + uint64_t time = 0; + if(res->resultInfo.TagPresent(7)){ + char *t = res->resultInfo.TimeInfo(); + if(t){ + CTimeInfo *t_info = (CTimeInfo *)t; + time = (t_info->time)>0 ? (t_info->time):0; + server->SetAgentTime(time); + } + } + //agent restart reconnect + if(res->versionInfo.ReConnect()){ + server->Close(); + } + return res; + } + delete res; // delete invalid result and loop + } + + // UNREACHABLE + return NULL; +} + +NCResult *NCRequest::ExecuteDgram(SocketAddress *peer, const DTCValue *kptr) +{ + int resend = 1; + int nrecv = 0; + int nsent = 0; + + while (1) + { + int err; + + if (resend) { + CPacket pk; + + if ((err = Encode(kptr, &pk)) < 0) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + nsent++; + if ((err = server->SendPacketDgram(peer, pk)) != 0) + { + if(peer==NULL && server->autoReconnect && nsent <= 1 && + (err==-ECONNRESET || err==-EPIPE || err==-ECONNREFUSED || err==-ENOTCONN)) + { + if((err = server->Reconnect()) != 0) { + if(err==-EAGAIN) { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + return_err_res(err, "API::connecting", "client api connect server error"); + } + resend = 1; + continue; + } + return_err_res(err, "API::sending", "client api send packet error"); + } + } + + NCResult *res = new NCResult(tdef); + res->versionInfo.SetSerialNr(server->LastSerialNr()); + err = server->DecodeResultDgram(peer, *res); + if(err < 0) { + // network error encountered + return res; + } + nrecv++; + + if (res->versionInfo.SerialNr() != server->LastSerialNr()) { + log_debug("SN different, receive again. my SN: %lu, result SN: %lu", + (unsigned long)server->LastSerialNr(), + (unsigned long)res->versionInfo.SerialNr()); + // receive again + resend = 0; + } else if (res->ResultCode() == -EC_CHECKSUM_MISMATCH && nrecv <= 1) { + resend = 1; + // tabledef changed, resend + } else { + // got valid result + if (res->ResultCode() >= 0 && cmd == DRequest::Get) + res->SetVirtualMap(fs); + + uint64_t time = 0; + if(res->resultInfo.TagPresent(7)){ + char *t = res->resultInfo.TimeInfo(); + if(t){ + CTimeInfo *t_info = (CTimeInfo *)t; + time = (t_info->time)>0 ? (t_info->time):0; + server->SetAgentTime(time); + } + } + return res; + } + delete res; // delete invalid result and loop + } + + /* UNREACHABLE */ + return NULL; +} + +NCResult *NCRequest::ExecuteNetwork(const DTCValue *kptr) { + NCResult *res = NULL; + if(!server->IsDgram()) + { + res = ExecuteStream(kptr); + } + else if(server->IsDgram()) + { + NCUdpPort *port = server->GetGlobalPort(); + //get UDPPORT{SN,fd} from udp pool + if (port == NULL) + { + return_err_res(err, "API::Execute", "get udp port error.udp port may exhaust"); + } + res = ExecuteDgram(&server->addr, kptr); + server->PutGlobalPort(port); + } + else + { + int err; + if((err = server->Connect()) != 0) + { + if(err==-EAGAIN) + { + // unix socket return EAGAIN if listen queue overflow + err = -EC_SERVER_BUSY; + } + return_err_res(err, "API::connecting", "client api connect server error"); + } + + res = ExecuteDgram(NULL, kptr); + } + return res; +} + +NCResult *NCRequest::ExecuteInternal(const DTCValue *kptr) +{ + if(tdef == NULL) { + return_err_res(err, "API::encoding", "internal error: request has no tdef"); + } + + int force = SetTabDef(); + + int err; + if( + (err = ui.Resolve(tdef, force)) || + (err = ci.Resolve(tdef, force)) || + (err = fs.Resolve(tdef, force)) + ) + { + return_err_res(err, "API::encoding", "client api encode error"); + } + + NCResultInternal *res = server->ExecuteInternal(*this, kptr); + + if(res == NULL) { + return_err_res(err, "API::sending", "internal client api execute error"); + } + + if(res->ResultCode() >= 0 && cmd==DRequest::Get) + res->SetVirtualMap(fs); + + return reinterpret_cast(res); +} + +NCResult *NCRequest::PreCheck(const DTCValue *kptr) { + + if( cmd == DRequest::Monitor ) + { + //Monitor Request need not check + cmd = DRequest::SvrAdmin;//MonitorRequest is a stub of RequestSvrAdmin.Agent should add MonitorRequest type later + return NULL; + } + if(err) + return_err_res(err, "API::encoding", "Init Operation Error"); + if(server==NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if(cmd==DRequest::ResultCode) + return_err_res(-EC_BAD_COMMAND, "API::encoding", "Unknown Request Type"); + if(server->badkey) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + if(server->badname) + return_err_res(-EC_BAD_TABLE_NAME, "API::encoding", "Table Name Mismatch"); + if(cmd!=DRequest::Insert && cmd!=DRequest::SvrAdmin && + !(flags & DRequest::Flag::MultiKeyValue) && kptr==NULL) + return_err_res(-EINVAL, "API::encoding", "Missing Key"); + if((flags & DRequest::Flag::MultiKeyValue) && kvl.IsFlat()==0) + return_err_res(-EINVAL, "API::encoding", "Missing Key Value"); + if (tdef==NULL) + { + int ret = 0; + uint64_t time_before = GET_TIMESTAMP(); + ret = server->Ping();//ping and get tabledef + uint64_t time_after = GET_TIMESTAMP(); + tdef = server->tdef; + + uint64_t timeInterval = 0; + if(time_after > time_before) + timeInterval = time_after - time_before; + + //log_info("timeInterval: %lu", timeInterval); + std::string accessKey = server->accessToken; + if(ret == -ETIMEDOUT){ + //server->dc->SetReportInfo(accessKey, CLIENT_CURVE, server->GetTimeout()); + server->dc->SetReportInfo(accessKey, AGENT_CURVE, server->GetTimeout()); + }else{ + //server->dc->SetReportInfo(accessKey, CLIENT_CURVE, timeInterval); + server->dc->SetReportInfo(accessKey, AGENT_CURVE, (server->GetAgentTime() != 0)?server->GetAgentTime():timeInterval); + } + server->dc->SetReportInfo(accessKey, CLIENT_CURVE, timeInterval); + + //top_percentile_report(accessKey, server->GetAddress(), timeInterval, ret, RT_SHARDING); + //top_percentile_report(accessKey, "", timeInterval, ret, RT_ALL); + server->dc->SetTopPercentileData(accessKey, server->GetAddress(), timeInterval, ret); + + log_info("NCRequest::PreCheck,seq:%lu, clientTime:%lu, agentTime:%lu", server->LastSerialNr(), timeInterval, server->GetAgentTime()); + // 清除 ping 之后获取的 agent-dtc 响应耗时 + server->SetAgentTime(0); + } + return NULL; +} + +int NCRequest::SetCompressFieldName() +{ + int iret = 0; + if (tdef && tdef->CompressFieldId()>0)//启用压缩必定有这个字段 + { + if (gzip==NULL)//读请求需要need字段 + { + if (cmd != DRequest::Get) + return 0; + iret = Need(tdef->FieldName(tdef->CompressFieldId()),0); + if (iret) + { + snprintf(_errmsg, sizeof(_errmsg), "need CompressField error,errorcode is %d",iret); + return -1; + } + } + else//写请求 + { + if (cmd == DRequest::Insert||cmd == DRequest::Replace) + { + iret = AddOperation(tdef->FieldName(tdef->CompressFieldId()),//name + DField::Set,DField::Signed,DTCValue::Make(compressFlag)); + } + else if (cmd == DRequest::Update) + { + iret = AddOperation(tdef->FieldName(tdef->CompressFieldId()),//name + DField::OR,DField::Signed,DTCValue::Make(compressFlag)); + } + else + { + snprintf(_errmsg, sizeof(_errmsg), "Request type[%d] not support compressset",cmd); + return -1; + } + if (iret) + { + snprintf(_errmsg, sizeof(_errmsg), "set CompressField error,errorcode is %d",iret); + return -1; + } + } + + } + return 0; +} +//if get,need compressfield,if write set compressfield; +NCResult *NCRequest::Execute(const DTCValue *kptr) { + NCResult *ret = NULL; + if(kptr==NULL && haskey) kptr = &key; + ret = PreCheck(kptr); + if(ret==NULL) + { + if (SetCompressFieldName()) + return_err_res(-EC_COMPRESS_ERROR, "API::Execute", "SetCompressFieldName error. please check request.ErrorMessage");; + + /*date:2014/06/09, author:xuxinxin 模调上报 */ + uint64_t time_before = GET_TIMESTAMP(); + if(server->HasInternalExecutor()) + ret = ExecuteInternal(kptr); + else + ret = ExecuteNetwork(kptr); + uint64_t time_after = GET_TIMESTAMP(); + + uint64_t timeInterval = 0; + if(time_after > time_before) + timeInterval = time_after - time_before; + + std::string accessKey = server->accessToken; + + if(ret != NULL && ret->resultInfo.ResultCode() == -ETIMEDOUT){ + //server->dc->SetReportInfo(accessKey, CLIENT_CURVE, server->GetTimeout()); + server->dc->SetReportInfo(accessKey, AGENT_CURVE, server->GetTimeout()); + }else{ + //server->dc->SetReportInfo(accessKey, CLIENT_CURVE, timeInterval); + server->dc->SetReportInfo(accessKey, AGENT_CURVE, (server->GetAgentTime() != 0)?server->GetAgentTime():timeInterval); + } + server->dc->SetReportInfo(accessKey, CLIENT_CURVE, timeInterval); + + //top_percentile_report(accessKey, server->GetAddress(), timeInterval, ret ? ret->resultInfo.ResultCode() : 1, RT_SHARDING); + //top_percentile_report(accessKey, "", timeInterval, ret ? ret->resultInfo.ResultCode() : 1, RT_ALL); + server->dc->SetTopPercentileData(accessKey, server->GetAddress(), timeInterval, ret ? ret->resultInfo.ResultCode() : 1); + + std::string stemp = accessKey.substr(0, 8); + uint32_t bid = 0; + sscanf(stemp.c_str(), "%u", &bid); + log_info("NCRequest::Execute,seq:%lu, clientTime:%lu, agentTime:%lu, bid:%u", server->LastSerialNr(), timeInterval, server->GetAgentTime(), bid); + // 清除 解包 之后获取的 agent-dtc 响应耗时 + server->SetAgentTime(0); + } + else + { + if(NULL != server) + { + log_info("NCRequest::Execute,seq:%lu, PreCheck return NULL", server->LastSerialNr()); + } + } + return ret; +} + +NCResult *NCRequest::Execute(int64_t k) { + if(server == NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if(server->keytype != DField::Signed) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + DTCValue v(k); + return Execute(&v); +} + +NCResult *NCRequest::Execute(const char *k, int l) { + if(server == NULL) + return_err_res(-EC_NOT_INITIALIZED, "API::encoding", "Server Not Initialized"); + if(server->keytype != DField::String) + return_err_res(-EC_BAD_KEY_TYPE, "API::encoding", "Key Type Mismatch"); + DTCValue v(k, l); + return Execute(&v); +} + +// packet encoding don't cache error code +int NCRequest::EncodeBuffer(char *&ptr, int &len, int64_t &magic, const DTCValue *kptr) { + if(kptr==NULL && haskey) kptr = &key; + int err = 0; + NCResult *ret = PreCheck(kptr); + if(ret!=NULL) + { + err = ret->ResultCode(); + delete ret; + return err; + } + + SetTabDef(); + if(tdef &&( + (err = ui.Resolve(tdef, 0)) || + (err = ci.Resolve(tdef, 0)) || + (err = fs.Resolve(tdef, 0)) + )) + { + return err; + } + + if((err = CPacket::EncodeSimpleRequest(*this, kptr, ptr, len)) != 0) { + return err; + } + + magic = server->LastSerialNr(); + return 0; +} + +int NCRequest::EncodeBuffer(char *&ptr, int &len, int64_t &magic, int64_t k) { + if(server == NULL) + return -EC_NOT_INITIALIZED; + if(server->keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + if(cmd==DRequest::SvrAdmin) + return -EC_BAD_COMMAND; + DTCValue v(k); + return EncodeBuffer(ptr, len, magic, &v); +} + +int NCRequest::EncodeBuffer(char *&ptr, int &len, int64_t &magic, const char *k, int l) { + if(server == NULL) + return -EC_NOT_INITIALIZED; + if(server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + DTCValue v(k, l); + return EncodeBuffer(ptr, len, magic, &v); +} +/* + * @descr: 无源模式设置超时时间 + * @param: key, dtc key value + * @param: t, 超时时间，相对时间 + * add by xuxinxin + * date: 2014/12/09 + */ +int NCRequest::SetExpireTime(const char* key, int t){ + int ret = 0; + if(key == NULL){ + log_error("invalid key value! key is null"); + return -EC_INVALID_KEY_VALUE; + } + if(t <= 0){ + log_error("invalid expireTime! expireTime:%d", t); + return -EC_INVALID_EXPIRETIME; + } + if(cmd != DRequest::Update) + { + log_error("invalid requset must be request!"); + return -EC_BAD_OPERATOR; + } + + switch(server->keytype){ + case DField::Signed: + case DField::Unsigned: + ret = SetKey(atoll(key)); + break; + case DField::String: + case DField::Binary: + ret = SetKey(key, strlen(key)); + break; + default: + break; + } + if(ret != 0) return ret; + + ret = AddOperation("_dtc_sys_expiretime", DField::Set, DField::Signed, DTCValue::Make(t)); + if(ret != 0) return ret; + NCResult * rst = Execute(); //rst 不会为空 + ret = rst->ResultCode(); + if(ret != 0){ + log_error("set expireTime fail, errmsg:%s, errfrom:%s", rst->resultInfo.ErrorMessage(), rst->resultInfo.ErrorFrom()); + } + delete rst; + rst = NULL; + return ret; +} + +/* + * @descr: 无源模式获取超时时间 + * @param: key, dtc key value + * add by xuxinxin + * date: 2014/12/11 + */ +int NCRequest::GetExpireTime(const char* key){ + int ret = 0; + if(key == NULL){ + log_error("invalid key value! key is null"); + return -EC_INVALID_KEY_VALUE; + } + + if(cmd != DRequest::Get) + { + log_error("invalid requset must be request!"); + return -EC_BAD_OPERATOR; + } + + switch(server->keytype){ + case DField::Signed: + case DField::Unsigned: + ret = SetKey(atoll(key)); + break; + case DField::String: + case DField::Binary: + ret = SetKey(key, strlen(key)); + break; + default: + break; + } + + ret = Need("_dtc_sys_expiretime", 0); + if(ret != 0) + { + log_error("get expireTime fail, need error, errcode: %d", ret); + return ret; + } + NCResult * rst = Execute(); //rst 不会为空 + ret = rst->ResultCode(); + if(ret < 0){ + log_error("get expireTime fail, errmsg:%s, errfrom:%s", rst->resultInfo.ErrorMessage(), rst->resultInfo.ErrorFrom()); + delete rst; + rst = NULL; + return ret; + } + if(rst->result==NULL) + { + log_error("result is null [rst->result==NULL]"); + delete rst; + rst = NULL; + return -EC_GET_EXPIRETIME_RESULT_NULL; + } + if(rst->result->TotalRows() <= 0){ + log_error("get expireTime fail, no data exist in dtc for key:%s", key); + delete rst; + rst = NULL; + return -EC_GET_EXPIRETIME_END_OF_RESULT; + } + ret = rst->result->DecodeRow(); + if(ret < 0){ + log_error("get expireTime fail, fetch_row error, errmsg:%s, errfrom:%s", rst->resultInfo.ErrorMessage(), rst->resultInfo.ErrorFrom()); + delete rst; + rst = NULL; + return ret; + } + int expiretime = 0; + int id = rst->FieldId("_dtc_sys_expiretime"); + if(id >= 0) { + const DTCValue *v; + if(id==0 && !(rst->result->FieldPresent(0))) + v = rst->ResultKey(); + else + v = rst->result->FieldValue(id); + + if(v) { + switch(rst->FieldType(id)) { + case DField::Signed: + case DField::Unsigned: + { + expiretime = v->s64; + break; + } + case DField::Float: + { + expiretime = llround(v->flt); + //return (int64_t)(v->flt); + break; + } + case DField::String: + { + if(v->str.ptr) + expiretime = atoll(v->str.ptr); + break; + } + } + } + }else{ + log_error("can not find field expiretime"); + delete rst; + rst = NULL; + return -EC_GET_EXPIRETIME_FIELD_EXPIRETIME_NULL; + } + + delete rst; + rst = NULL; + return expiretime; +} diff --git a/src/search_local/index_storage/api/c_api_cc/dtcsvr.cc b/src/search_local/index_storage/api/c_api_cc/dtcsvr.cc new file mode 100644 index 0000000..a4e3a2c --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcsvr.cc @@ -0,0 +1,1468 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include "dtcint.h" +#include +#include +#include "curl_http.h" +#include "json/json.h" + +using namespace std; + +const uint32_t ACCESS_KEY_LEN = 40; +#define CLIENT_CURVE 5 +#define AGENT_CURVE 8 + +const char* dtc_api_ver="dtc-api-ver:4.3.4"; +const char* dtc_api_complite="dtc-api-complite-date:"__DATE__" "__TIME__; + +int NCServer::_network_mode = 0; +char * NCServer::_server_address = NULL; +char * NCServer::_server_tablename = NULL; +extern "C" void set_network_mode(void) +{ + NCServer::_network_mode = 1; +} + +extern "C" void set_server_address(const char * address) +{ + NCServer::_server_address = strdup(address); +} + +extern "C" void set_server_tablename(const char * tablename) +{ + NCServer::_server_tablename = strdup(tablename); +} + +NCServer::NCServer(void) +{ + keytype = DField::None; + + tablename = NULL; + appname = NULL; + + autoUpdateTable = 1; + autoReconnect = 1; + tdef = NULL; + admin_tdef = NULL; + + unsigned int a = time(NULL); + a = rand_r(&a) + (long)this; + lastSN = rand_r(&a); + + completed = 0; + badkey = 0; + badname = 0; + autoping = 0; + errstr = NULL; + + timeout = 5000; + realtmo = 0; + + iservice = NULL; + executor = NULL; + netfd = -1; + lastAct = 0; + pingReq = NULL; + + ownerPool = NULL; + ownerId = 0; + dc = DataConnector::getInstance(); + + m_iErrCount = 0; + m_iTotalCount = 0; + m_iTotalElaps = 0; + m_RemoveCount = 0; +} + + +// copy most members, zero connection state +// copy address if use generic address +NCServer::NCServer(const NCServer &that) : + addr(that.addr), + keyinfo(that.keyinfo) +{ +#define _MCOPY(x) this->x = that.x +#define _MZERO(x) this->x = 0 +#define _MNEG(x) this->x = -1 +#define _MDUP(x) this->x = that.x ? STRDUP(that.x) : NULL + + _MDUP(tablename); + _MDUP(appname); + _MCOPY(keytype); + _MCOPY(autoUpdateTable); + _MCOPY(autoReconnect); + + _MCOPY(completed); + _MCOPY(badkey); + _MCOPY(badname); + _MCOPY(autoping); + // errstr always compile time const string + _MCOPY(errstr); + + _MCOPY(tdef); + _MCOPY(admin_tdef); + if(tdef) tdef->INC(); + if(admin_tdef) admin_tdef->INC(); + + lastAct = time(NULL); + _MCOPY(timeout); + _MZERO(realtmo); // this is sockopt set to real netfd + _MCOPY(iservice); + _MCOPY(executor); + _MNEG(netfd); // real socket fd, cleared + _MZERO(pingReq); // cached ping request object + + unsigned int a = lastAct; + a = rand_r(&a) + (long)this; + lastSN = rand_r(&a); + + + _MZERO(ownerPool); // serverpool linkage + _MZERO(ownerId); // serverpool linkage + +#undef _MCOPY +#undef _MZERO +#undef _MNEG +#undef _MDUP +} + +NCServer::~NCServer(void) { + DELETE(pingReq); + Close(); + FREE_IF(tablename); + FREE_IF(appname); + DEC_DELETE(tdef); + DEC_DELETE(admin_tdef); + DataConnector::getInstance()->SendData(); // 防止10s 间隔时间未到，程序先结束了少报一次 + DataConnector::getInstance()->SendTopPercentileData(); +} + +DataConnector* NCServer::dc = NULL;//DataConnector::getInstance(); + +void NCServer::CloneTabDef(const NCServer& source) +{ + DEC_DELETE(tdef); + tdef = source.tdef; + tdef->INC(); + if(tablename != NULL) + FREE(tablename); + tablename = STRDUP(source.tablename); + keytype = source.keytype; + completed = 1; +} + +int NCServer::SetAddress(const char *h, const char *p) { + if(h==NULL) { + errstr = "No Host Specified"; + return -EC_BAD_HOST_STRING; + } + + char *oldhost = NULL; + if(addr.Name() != NULL) { + // dup string put on stack + oldhost = strdupa(addr.Name()); + } + + const char *err = addr.SetAddress(h, p); + if(err) { + this->errstr = err; + return -EC_BAD_HOST_STRING; + } + + // un-changed + if(oldhost!=NULL && !strcmp(addr.Name(), oldhost)) + return 0; + + Close(); + + //set network model + executor = NULL; + if(!_network_mode&&iservice && iservice->MatchListeningPorts(addr.Name(), NULL)) { + executor = iservice->QueryTaskExecutor(); + DTCTableDefinition *t1 = iservice->QueryTableDefinition(); + if(t1 != this->tdef) { + DEC_DELETE(this->tdef); + this->tdef = t1; + } + DTCTableDefinition *t2 = iservice->QueryAdminTableDefinition(); + if(t2 != this->tdef) { + DEC_DELETE(this->admin_tdef); + this->admin_tdef = t2; + } + } + + return 0; +} + +int NCServer::IntKey(void) { + if(keytype != DField::None && keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + keytype = DField::Signed; + if(tablename != NULL) completed = 1; + return 0; +} + +int NCServer::StringKey(void) { + if(keytype != DField::None && keytype != DField::String) + return -EC_BAD_KEY_TYPE; + keytype = DField::String; + if(tablename != NULL) completed = 1; + return 0; +} + +int NCServer::AddKey(const char* name, uint8_t type) +{ + int ret = keyinfo.AddKey(name, type); + // TODO verify tabledef + if(ret==0) { + if(tablename != NULL) completed = 1; + if(keytype == DField::None) + keytype = type; + } + return 0; +} + +int NCServer::SetTableName(const char *name) { + if(name==NULL) return -EC_BAD_TABLE_NAME; + + if(tablename) + return mystrcmp(name, tablename, 256)==0 ? 0 : -EC_BAD_TABLE_NAME; + + tablename = STRDUP(name); + + if(&NCServer::CheckInternalService != 0 && !_network_mode) { + CheckInternalService(); + } + + if(keytype != DField::None) completed = 1; + + return 0; +} + +int NCServer::FieldType(const char *name) { + if(tdef == NULL) + Ping(); + + if(tdef != NULL){ + int id = tdef->FieldId(name); + if(id >= 0) + return tdef->FieldType(id); + } + + return DField::None; +} + +//fixed, sec --> msec +void NCServer::SetMTimeout(int n) { + timeout = n<=0 ? 5000 : n; + UpdateTimeout(); +} + +void NCServer::UpdateTimeout(void) { + if(netfd >= 0 && realtmo != timeout) { + struct timeval tv = { timeout/1000, (timeout%1000)*1000 }; + setsockopt(netfd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + if(timeout) { + if(tv.tv_usec > 1000000) + { + tv.tv_usec += 100000; // add more 100ms + }else { + tv.tv_usec += 100; // add more 100us + } + if(tv.tv_usec >= 1000000) + { + tv.tv_usec -= 1000000; + tv.tv_sec +=1; + } + } + setsockopt(netfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + realtmo = timeout; + } +} + +void NCServer::UpdateTimeoutAnyway(void){ + if(netfd >= 0) { + struct timeval tv = { timeout/1000, (timeout%1000)*1000 }; + setsockopt(netfd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + if(timeout) { + if(tv.tv_usec > 1000000) + { + tv.tv_usec += 100000; // add more 100ms + }else { + tv.tv_usec += 100; // add more 100us + } + if(tv.tv_usec >= 1000000) + { + tv.tv_usec -= 1000000; + tv.tv_sec +=1; + } + } + setsockopt(netfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + realtmo = timeout; + } +} + +void NCServer::Close(void){ + if(netfd>=0) { + close(netfd); + netfd = -1; + realtmo=0; + } +} + +int NCServer::BindTempUnixSocket(void) { + struct sockaddr_un tempaddr; + tempaddr.sun_family = AF_UNIX; + snprintf(tempaddr.sun_path, sizeof(tempaddr.sun_path), + "@dtcapi-%d-%d-%d", getpid(), netfd, (int)time(NULL)); + socklen_t len = SUN_LEN(&tempaddr); + tempaddr.sun_path[0] = 0; + return bind(netfd, (const sockaddr *)&tempaddr, len); +} + +int NCServer::Connect(void) { + if(ownerPool != NULL) + return -EC_PARALLEL_MODE; + if(executor != NULL) + return 0; + if(netfd >= 0) + return 0; + + int err = -EC_NOT_INITIALIZED; + + if(addr.SocketFamily() != 0) { + netfd = addr.CreateSocket(); + if(netfd < 0) { + log_error("create socket error: %d, %m", errno); + err = -errno; + } else if(addr.SocketFamily()==AF_UNIX && IsDgram() && BindTempUnixSocket() < 0) { + log_error("bind unix socket error: %d, %m", errno); + err = -errno; + Close(); + } else { + int iRes = -1; + //先将netfd设置为非阻塞模式，然后进行connect，返回0，则建立连接成功；返回其它失败 + fcntl(netfd, F_SETFL, fcntl(netfd, F_GETFL, 0) | O_NONBLOCK); + if(addr.ConnectSocket(netfd)==0) { + iRes = 0; + } + else{ + int iTimeout = timeout<=0 ? 5000 : timeout; + //struct pollfd *pollfds = (struct pollfd *) calloc(1, sizeof(struct pollfd)); + struct pollfd pollfds[1]; + + pollfds[0].fd = netfd; + pollfds[0].events = POLLIN | POLLOUT | POLLERR; + int ready_num = poll(pollfds, 1, iTimeout); + if (ready_num < 0) { + log_error("network error in connect, errno [%d], errmsg [%s]", errno, strerror(errno)); + } + else if (ready_num == 0) { + log_error("connect time out, errno [%d], errmsg [%s]", errno, strerror(errno)); + } + else { + if (pollfds[0].revents & POLLERR) { + log_error("network error in connect, errno [%d], errmsg [%s]", errno, strerror(errno)); + } + else if (pollfds[0].revents & POLLOUT || pollfds[0].revents & POLLIN) { + iRes = 0; + } + + } + } + //将netfd恢复为阻塞模式 + fcntl(netfd, F_SETFL, fcntl(netfd, F_GETFL, 0) & ~O_NONBLOCK); + int keepalive = 1; //开启keepalive属性 + int keepidle = 60; //如该连接在60秒内没有任何数据往来,则进行探测 + int keepinterval = 5; //探测时发包的时间间隔为5 秒 + int keepcount = 3; //探测尝试的次数。如果第1次探测包就收到响应了,则后2次的不再发。 + setsockopt(netfd, SOL_SOCKET, SO_KEEPALIVE, (void *)&keepalive , sizeof(keepalive )); + setsockopt(netfd, SOL_TCP, TCP_KEEPIDLE, (void*)&keepidle , sizeof(keepidle )); + setsockopt(netfd, SOL_TCP, TCP_KEEPINTVL, (void *)&keepinterval , sizeof(keepinterval )); + setsockopt(netfd, SOL_TCP, TCP_KEEPCNT, (void *)&keepcount , sizeof(keepcount )); + UpdateTimeoutAnyway(); + + if(0 == iRes) + return 0; + log_error("connect dtc server error: %d,%m", errno); + err = -errno; + if(err==-EINPROGRESS) err = -ETIMEDOUT; + Close(); + } + } + return err; +} + +int NCServer::Reconnect(void) { + if(ownerPool != NULL) + return -EC_PARALLEL_MODE; + if(executor != NULL) + return 0; + if(netfd < 0) + return -ENOTCONN; + + if(addr.ConnectSocket(netfd)==0) { + return 0; + } + log_error("connect dtc server error: %d,%m", errno); + int err = -errno; + if(err==-EINPROGRESS) err = -ETIMEDOUT; + Close(); + return err; +} + +int NCServer::SendPacketStream(CPacket &pk) { + int err; +#ifndef SIOCOUTQ + if(1) //if(!IsDgram()) + { + char tmp[1]; + err = recv(netfd, tmp, sizeof(tmp), MSG_PEEK|MSG_DONTWAIT); + if(err==0 || (err<0 && errno != EAGAIN && errno != EWOULDBLOCK)) + { + if(!autoReconnect) + { + log_error("dtc server close connection: %d,%m", err); + return err; + } + else + { + log_debug("%s", "dtc server close connection, re-connect now."); + Close(); + err = Connect(); + if(err) return err; + } + } + } +#endif + + while((err = pk.Send(netfd)) == SendResultMoreData) + { + if(errno==EINPROGRESS) + { + log_error("socket[%d] send data timeout: %d,%m", netfd, errno); + // timeout + if(1) // if(!IsDgram()) + Close(); + return -ETIMEDOUT; + } + } + if(err != SendResultDone) + { + log_error("socket[%d] send data error: %d,%m", netfd, err); + err = -errno; + if(1) // if(!IsDgram()) + Close(); + return err; + } + return 0; +} + +// always return network error code +// EAGAIN must re-sent +int NCServer::DecodeResultStream(NCResult &tk) +{ + if(netfd < 0) + { + tk.SetError(-ENOTCONN, "API::recving", "connection is closed"); + return -ENOTCONN; + } + + int err; + CSimpleReceiver receiver(netfd); + + uint64_t beginTime = 0; + //log_debug("wait response time stamp = %lld", (long long)(beginTime = GET_TIMESTAMP())); + + while(1) + { + errno = 0; + err = tk.Decode(receiver); + + if(err == DecodeFatalError) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + + err = -errno; + if(err==0) + err = -ECONNRESET; +#ifdef SIOCOUTQ + int value = 0; + ioctl(netfd, SIOCOUTQ, &value); + // unsent bytes + if(value > 0) { + err = -EAGAIN; + tk.SetError(err, "API::sending", "client send packet error"); + Close(); + return err; + } +#endif + Close(); + // EAGAIN never return FatalError, should be DecodeWaitData + tk.SetError(err, "API::recving", "client recv packet error"); + return err; + } + + if(err == DecodeDataError) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + + // packet is valid + return 0; + } + + if(err == DecodeDone) + { + break; + } + + if(errno == EINPROGRESS || errno == EAGAIN) + { + log_error("socket[%d] decode data error: %d, %m", netfd, errno); + log_debug("use time %ldms", (long)((GET_TIMESTAMP()-beginTime)/1000)); + + Close(); + tk.SetError(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + return -ETIMEDOUT; + } + } + + //log_debug("decode result use time: %ldms", (long)((GET_TIMESTAMP()-beginTime)/1000)); + + SaveDefinition(&tk); + if(autoping) { + time(&lastAct); + err = tk.versionInfo.KeepAliveTimeout(); + if(err<15) err = 15; + lastAct += err - 1; + } + return 0; +} + +int NCServer::SendPacketDgram(SocketAddress *peer, CPacket &pk) +{ + int err = 0; + + while ((err = pk.SendTo(netfd, peer)) == SendResultMoreData) + { + if (errno == EINPROGRESS) + { + log_error("socket[%d] send data timeout: %d,%m", netfd, errno); + // timeout + return -ETIMEDOUT; + } + } + + if (err != SendResultDone) + { + log_error("socket[%d] send data error: %d,%m", netfd, err); + err = -errno; + return err; + } + return 0; +} + +static inline char *RecvFromFixedSize(int fd, int len, int &err, struct sockaddr *peeraddr, socklen_t *addrlen) +{ + int blen = len <= 0 ? 1 : len; + char *buf = (char *)MALLOC(blen); + blen = recvfrom(fd, buf, blen, 0, peeraddr, addrlen); + if(blen < 0) { + err = errno; + free(buf); + buf = NULL; + } else if(blen != len) { + err = EIO; + free(buf); + buf = NULL; + } else { + err = 0; + } + + return buf; +} + +static char *RecvPacketPeek(int fd, int &len, int &err) +{ + // MSG_PEEK + MSG_TRUNC get next packet size + char dummy[1]; + len = recv(fd, dummy, 1, MSG_PEEK|MSG_TRUNC); + if(len < 0) { + err = errno; + return NULL; + } + + return RecvFromFixedSize(fd, len, err, NULL, NULL); +} + +static char *RecvPacketPeekPeer(int fd, SocketAddress *peer, int &len, int &err) +{ + struct sockaddr peeraddr; + socklen_t sock_len; + + for (int n=0; n<10; n++) { + char dummy[1]; + + sock_len = sizeof(peeraddr); + len = recvfrom(fd, dummy, 1, MSG_PEEK|MSG_TRUNC, &peeraddr, &sock_len); + if(len < 0) { + err = errno; + return NULL; + } + + if(peer->Equal(&peeraddr, sock_len, SOCK_DGRAM)) + break; + } + + return RecvFromFixedSize(fd, len, err, &peeraddr, &sock_len); +} + +static char *RecvPacketPoll(int fd, int msec, int &len, int &err) +{ + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + int n; + + while( (n = poll(&pfd, 1, msec)) < 0 && errno == EINTR) + /* LOOP IF EINTR */; + + if(n == 0) { + err = EAGAIN; + return NULL; + } + if(n < 0) { + err = errno; + return NULL; + } + + len = 0; + ioctl(fd, SIOCINQ, &len); + + return RecvFromFixedSize(fd, len, err, NULL, NULL); +} + +// always return network error code +// EAGAIN must re-sent +int NCServer::DecodeResultDgram(SocketAddress *peer, NCResult &tk) +{ + if(netfd < 0) + { + tk.SetError(-ENOTCONN, "API::recving", "connection is closed"); + return -ENOTCONN; + } + + int saved_errno; // saved errno + int len; // recv len + char *buf; // packet buffer + uint64_t beginTime = 0; + + log_debug("wait response time stamp = %lld", (long long)(beginTime = GET_TIMESTAMP())); + + if(addr.SocketFamily() == AF_UNIX) { + if(peer == NULL) { + buf = RecvPacketPoll(netfd, timeout, len, saved_errno); + } else { + if(peer->ConnectSocket(netfd) < 0) { + saved_errno = errno; + log_error("connect dtc server error: %d,%m", saved_errno); + if(saved_errno == EAGAIN) { + // unix socket return EAGAIN if listen queue overflow + saved_errno = EC_SERVER_BUSY; + } + tk.SetError(-saved_errno, "API::connecting", "client api connect server error"); + return -saved_errno; + } + + buf = RecvPacketPoll(netfd, timeout, len, saved_errno); + + // Un-connect + struct sockaddr addr; + addr.sa_family = AF_UNSPEC; + connect(netfd, &addr, sizeof(addr)); + } + } else { + if(peer == NULL) { + buf = RecvPacketPeek(netfd, len, saved_errno); + } else { + buf = RecvPacketPeekPeer(netfd, peer, len, saved_errno); + } + } + log_debug("receive result use time: %ldms", (long)((GET_TIMESTAMP()-beginTime)/1000)); + + if(buf == NULL) + { + errno = saved_errno; + log_error("socket[%d] recv udp data error: %d,%m", netfd, saved_errno); + + if(saved_errno == EAGAIN) { + // change errcode to TIMEDOUT; + saved_errno = ETIMEDOUT; + tk.SetError(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + } else { + if(saved_errno==0) + saved_errno = ECONNRESET; + tk.SetError(-saved_errno, "API::recving", "connection reset by peer"); + } + return -saved_errno; + } + + int err = tk.Decode(buf, len, 1 /*eat buf*/); + + switch(err) { + case DecodeFatalError: + log_error("socket[%d] decode udp data error: invalid packet content", netfd); + tk.SetError(-ECONNRESET, "API::recving", "invalid packet content"); + FREE_IF(buf); // buf not eaten + return -ECONNRESET; + + case DecodeWaitData: + default: + // UNREACHABLE, DecodePacket never return these code + log_error("socket[%d] decode udp data error: %d, %m", netfd, errno); + tk.SetError(-ETIMEDOUT, "API::recving", "client recv packet timedout"); + return -ETIMEDOUT; + + case DecodeDataError: + // packet maybe valid + break; + + case DecodeDone: + SaveDefinition(&tk); + break; + + } + + return 0; +} + +//Get udpport form udppool +NCUdpPort *NCServer::GetGlobalPort() +{ + NCUdpPort *udpport = NCUdpPort::Get(addr.SocketFamily()); + + if (udpport) + { + netfd = udpport->fd; + lastSN = udpport->sn; + realtmo = udpport->timeout; + UpdateTimeout(); + } + + return udpport; +} + +//Put the udpport to the udppool +void NCServer::PutGlobalPort(NCUdpPort *udpport) +{ + if (udpport) + { + if (netfd > 0) + { + udpport->sn = lastSN; + udpport->timeout = realtmo; + netfd = -1; + udpport->Put(); + } + else + { + //if fd error, delete this udpport object + udpport->Eat(); + } + } +} + +void NCServer::SaveDefinition(NCResult *tk) +{ + if(strcmp("*", tablename)!=0 && tk->ResultCode()==-EC_TABLE_MISMATCH) { + badname = 1; + errstr = "Table Name Mismatch"; + return; + } + if(strcmp("*", tablename)!=0 && tk->ResultCode()==-EC_BAD_KEY_TYPE) { + badkey = 1; + errstr = "Key Type Mismatch"; + return; + } + + DTCTableDefinition *t = tk->RemoteTableDefinition(); + if(t == NULL) return; + + if(t->IsAdminTable()){ + if(admin_tdef) + { + if(admin_tdef->IsSameTable(t)) return; + if(!autoUpdateTable){ + badname = 1; + errstr = "Table Mismatch"; + tk->SetError(-EC_TABLE_MISMATCH, "API::executed", "AdminTable Mismatch"); + return; + } + DEC_DELETE(admin_tdef); + } + admin_tdef = t; + admin_tdef->INC(); + } + else{ + if(tdef) + { + if(tdef->IsSameTable(t)) return; + if(!autoUpdateTable){ + badname = 1; + errstr = "Table Mismatch"; + tk->SetError(-EC_TABLE_MISMATCH, "API::executed", "Table Mismatch"); + return; + } + DEC_DELETE(tdef); + } + tdef = t; + tdef->INC(); + + FREE(tablename); + tablename = STRDUP(tdef->TableName()); + keytype = tdef->KeyType(); + + //bugfix， by ada + if(keytype == DField::Unsigned) + keytype = DField::Signed; + + if(keytype == DField::Binary) + keytype = DField::String; + } +} + +void NCServer::TryPing(void) { + if(autoping && netfd >= 0) { + time_t now; + time(&now); + if(now >= lastAct) + Ping(); + } +} + +int NCServer::Ping(void) { + if(ownerPool != NULL) + return -EC_PARALLEL_MODE; + if(executor != NULL) + return 0; + NCRequest r(this, DRequest::Nop); + NCResult *t = r.ExecuteNetwork(); + int ret = t->ResultCode(); + delete t; + return ret; +} + +NCResult *NCServer::DecodeBuffer(const char *buf, int len) +{ + NCResult *res = new NCResult(tdef); + + switch(len <= (int)sizeof(CPacketHeader) ? DecodeFatalError : res->Decode(buf, len)) + { + default: + res->SetError(-EINVAL, "API::decoding", "invalid packet content"); + break; + case DecodeDataError: + case DecodeDone: + break; + } + return res; +} + +int NCServer::CheckPacketSize(const char *buf, int len) +{ + return NCResult::CheckPacketSize(buf, len); +} + +/*date:2014/06/04, author:xuxinxin*/ +int NCServer::SetAccessKey(const char *token) +{ + int ret = 0; + std::string str; + if(token == NULL) + return -EC_BAD_ACCESS_KEY; + else + str = token; + if(str.length() != ACCESS_KEY_LEN) + { + log_error("Invalid accessKey!"); + accessToken = ""; + return -EC_BAD_ACCESS_KEY; + } + else + accessToken = str; + + /* 截取 AccessKey中前32位以后的bid， 以便上报至数据采集 dtchttpd */ + ret = dc->SetBussinessId(accessToken); + //ret = dc->SetTopPercentileConfig(accessToken, this->GetAddress()); + return 0; +} + +DataConnector* DataConnector::pDataConnector = NULL; + +pthread_mutex_t wakeLock; +static void *Process(void *p) +{ + sigset_t sset; + sigfillset(&sset); + sigdelset(&sset, SIGSEGV); + sigdelset(&sset, SIGBUS); + sigdelset(&sset, SIGABRT); + sigdelset(&sset, SIGILL); + sigdelset(&sset, SIGCHLD); + sigdelset(&sset, SIGFPE); + pthread_sigmask(SIG_BLOCK, &sset, &sset); + + int ret = 0; + time_t next = 0; + struct timeval tv; + gettimeofday(&tv, NULL); + int offset = random() % 10; + next = (tv.tv_sec / 10) * 10 + 10 + offset; + struct timespec ts; + ts.tv_sec = next; + ts.tv_nsec = 0; + + while(pthread_mutex_timedlock(&wakeLock, &ts)!=0) + { + gettimeofday(&tv, NULL); + if(tv.tv_sec >= next) + { + ret = DataConnector::getInstance()->SendData(); + ret = DataConnector::getInstance()->SendTopPercentileData(); + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10 + offset; + } + ts.tv_sec = next; + ts.tv_nsec = 0; + } + pthread_mutex_unlock(&wakeLock); + return NULL; +} + +/* date:2014/06/09, author:xuxinxin */ +DataConnector::DataConnector() +{ +} + +DataConnector::~DataConnector() +{ +} + +int DataConnector::SendData() +{ + int ret = 0; + std::map mapData; + GetReportInfo(mapData); + int flag = 0; + for(std::map::iterator itrflag = mapData.begin(); itrflag != mapData.end(); ++itrflag) + { + flag += itrflag->second.TotalTime + itrflag->second.TotalRequests; + } + if(flag == 0) + { + return 0; + } + int HttpServiceTimeOut = 2; + std::string strServiceUrl = "http://report.dtc.jd.com:9090"; + std::string strRsp = ""; + CurlHttp curlHttp; + Json::FastWriter writer; + Json::Value statics; + + for(std::map::iterator itr = mapData.begin(); itr != mapData.end(); ++itr) + { + log_info("bid:%d, curve:%u, TotalTime:%lu, TotalRequests:%u", itr->first.bid, itr->first.curve, itr->second.TotalTime, itr->second.TotalRequests); + uint32_t bid = 0; + uint32_t curve = 0; + bid = itr->first.bid; + curve = itr->first.curve; + + uint64_t totalTime = itr->second.TotalTime; + uint32_t reqNum = itr->second.TotalRequests; + Json::Value content; + Json::Value inner; + Json::Value inArr; + + Json::Value middle; + if(0 != totalTime && 0 != reqNum) + { + inner["curve"] = curve; + inner["etype"] = "1"; + inner["data1"] = static_cast(totalTime); + inner["data2"] = reqNum; + inner["extra"] = ""; + inner["cmd"] = "0"; + inArr.append(inner); + + middle["bid"] = bid; + middle["content"] =inArr; + + statics.append(middle); + } + else + { + log_error("empty data in bid:[%d]!curve:[%d]", bid, curve); + } + + } + Json::Value body; + body["cmd"] = 3; + body["statics"]=statics; + BuffV buf; + std::string strBody = writer.write(body); + log_info("HttpBody = [%s]", strBody.c_str()); + curlHttp.SetHttpParams("%s", strBody.c_str()); + curlHttp.SetTimeout(HttpServiceTimeOut); + ret = curlHttp.HttpRequest(strServiceUrl, &buf, false); + if(ret != 0){ + log_error("report http error! curlHttp.HttpRequest error ret:%d", ret); + } + strRsp = buf.Ptr(); + log_info("ret:%d, strRsp:%s", ret, strRsp.c_str()); + + Json::Value root; + Json::Reader reader; + if (!reader.parse(strRsp.c_str(), root)) + { + log_error("parse Json failed, strRsp:%s", strRsp.c_str()); + } + + std::string strRetCode = root["retCode"].asString(); + log_info("strRetCode:%s", strRetCode.c_str()); + + if(strRetCode == "1") + { + log_info("call dtchttpd success!"); + } + else + { + log_error("call dtchttpd failed! strRetCode:%s", strRetCode.c_str()); + } + return 0; +} + +int DataConnector::SetReportInfo(const std::string str, const uint32_t curve, const uint64_t t) +{ + uint32_t bid = 0; + std::string stemp = str.substr(0,8); + sscanf(stemp.c_str(), "%u", &bid); + struct bidCurve bc; + bc.bid = bid; + bc.curve = curve; + + if(mapBi.find(bc) != mapBi.end()) + { + do + { + CScopedLock lock(_lock); + mapBi[bc].TotalTime += t; + mapBi[bc].TotalRequests += 1; + }while(0); + } + return 0; +} + +void DataConnector::GetReportInfo(std::map &map) +{ + CScopedLock lock(_lock); + for(std::map::iterator it = mapBi.begin(); it != mapBi.end(); ++it) + { + struct businessStatistics bs; + map[it->first] = it->second; + /* mapBi 取出来之后，不管后面是否上报成功，该bid对应的数据都丢弃，重置为0 */ + mapBi[it->first] = bs; + } +} + +int DataConnector::SetBussinessId(std::string str) +{ + CScopedLock lock(_lock); + if(mapBi.size() == 0) + { + pthread_mutex_init(&wakeLock, NULL); + if(pthread_mutex_trylock(&wakeLock) == 0) + { + int ret = pthread_create(&threadid, 0, Process, (void *)NULL); + if(ret != 0) + { + errno = ret; + return -1; + } + } + } + std::string stemp = str.substr(0, 8); + uint32_t bid = 0; + sscanf(stemp.c_str(), "%u", &bid); + + struct businessStatistics bs; + struct bidCurve bclient, bagent; + bclient.bid = bid; + bclient.curve = CLIENT_CURVE; + + bagent.bid = bid; + bagent.curve = AGENT_CURVE; + mapBi.insert(std::make_pair(bclient, bs)); + mapBi.insert(std::make_pair(bagent, bs)); + return 0; +} + +int DataConnector::SendTopPercentileData() +{ + std::map mapStat; + GetTopPercentileData(mapStat); + + int HttpServiceTimeOut = 2; + std::string strServiceUrl = "http://tp99dtc.m.jd.local/tp99/forSdk"; + std::string strRsp = ""; + CurlHttp curlHttp; + Json::FastWriter writer; + Json::Value statics; + + uint8_t isHaveData = 0; + for(std::map::iterator it = mapStat.begin(); it != mapStat.end(); ++it) + { + if(0 == it->second.uiTotalRequests || 0 == it->second.uiTotalTime) + continue; + + if(!isHaveData) + isHaveData = 1; + + Json::Value sharding; + sharding["bid"] = it->second.uiBid; + sharding["ip"] = it->second.uiAgentIP; + sharding["port"] = it->second.uiAgentPort; + sharding["totalCount"] = it->second.uiTotalRequests; + sharding["totalTime"] = static_cast(it->second.uiTotalTime); + sharding["failCount"] = it->second.uiFailCount; + sharding["maxTime"] = static_cast(it->second.uiMaxTime); + sharding["minTime"] = static_cast(it->second.uiMinTime); + + for(uint16_t iNum = 0; iNum < (sizeof(it->second.statArr) / sizeof(it->second.statArr[0])); ++ iNum) + { + sharding["stat"].append(it->second.statArr[iNum]); + } + + statics["content"].append(sharding); + } + + if(!isHaveData) + return 0; + struct timeval stTV; + gettimeofday(&stTV, NULL); + statics["datetime"] = static_cast(stTV.tv_sec); + + BuffV buf; + int ret = 0; + std::string strStat = writer.write(statics); + log_info("HttpBody = [%s]", strStat.c_str()); + curlHttp.SetHttpParams("%s", strStat.c_str()); + curlHttp.SetTimeout(HttpServiceTimeOut); + ret = curlHttp.HttpRequest(strServiceUrl, &buf, false); + if(ret != 0) + { + log_error("report http error! curlHttp.HttpRequest error ret:%d", ret); + return 0; + } + + strRsp = buf.Ptr(); + log_info("ret:%d, strRsp:%s", ret, strRsp.c_str()); + + Json::Value root; + Json::Reader reader; + if (!reader.parse(strRsp.c_str(), root)) + { + log_error("parse Json failed, strRsp:%s", strRsp.c_str()); + return 0; + } + + std::string strRetCode = root["retCode"].asString(); + log_info("strRetCode:%s", strRetCode.c_str()); + + if(strRetCode == "1") + log_info("call dtchttpd success!"); + else + log_error("call dtchttpd failed! strRetCode:%s", strRetCode.c_str()); + + return 0; +} + +/* + *@strAgentAddr:192.168.145.135:20024/tcp + */ +int DataConnector::SetTopPercentileData(const std::string strAccessKey, const std::string strAgentAddr, const uint64_t elapse, const int status) +{ + if(strAccessKey.empty()) + return -1; + if(strAgentAddr.empty()) + return -1; + if(!isdigit(strAgentAddr[0])) + return -1; + + uint32_t uiBid = 0; + std::string stemp = strAccessKey.substr(0,8); + sscanf(stemp.c_str(), "%u", &uiBid); + + uint64_t uiIP; + uint64_t uiPort; + if(ParseAddr(strAgentAddr, &uiIP, &uiPort) < 0) + return -1; + + int16_t iRealSite = CTopPercentileSection::GetTPSection(elapse); + if(iRealSite < 0 || iRealSite >= sizeof(kpi_sample_count) / sizeof(kpi_sample_count[0])) + return -1; + + uint64_t uiKey = (uiIP << 32) | (uiPort << 16) | (uint64_t)uiBid; + std::map::iterator it; + CScopedLock lock(m_tp_lock); + if((it = mapTPStat.find(uiKey)) == mapTPStat.end()) + { + DataConnector::top_percentile_statistics tpStat; + tpStat.uiBid = uiBid; + tpStat.uiAgentIP = uiIP; + tpStat.uiAgentPort = uiPort; + std::pair::iterator, bool> pairRet; + pairRet = mapTPStat.insert(std::make_pair(uiKey, tpStat)); + if(!pairRet.second) + return -1; + it = pairRet.first; + } + + if(it->second.uiMaxTime < elapse) + it->second.uiMaxTime = elapse; + if(0 == it->second.uiMinTime) + it->second.uiMinTime = elapse; + else if(it->second.uiMinTime > elapse) + it->second.uiMinTime = elapse; + if(0 != status) + it->second.uiFailCount ++; + it->second.uiTotalRequests ++; + it->second.uiTotalTime += elapse; + it->second.statArr[iRealSite] ++; + + return 0; +} + +void DataConnector::GetTopPercentileData(std::map &mapStat) +{ + CScopedLock lock(m_tp_lock); + std::swap(mapTPStat, mapStat); +} + +/* + *@strAccessKey:00001460ccea307caad065107c936c88bc59b1a4 + *@strAgentAddr:192.168.145.135:20024/tcp + */ +int DataConnector::SetTopPercentileConfig(const std::string strAccessKey, const std::string strAgentAddr) +{ + if(strAccessKey.empty()) + return -1; + if(strAgentAddr.empty()) + return -1; + if(!isdigit(strAgentAddr[0])) + return -1; + uint64_t uiIP = 0; + uint64_t uiPort = 0; + if(ParseAddr(strAgentAddr, &uiIP, &uiPort) < 0) + return -1; + uint32_t uiBid = 0; + std::string strTemp = strAccessKey.substr(0,8); + sscanf(strTemp.c_str(), "%u", &uiBid); + struct top_percentile_statistics tpStat; + tpStat.uiBid = uiBid; + tpStat.uiAgentIP = (uint32_t)uiIP; + tpStat.uiAgentPort = (uint16_t)uiPort; + mapTPStat.insert(std::make_pair((uiIP << 32) | (uiPort << 16) | uiBid, tpStat)); + return 0; +} + +/* + *@strAddr:192.168.145.135:20024/tcp + */ +int DataConnector::ParseAddr(const std::string strAddr, uint64_t *uipIP /* = NULL */, uint64_t *uipPort /* = NULL */) +{ + if(strAddr.empty()) + return -1; + if(!isdigit(strAddr[0])) + return -1; + const char *szAgentAddr = strAddr.c_str(); + const char *szPortProtocol = strchr(strAddr.c_str(), ':'); + if(NULL == szPortProtocol) + return -1; + //":"后面必然需要有端口，若是长度小于2，则数据有问题 + if(strlen((char *)szPortProtocol) < 2) + return -1; + if(!isdigit(szPortProtocol[1])) + return -1; + struct in_addr addr; + uint32_t uiLen = szPortProtocol - szAgentAddr; + char szIP[uiLen + 1]; + memcpy(szIP, szAgentAddr, uiLen); + szIP[uiLen] = 0; + if(1 != inet_pton(AF_INET, szIP, &addr)) + return -1; + *uipIP = 0; + *uipPort = 0; + *uipIP = (uint64_t)ntohl(addr.s_addr); + *uipPort = (uint64_t)atoi(szPortProtocol + 1); + if(0 == *uipIP || 0 == *uipPort) + return -1; + return 0; +} + +/* + *以下数组中的值对应的为kpi_sample_count数组的下标，即跟随在其后注释中的数值所在的位置 + */ +int16_t CTopPercentileSection::GetTPSection(uint64_t elapse) +{ + if(elapse < THOUSAND) + return GetLTThousand(elapse); + else if(THOUSAND <= elapse && elapse < TENTHOUSAND) + return GetGTEThousandLTTenThousand(elapse); + else if(TENTHOUSAND <= elapse && elapse < HUNDREDTHOUSAND) + return GetGTETenThousandLTHundredThousand(elapse); + else if(HUNDREDTHOUSAND <= elapse && elapse < MILLION) + return GetGTEHundredThousandLTMillion(elapse); + else + return GetGTEMillion(elapse); + + return -1; +} + +int16_t CTopPercentileSection::GetLTThousand(uint64_t elapse) +{ + if(elapse < THOUSAND) + { + static int16_t iSiteArr[] = + { + 0,/*200us下标*/ + 0,/*200us下标*/ + 1,/*500us下标*/ + 1,/*500us下标*/ + 1,/*500us下标*/ + 2,/*1000us下标*/ + 2,/*1000us下标*/ + 2,/*1000us下标*/ + 2,/*1000us下标*/ + 2/*1000us下标*/ + }; + int16_t iSite = elapse / HUNDRED; + if(iSite < 0 || iSite >= int16_t(sizeof(iSiteArr) / sizeof(iSiteArr[0]))) + return -1; + return iSiteArr[iSite]; + } + return -1; +} + +int16_t CTopPercentileSection::GetGTEThousandLTTenThousand(uint64_t elapse) +{ + if(THOUSAND <= elapse && elapse < TENTHOUSAND) + { + static int16_t iSiteArr[] = + { + 3,/*2000us下标*/ + 4,/*3000us下标*/ + 5,/*4000us下标*/ + 6,/*5000us下标*/ + 7,/*6000us下标*/ + 8,/*7000us下标*/ + 9,/*8000us下标*/ + 10,/*9000us下标*/ + 11/*10000us下标*/ + }; + int16_t iSite = elapse / THOUSAND - 1; + if(iSite < 0 || iSite >= int16_t(sizeof(iSiteArr) / sizeof(iSiteArr[0]))) + return -1; + return iSiteArr[iSite]; + } + return -1; +} + +int16_t CTopPercentileSection::GetGTETenThousandLTHundredThousand(uint64_t elapse) +{ + if(TENTHOUSAND <= elapse && elapse < HUNDREDTHOUSAND) + { + static int16_t iSiteArr[] = + { + 12,/*20000us下标*/ + 13,/*30000us下标*/ + 14,/*40000us下标*/ + 15,/*50000us下标*/ + 16,/*60000us下标*/ + 17,/*70000us下标*/ + 18,/*80000us下标*/ + 19,/*90000us下标*/ + 20/*100000us下标*/ + }; + int16_t iSite = elapse / TENTHOUSAND - 1; + if(iSite < 0 || iSite >= int16_t(sizeof(iSiteArr) / sizeof(iSiteArr[0]))) + return -1; + return iSiteArr[iSite]; + } + return -1; +} + +int16_t CTopPercentileSection::GetGTEHundredThousandLTMillion(uint64_t elapse) +{ + if(HUNDREDTHOUSAND <= elapse && elapse < MILLION) + { + static int16_t iSiteArr[] = + { + 21,/*200000us下标*/ + 22,/*300000us下标*/ + 23,/*400000us下标*/ + 24,/*500000us下标*/ + 25,/*600000us下标*/ + 26,/*700000us下标*/ + 27,/*800000us下标*/ + 28,/*900000us下标*/ + 29/*1000000us下标*/ + }; + int16_t iSite = elapse / HUNDREDTHOUSAND - 1; + if(iSite < 0 || iSite >= int16_t(sizeof(iSiteArr) / sizeof(iSiteArr[0]))) + return -1; + return iSiteArr[iSite]; + } + return -1; +} + +int16_t CTopPercentileSection::GetGTEMillion(uint64_t elapse) +{ + if(elapse >= MILLION) + { + static int16_t iSiteArr[] = + { + 30,/*2000000us下标*/ + 31,/*5000000us下标*/ + 31,/*5000000us下标*/ + 31,/*5000000us下标*/ + 32,/*10000000us下标*/ + 32,/*10000000us下标*/ + 32,/*10000000us下标*/ + 32,/*10000000us下标*/ + 32,/*10000000us下标*/ + 32/*10000000us下标*/ + }; + int16_t iSite = elapse / MILLION - 1; + int16_t iArrLen = int16_t(sizeof(iSiteArr) / sizeof(iSiteArr[0])); + iSite = (iSite >= iArrLen) ? (iArrLen - 1) : iSite; + if(iSite < 0) + return -1; + return iSiteArr[iSite]; + } + return -1; +} diff --git a/src/search_local/index_storage/api/c_api_cc/dtcwrap.cc b/src/search_local/index_storage/api/c_api_cc/dtcwrap.cc new file mode 100644 index 0000000..eb35f37 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcwrap.cc @@ -0,0 +1,1295 @@ +#include +#include +#include + +#include "dtcapi.h" +#include "dtcint.h" +#include +#include +#include "compiler.h" + + +using namespace DTC; + +#define CAST(type, var) type *var = (type *)addr +#define CAST2(type, var, src) type *var = (type *)src->addr +#define CAST2Z(type, var, src) type *var = (type *)((src)?(src)->addr:0) +#define CAST3(type, var, src) type *var = (type *)src.addr + +//WRAPPER for log +__EXPORT +void DTC::init_log (const char *app, const char *dir) +{ + ::_init_log_(app, dir); +} + +__EXPORT +void DTC::set_log_level(int n) +{ + ::_set_log_level_(n); +} + +__EXPORT +int DTC::set_key_value_max(unsigned int count) +{ + if(count >= 8 && count <= 10000) { + NCKeyValueList::KeyValueMax = count; + } else { + return -EINVAL; + } + return 0; +}; + +// WRAPPER for Server +__EXPORT +Server::Server(void) { + NCServer *srv = new NCServer; + srv->INC(); + addr = srv; +} + +__EXPORT +Server::Server(const Server &that) { + CAST3(NCServer, old, that); + NCServer *srv = new NCServer(*old); + srv->INC(); + addr = srv; +} + +__EXPORT +Server::~Server(void) { + CAST(NCServer, srv); + DEC_DELETE(srv); +} + +__EXPORT +int Server::SetAddress(const char *host, const char *port) { + CAST(NCServer, srv); + return srv->SetAddress(host, port); +} + +__EXPORT +int Server::SetTableName(const char *name) { + CAST(NCServer, srv); + return srv->SetTableName(name); +} + +__EXPORT +void Server::SetCompressLevel(int level) { + CAST(NCServer, srv); + srv->SetCompressLevel(level); +} + +__EXPORT +const char * Server::GetAddress(void) const { + CAST(NCServer, srv); + return srv ? srv->GetAddress() : NULL; +} + +__EXPORT +const char * Server::GetTableName(void) const { + CAST(NCServer, srv); + return srv ? srv->GetTableName() : NULL; +} + +__EXPORT +const char * Server::GetServerAddress(void) const { + CAST(NCServer, srv); + return srv ? srv->GetServerAddress() : NULL; +} + +__EXPORT +const char * Server::GetServerTableName(void) const { + CAST(NCServer, srv); + return srv ? srv->GetServerTableName() : NULL; +} +__EXPORT +void Server::CloneTabDef(const Server& source) +{ + CAST(NCServer, srv); + CAST3(NCServer, ssrv, source); + srv->CloneTabDef(*ssrv); +} + +__EXPORT +int Server::IntKey(void) { + CAST(NCServer, srv); + return srv->IntKey(); +} + +__EXPORT +int Server::StringKey(void) { + CAST(NCServer, srv); + return srv->StringKey(); +} + +__EXPORT +int Server::BinaryKey(void) { + CAST(NCServer, srv); + return srv->StringKey(); +} + +__EXPORT +int Server::AddKey(const char* name, int type) { + CAST(NCServer, srv); + return srv->AddKey(name, (uint8_t)type); +} + +__EXPORT +int Server::FieldType(const char* name) { + CAST(NCServer, srv); + return srv->FieldType(name); +} + +__EXPORT +const char * Server::ErrorMessage(void) const { + CAST(NCServer, srv); + return srv->ErrorMessage(); +} + +//fixed, sec ---> msec +__EXPORT +void Server::SetMTimeout(int n) { + CAST(NCServer, srv); + return srv->SetMTimeout(n); +} + +__EXPORT +void Server::SetTimeout(int n) { + CAST(NCServer, srv); + return srv->SetMTimeout(n*1000); +} + +__EXPORT +int Server::Connect(void) { + CAST(NCServer, srv); + return srv->Connect(); +} + +__EXPORT +void Server::Close(void) { + CAST(NCServer, srv); + return srv->Close(); +} + +__EXPORT +int Server::Ping(void) { + CAST(NCServer, srv); + return srv->Ping(); +} + +__EXPORT +void Server::AutoPing(void) { + CAST(NCServer, srv); + return srv->AutoPing(); +} + +__EXPORT +void Server::SetFD(int fd) { + CAST(NCServer, srv); + srv->SetFD(fd); +} + +__EXPORT +void Server::SetAutoUpdateTab(bool autoUpdate) +{ + CAST(NCServer, srv); + srv->SetAutoUpdateTab(autoUpdate); +} + +__EXPORT +void Server::SetAutoReconnect(int autoReconnect) +{ + CAST(NCServer, srv); + srv->SetAutoReconnect(autoReconnect); +} + +__EXPORT +void Server::SetAccessKey(const char *token) +{ + CAST(NCServer, srv); + srv->SetAccessKey(token); +} + +//QOS ADD BY NEOLV +__EXPORT +void Server::IncErrCount() +{ + CAST(NCServer, srv); + srv->IncErrCount(); +} +__EXPORT +uint64_t Server::GetErrCount() +{ + CAST(NCServer, srv); + return srv->GetErrCount(); +} +__EXPORT +void Server::ClearErrCount() +{ + CAST(NCServer, srv); + srv->ClearErrCount(); +} + +__EXPORT +void Server::IncRemoveCount() +{ + CAST(NCServer, srv); + srv->IncRemoveCount(); +} +__EXPORT +int Server::GetRemoveCount() +{ + CAST(NCServer, srv); + return srv->GetRemoveCount(); +} +__EXPORT +void Server::ClearRemoveCount() +{ + CAST(NCServer, srv); + srv->ClearRemoveCount(); +} +__EXPORT +void Server::IncTotalReq() +{ + CAST(NCServer, srv); + srv->IncTotalReq(); +} +__EXPORT +uint64_t Server::GetTotalReq() +{ + CAST(NCServer, srv); + return srv->GetTotalReq(); +} +__EXPORT +void Server::ClearTotalReq() +{ + CAST(NCServer, srv); + srv->ClearTotalReq(); +} + +__EXPORT +void Server::AddTotalElaps(uint64_t iElaps) +{ + CAST(NCServer, srv); + srv->AddTotalElaps(iElaps); +} +__EXPORT +uint64_t Server::GetTotalElaps() +{ + CAST(NCServer, srv); + return srv->GetTotalElaps(); +} +__EXPORT +void Server::ClearTotalElaps() +{ + CAST(NCServer, srv); + srv->ClearTotalElaps(); +} + + +// WRAPER for Request + +__EXPORT +Request::Request(Server *srv0, int op) { + CAST2Z(NCServer, srv, srv0); + NCRequest *req = new NCRequest(srv, op); + addr = (void *)req; +} + +Request::Request() +{ + NCRequest *req = new NCRequest(NULL, 0); + addr = (void *)req; +} + +__EXPORT +Request::~Request(void) { + CAST(NCRequest, req); + delete req; +} + +__EXPORT +int Request::AttachServer(Server *srv0) { + CAST2Z(NCServer, srv, srv0); + CAST(NCRequest, r); + return r->AttachServer(srv); +} + +__EXPORT +void Request::Reset(void) { + CAST(NCRequest, old); + if(old) { + int op = old->cmd; + NCServer *srv = old->server; + srv->INC(); + old->~NCRequest(); + memset(old, 0, sizeof(NCRequest)); + new(addr) NCRequest(srv, op); + srv->DEC(); + } +} + +__EXPORT +void Request::Reset(int op) +{ + CAST(NCRequest, old); + + if (old) + { + NCServer *srv = old->server; + srv->INC(); + old->~NCRequest(); + memset(old, 0, sizeof(NCRequest)); + new(addr) NCRequest(srv, op); + srv->DEC(); + } +} + +__EXPORT +void Request::SetAdminCode(int code) { + CAST(NCRequest, r); + r->SetAdminCode(code); +} + +__EXPORT +void Request::SetHotBackupID(long long v) { + CAST(NCRequest, r); + r->SetHotBackupID(v); +} + +__EXPORT +void Request::SetMasterHBTimestamp(long long t) { + CAST(NCRequest, r); + r->SetMasterHBTimestamp(t); +} + +__EXPORT +void Request::SetSlaveHBTimestamp(long long t) { + CAST(NCRequest, r); + r->SetSlaveHBTimestamp(t); +} + +__EXPORT +void Request::NoCache(void) { + CAST(NCRequest, r); return r->EnableNoCache(); +} + +__EXPORT +void Request::NoNextServer(void) { + CAST(NCRequest, r); return r->EnableNoNextServer(); +} + +__EXPORT +int Request::Need(const char *name) { + CAST(NCRequest, r); return r->Need(name, 0); +} + +__EXPORT +int Request::Need(const char *name, int vid) { + CAST(NCRequest, r); return r->Need(name, vid); +} + +__EXPORT +int Request::FieldType(const char *name) { + CAST(NCRequest, r); return r->FieldType(name); +} + +__EXPORT +void Request::Limit(unsigned int st, unsigned int cnt) { + CAST(NCRequest, r); return r->Limit(st, cnt); +} + +__EXPORT +void Request::UnsetKey(void) { + CAST(NCRequest, r); + r->UnsetKey(); + r->UnsetKeyValue(); +} + +__EXPORT +int Request::SetKey(long long key) { + CAST(NCRequest, r); return r->SetKey((int64_t)key); +} + +__EXPORT +int Request::SetKey(const char *key) { + CAST(NCRequest, r); return r->SetKey(key, strlen(key)); +} + +__EXPORT +int Request::SetKey(const char *key, int len) { + CAST(NCRequest, r); return r->SetKey(key, len); +} + +__EXPORT +int Request::AddKeyValue(const char* name, long long v) { + CAST(NCRequest, r); + DTCValue key(v); + return r->AddKeyValue(name, key, KeyTypeInt); +} + +__EXPORT +int Request::AddKeyValue(const char* name, const char *v) { + CAST(NCRequest, r); + DTCValue key(v); + return r->AddKeyValue(name, key, KeyTypeString); +} + +__EXPORT +int Request::AddKeyValue(const char* name, const char *v, int len) { + CAST(NCRequest, r); + DTCValue key(v, len); + return r->AddKeyValue(name, key, KeyTypeString); +} + +__EXPORT +int Request::SetCacheID(long long id){ + CAST(NCRequest, r); return r->SetCacheID(id); +} + +__EXPORT +Result *Request::Execute(void) { + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->Execute(); + return s; +} + +__EXPORT +Result *Request::Execute(long long k) { + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->Execute((int64_t)k); + return s; +} + +__EXPORT +Result *Request::Execute(const char *k) { + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->Execute(k, k?strlen(k):0); + return s; +} + +__EXPORT +Result *Request::Execute(const char *k, int l) { + Result *s = new Result(); + CAST(NCRequest, r); + s->addr = r->Execute(k, l); + return s; +} + +__EXPORT +int Request::Execute(Result &s) { + CAST(NCRequest, r); + s.Reset(); + s.addr = r->Execute(); + return s.ResultCode(); +} + +__EXPORT +int Request::Execute(Result &s, long long k) { + CAST(NCRequest, r); + s.Reset(); + s.addr = r->Execute((int64_t)k); + return s.ResultCode(); +} + +__EXPORT +int Request::Execute(Result &s, const char *k) { + CAST(NCRequest, r); + s.Reset(); + s.addr = r->Execute(k, k?strlen(k):0); + return s.ResultCode(); +} + +__EXPORT +int Request::Execute(Result &s, const char *k, int l) { + CAST(NCRequest, r); + s.Reset(); + s.addr = r->Execute(k, l); + return s.ResultCode(); +} +//增加错误信息，方便以后问题定位. add by foryzhou +__EXPORT +const char * Request::ErrorMessage(void) const { + CAST(NCRequest, r); + if(r==NULL) return NULL; + return r->ErrorMessage(); +} + +#define _DECLACT_(f0,t0,f1,f2,t1) \ + __EXPORT \ + int Request::f0(const char *n, t0 v) { \ + CAST(NCRequest, r); \ + return r->f1(n, DField::f2, DField::t1, DTCValue::Make(v)); \ + } +#define _DECLACT2_(f0,t0,f1,f2,t1) \ + __EXPORT \ + int Request::f0(const char *n, t0 v, int l) { \ + CAST(NCRequest, r); \ + return r->f1(n, DField::f2, DField::t1, DTCValue::Make(v,l)); \ + } + + + +_DECLACT_(EQ, long long, AddCondition, EQ, Signed); +_DECLACT_(NE, long long, AddCondition, NE, Signed); +_DECLACT_(GT, long long, AddCondition, GT, Signed); +_DECLACT_(GE, long long, AddCondition, GE, Signed); +_DECLACT_(LT, long long, AddCondition, LT, Signed); +_DECLACT_(LE, long long, AddCondition, LE, Signed); +_DECLACT_(EQ, const char *, AddCondition, EQ, String); +_DECLACT_(NE, const char *, AddCondition, NE, String); +_DECLACT2_(EQ, const char *, AddCondition, EQ, String); +_DECLACT2_(NE, const char *, AddCondition, NE, String); +_DECLACT_(Set, long long, AddOperation, Set, Signed); +_DECLACT_(OR, long long, AddOperation, OR, Signed); +_DECLACT_(Add, long long, AddOperation, Add, Signed); + +__EXPORT +int Request::Sub(const char *n, long long v) { + CAST(NCRequest, r); + return r->AddOperation(n, DField::Add, DField::Signed, DTCValue::Make(-(int64_t)v)); +} + +_DECLACT_(Set, double, AddOperation, Set, Float); +_DECLACT_(Add, double, AddOperation, Add, Float); + +__EXPORT +int Request::Sub(const char *n, double v) { + CAST(NCRequest, r); + return r->AddOperation(n, DField::Add, DField::Signed, DTCValue::Make(-v)); +} + +_DECLACT_(Set, const char *, AddOperation, Set, String); +_DECLACT2_(Set, const char *, AddOperation, Set, String); +#undef _DECLACT_ +#undef _DECLACT2_ + +__EXPORT +int Request::CompressSet(const char *n,const char * v,int len) +{ + CAST(NCRequest, r); + return r->CompressSet(n,v,len); +} + +__EXPORT +int Request::CompressSetForce(const char *n,const char * v,int len) +{ + CAST(NCRequest, r); + return r->CompressSetForce(n,v,len); +} + +__EXPORT +int Request::SetMultiBits(const char *n, int o, int s, unsigned int v) { + if(s <=0 || o >= (1 << 24) || o < 0) + return 0; + if(s >= 32) + s = 32; + + CAST(NCRequest, r); + // 1 byte 3 byte 4 byte + // s o v + uint64_t t = ((uint64_t)s << 56)+ ((uint64_t)o << 32) + v; + return r->AddOperation(n, DField::SetBits, DField::Unsigned, DTCValue::Make(t)); +} + +__EXPORT +int Request::SetExpireTime(const char* key, int time){ + CAST(NCRequest, r); + return r->SetExpireTime(key, time); +} + +__EXPORT +int Request::GetExpireTime(const char* key){ + CAST(NCRequest, r); + return r->GetExpireTime(key); +} + + +// WRAPER for Result +__EXPORT +Result::Result(void) { + addr = NULL; +} + +__EXPORT +Result::~Result(void) { + Reset(); +} + +__EXPORT +void Result::SetError(int err, const char *via, const char *msg) { + Reset(); + addr = new NCResult(err, via, msg); +} + +__EXPORT +void Result::Reset(void) { + if(addr != NULL) { + CAST(NCResult, task); + CAST(NCResultInternal, itask); + if(task->Role()==TaskRoleClient) + delete task; + else + delete itask; + } + addr = NULL; +} + +__EXPORT +int Result::FetchRow(void) { + CAST(NCResult, r); + if(r==NULL) return -EC_NO_MORE_DATA; + if(r->ResultCode()<0) return r->ResultCode(); + if(r->result==NULL) return -EC_NO_MORE_DATA; + return r->result->DecodeRow(); +} + +__EXPORT +int Result::Rewind(void) { + CAST(NCResult, r); + if(r==NULL) return -EC_NO_MORE_DATA; + if(r->ResultCode()<0) return r->ResultCode(); + if(r->result==NULL) return -EC_NO_MORE_DATA; + return r->result->Rewind(); +} + +__EXPORT +int Result::ResultCode(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->ResultCode(); +} + +__EXPORT +const char * Result::ErrorMessage(void) const { + CAST(NCResult, r); + if(r==NULL) return NULL; + int code = r->ResultCode(); + const char *msg = r->resultInfo.ErrorMessage(); + if(msg==NULL && code < 0) { + char a[1]; + msg = strerror_r(-code, a, 1); + if(msg==a) msg = NULL; + } + return msg; +} + +__EXPORT +const char * Result::ErrorFrom(void) const { + CAST(NCResult, r); + if(r==NULL) return NULL; + return r->resultInfo.ErrorFrom(); +} + +__EXPORT +long long Result::HotBackupID() const { + CAST(NCResult, r); + return r->versionInfo.HotBackupID(); +} + +__EXPORT +long long Result::MasterHBTimestamp() const { + CAST(NCResult, r); + return r->versionInfo.MasterHBTimestamp(); +} + +__EXPORT +long long Result::SlaveHBTimestamp() const { + CAST(NCResult, r); + return r->versionInfo.SlaveHBTimestamp(); +} + +__EXPORT +char* Result::ServerInfo() const { + CAST(NCResult, r); + return r->resultInfo.ServerInfo(); +} + +__EXPORT +long long Result::BinlogID() const { + CServerInfo *p = (CServerInfo *)ServerInfo(); + return p ? p->binlog_id : 0; +} + +__EXPORT +long long Result::BinlogOffset() const { + CServerInfo *p = (CServerInfo *)ServerInfo(); + return p ? p->binlog_off : 0; +} +__EXPORT +long long Result::MemSize() const +{ + CServerInfo *p = (CServerInfo *)ServerInfo(); + return p ? p->memsize : 0; +} +__EXPORT +long long Result::DataSize() const +{ + CServerInfo *p = (CServerInfo *)ServerInfo(); + return p ? p->datasize : 0; +} + +__EXPORT +int Result::NumRows(void) const { + CAST(NCResult, r); + if(r==NULL || r->result==NULL) return 0; + return r->result->TotalRows(); +} + +__EXPORT +int Result::TotalRows(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->resultInfo.TotalRows(); +} + +__EXPORT +long long Result::InsertID(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->resultInfo.InsertID(); +} + +__EXPORT +int Result::NumFields(void) const { + CAST(NCResult, r); + if(r==NULL || r->result==NULL) return 0; + return r->result->NumFields(); +} + +__EXPORT +const char* Result::FieldName(int n) const { + CAST(NCResult, r); + if(r==NULL || r->result==NULL) return NULL; + if(n<0 || n>=r->result->NumFields()) return NULL; + return r->FieldName(r->result->FieldId(n)); +} + +__EXPORT +int Result::FieldPresent(const char* name) const { + CAST(NCResult, r); + if(r==NULL || r->result==NULL) return 0; + int id = r->FieldId(name); + if(id < 0) return 0; + return r->result->FieldPresent(id); +} + +__EXPORT +int Result::FieldType(int n) const { + CAST(NCResult, r); + if(r==NULL || r->result==NULL) return FieldTypeNone; + if(n<0 || n>=r->result->NumFields()) return FieldTypeNone; + return r->FieldType(r->result->FieldId(n)); +} + +__EXPORT +int Result::AffectedRows(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->resultInfo.AffectedRows(); +} + +__EXPORT +long long Result::IntKey(void) const { + CAST(NCResult, r); + const DTCValue *v = NULL; + if(r && r->FlagMultiKeyResult() && r->result){ + v = r->result->FieldValue(0); + } + else if(r && r->ResultKey()){ + v = r->ResultKey(); + } + if(v != NULL){ + switch(r->FieldType(0)) { + case DField::Signed: + case DField::Unsigned: + return v->s64; + } + } +/* + if(r && r->ResultKey()) { + switch(r->FieldType(0)) { + case DField::Signed: + case DField::Unsigned: + return r->ResultKey()->s64; + } + } +*/ + return 0; +} + +__EXPORT +const char *Result::StringKey(void) const { + CAST(NCResult, r); + const DTCValue *v = NULL; + if(r && r->FlagMultiKeyResult() && r->result){ + v = r->result->FieldValue(0); + } + else if(r && r->ResultKey()){ + v = r->ResultKey(); + } + if(v != NULL){ + switch(r->FieldType(0)) { + case DField::Binary: + case DField::String: + return v->bin.ptr; + } + } + + return NULL; +} + +__EXPORT +const char *Result::StringKey(int *lp) const { + CAST(NCResult, r); + const DTCValue *v = NULL; + if(r && r->FlagMultiKeyResult() && r->result){ + v = r->result->FieldValue(0); + } + else if(r && r->ResultKey()){ + v = r->ResultKey(); + } + if(v != NULL){ + switch(r->FieldType(0)) { + case DField::Binary: + case DField::String: + if(lp) *lp = v->bin.len; + return v->bin.ptr; + } + } + + return NULL; +} + +const char *Result::StringKey(int &l) const { + CAST(NCResult, r); + const DTCValue *v = NULL; + if(r && r->FlagMultiKeyResult() && r->result){ + v = r->result->FieldValue(0); + } + else if(r && r->ResultKey()){ + v = r->ResultKey(); + } + if(v != NULL){ + switch(r->FieldType(0)) { + case DField::Binary: + case DField::String: + l = v->bin.len; + return v->bin.ptr; + } + } + + return NULL; +} + +__EXPORT +const char *Result::BinaryKey(void) const { + return StringKey(); +} + +__EXPORT +const char *Result::BinaryKey(int *lp) const { + return StringKey(lp); +} + +__EXPORT +const char *Result::BinaryKey(int &l) const { + return StringKey(l); +} + +static inline int64_t GetIntValue(NCResult *r, int id) { + if(id >= 0) { + const DTCValue *v; + if(id==0 && !(r->result->FieldPresent(0))) + v = r->ResultKey(); + else + v = r->result->FieldValue(id); + + if(v) { + switch(r->FieldType(id)) { + case DField::Signed: + case DField::Unsigned: + return v->s64; + case DField::Float: + return llround(v->flt); + //return (int64_t)(v->flt); + case DField::String: + if(v->str.ptr) + return atoll(v->str.ptr); + } + } + } + return 0; +} + +static inline double GetFloatValue(NCResult *r, int id) { + if(id >= 0) { + const DTCValue *v; + if(id==0 && !(r->result->FieldPresent(0))) + v = r->ResultKey(); + else + v = r->result->FieldValue(id); + + if(v) { + switch(r->FieldType(id)) { + case DField::Signed: + return (double)v->s64; + case DField::Unsigned: + return (double)v->u64; + case DField::Float: + return v->flt; + case DField::String: + if(v->str.ptr) + return atof(v->str.ptr); + } + } + } + return NAN; +} + +static inline const char *GetStringValue(NCResult *r, int id, int *lenp) { + if(id >= 0) { + const DTCValue *v; + if(id==0 && !(r->result->FieldPresent(0))) + v = r->ResultKey(); + else + v = r->result->FieldValue(id); + + if(v) { + switch(r->FieldType(id)) { + case DField::String: + if(lenp) *lenp = v->bin.len; + return v->str.ptr; + } + } + } + return NULL; +} + +static inline const char *GetBinaryValue(NCResult *r, int id, int *lenp) { + if(id >= 0) { + const DTCValue *v; + if(id==0 && !(r->result->FieldPresent(0))) + v = r->ResultKey(); + else + v = r->result->FieldValue(id); + + if(v) { + switch(r->FieldType(id)) { + case DField::String: + case DField::Binary: + if(lenp) *lenp = v->bin.len; + return v->str.ptr; + } + } + } + return NULL; +} + +__EXPORT +long long Result::IntValue(const char *name) const { + CAST(NCResult, r); + if(r && r->result) + return GetIntValue(r, r->FieldId(name)); + return 0; +} + +__EXPORT +long long Result::IntValue(int id) const { + CAST(NCResult, r); + if(r && r->result) + return GetIntValue(r, r->FieldIdVirtual(id)); + return 0; +} + +__EXPORT +double Result::FloatValue(const char *name) const { + CAST(NCResult, r); + if(r && r->result) + return GetFloatValue(r, r->FieldId(name)); + return NAN; +} + +__EXPORT +double Result::FloatValue(int id) const { + CAST(NCResult, r); + if(r && r->result) + return GetFloatValue(r, r->FieldIdVirtual(id)); + return NAN; +} + +__EXPORT +const char *Result::StringValue(const char *name, int *lenp) const { + if(lenp) *lenp = 0; + CAST(NCResult, r); + if(r && r->result) + return GetStringValue(r, r->FieldId(name), lenp); + return NULL; +} + +__EXPORT +const char *Result::StringValue(int id, int *lenp) const { + if(lenp) *lenp = 0; + CAST(NCResult, r); + if(r && r->result) + return GetStringValue(r, r->FieldIdVirtual(id), lenp); + return NULL; +} + +__EXPORT +const char *Result::StringValue(const char *name, int &len) const { + return StringValue(name, &len); +} + +__EXPORT +const char *Result::StringValue(int id, int &len) const { + return StringValue(id, &len); +} + +__EXPORT +const char *Result::StringValue(const char *name) const { + return StringValue(name, NULL); +} + +__EXPORT +const char *Result::StringValue(int id) const { + return StringValue(id, NULL); +} + +__EXPORT +const char *Result::BinaryValue(const char *name, int *lenp) const { + if(lenp) *lenp = 0; + CAST(NCResult, r); + if(r && r->result) + return GetBinaryValue(r, r->FieldId(name), lenp); + return NULL; +} + +//所以UnCompressBinaryValue接口的调用均会复制一份buffer，用户得到数据后需要手动释放 +__EXPORT +int Result::UnCompressBinaryValue(const char *name,char **buf,int *lenp){ + if(buf==NULL||lenp==NULL) + return -EC_UNCOMPRESS_ERROR; + + int iret = 0; + *lenp = 0; + *buf = NULL; + const char * buf_dup = NULL; + uint64_t compressflag = 0; + + CAST(NCResult, r); + if(r && r->result) + { + iret = r->initCompress(); + if (iret) return iret; + if (name == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "field name is null"); + return -EC_UNCOMPRESS_ERROR; + } + int fieldId = r->FieldId(name); + if (fieldId >63)//fieldid must less than 64,because compressflag is 8 bytes uint + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "fieldid must less than 64,because compressflag is 8 bytes uint"); + return -EC_UNCOMPRESS_ERROR; + } + buf_dup = GetBinaryValue(r, fieldId, lenp); + if (buf_dup==NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "BinaryValue of this field is NULL,please check fieldname and fieldtype is binary or not"); + return -EC_UNCOMPRESS_ERROR; + } + + if (r->CompressId() < 0)//没启用压缩 + { + *buf = (char *)MALLOC(*lenp); + if (*buf==NULL) + return -ENOMEM; + memcpy(*buf,buf_dup,*lenp); + return 0; + } + compressflag = GetIntValue(r,r->CompressId()); + //启用了压缩，但是该字段没有被压缩 + if (!(compressflag&(0x1<gzip->UnCompress(buf,lenp,buf_dup,*lenp); + if (iret == -111111) + return -EC_UNCOMPRESS_ERROR; + return iret; + } + return -EC_UNCOMPRESS_ERROR; +} + +//所以UnCompressBinaryValue接口的调用均会复制一份buffer，用户得到数据后需要手动释放 +//忽略compressflag +__EXPORT +int Result::UnCompressBinaryValueForce(const char *name,char **buf,int *lenp){ + if(buf==NULL||lenp==NULL) + return -EC_UNCOMPRESS_ERROR; + + int iret = 0; + *lenp = 0; + *buf = NULL; + const char * buf_dup = NULL; + + CAST(NCResult, r); + if(r && r->result) + { + if (name == NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "field name is null"); + return -EC_UNCOMPRESS_ERROR; + } + iret = r->initCompress(); + if (iret) return iret; + int fieldId = r->FieldId(name); + buf_dup = GetBinaryValue(r, fieldId, lenp); + if (buf_dup==NULL) + { + snprintf(r->gzip->_errmsg, sizeof(r->gzip->_errmsg), "BinaryValue of this field is NULL,please check fieldname and fieldtype is binary or not"); + return -EC_UNCOMPRESS_ERROR; + } + //需要解压 + iret = r->gzip->UnCompress(buf,lenp,buf_dup,*lenp); + if (iret == -111111) + return -EC_UNCOMPRESS_ERROR; + return iret; + } + return -EC_UNCOMPRESS_ERROR; +} + +__EXPORT +const char *Result::UnCompressErrorMessage() const +{ + CAST(NCResult, r); + if(r && r->gzip) + return r->gzip->ErrorMessage(); + return NULL; +} + +__EXPORT +const char *Result::BinaryValue(int id, int *lenp) const { + if(lenp) *lenp = 0; + CAST(NCResult, r); + if(r && r->result) + return GetBinaryValue(r, r->FieldIdVirtual(id), lenp); + return NULL; +} + +__EXPORT +const char *Result::BinaryValue(const char *name, int &len) const { + return BinaryValue(name, &len); +} + +__EXPORT +const char *Result::BinaryValue(int id, int &len) const { + return BinaryValue(id, &len); +} + +__EXPORT +const char *Result::BinaryValue(const char *name) const { + return BinaryValue(name, NULL); +} + +__EXPORT +const char *Result::BinaryValue(int id) const { + return BinaryValue(id, NULL); +} + +__EXPORT +long long Result::Tag(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->GetApiTag(); +}; + +__EXPORT +void *Result::TagPtr(void) const { + CAST(NCResult, r); + if(r==NULL) return NULL; + return (void *)(long)r->GetApiTag(); +}; + +__EXPORT +long long Result::Magic(void) const { + CAST(NCResult, r); + if(r==NULL) return 0; + return r->versionInfo.SerialNr(); +}; + +__EXPORT +long long Result::ServerTimestamp(void) const +{ + CAST(NCResult, r); + + if (r == NULL) + { + return 0; + } + + return r->resultInfo.Timestamp(); +}; + +__EXPORT +int Server::DecodePacket(Result &s, const char *buf, int len) +{ + CAST(NCServer, srv); + s.Reset(); + s.addr = srv->DecodeBuffer(buf, len); + return s.ResultCode(); +} + +__EXPORT +int Server::CheckPacketSize(const char *buf, int len) +{ + return NCServer::CheckPacketSize(buf, len); +} + +__EXPORT +int Request::EncodePacket(char *&ptr, int &len, long long &m) { + CAST(NCRequest, r); + int64_t mm; + int ret = r->EncodeBuffer(ptr, len, mm); + m = mm; + return ret; +} + +__EXPORT +int Request::EncodePacket(char *&ptr, int &len, long long &m, long long k) { + CAST(NCRequest, r); + int64_t mm; + int ret = r->EncodeBuffer(ptr, len, mm, (int64_t)k); + m = mm; + return ret; +} + +__EXPORT +int Request::EncodePacket(char *&ptr, int &len, long long &m, const char *k) { + CAST(NCRequest, r); + int64_t mm; + int ret = r->EncodeBuffer(ptr, len, mm, k, k?strlen(k):0); + m = mm; + return ret; +} + +__EXPORT +int Request::EncodePacket(char *&ptr, int &len, long long &m, const char *k, int l) { + CAST(NCRequest, r); + int64_t mm; + int ret = r->EncodeBuffer(ptr, len, mm, k, l); + m = mm; + return ret; +} + diff --git a/src/search_local/index_storage/api/c_api_cc/dtcwrapp.cc b/src/search_local/index_storage/api/c_api_cc/dtcwrapp.cc new file mode 100644 index 0000000..614ee7b --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/dtcwrapp.cc @@ -0,0 +1,231 @@ +#include +#include +#include + +#include "dtcapi.h" +#include "dtcpool.h" + +using namespace DTC; + +#define CAST(type, var) type *var = (type *)addr +#define CAST2(type, var, src) type *var = (type *)src->addr +#define CAST3(type, var, src) type *var = (type *)src.addr + +// WRAPPER for Server +__EXPORT +ServerPool::ServerPool(int ms, int mr) { + NCPool *pl = new NCPool(ms, mr); + addr = pl; +} + +__EXPORT +ServerPool::~ServerPool(void) { + CAST(NCPool, pl); + delete pl; +} + +__EXPORT +int ServerPool::AddServer(Server *srv, int mreq, int mconn) +{ + CAST(NCPool, pl); + CAST2(NCServer, s, srv); + + return pl->AddServer(s, mreq, mconn); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, long long tag) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if(r->server == NULL) + return -EC_NOT_INITIALIZED; + + return pl->AddRequest(r, tag, NULL); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, void *tag) +{ + return AddRequest(req, (long long)(long)tag); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, long long tag, long long k) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if(r->server == NULL) + return -EC_NOT_INITIALIZED; + if(r->server->keytype != DField::Signed) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k); + return pl->AddRequest(r, tag, &v); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, void *tag, long long k) +{ + return AddRequest(req, (long long)(long)tag, k); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, long long tag, const char *k) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if(r->server == NULL) + return -EC_NOT_INITIALIZED; + if(r->server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k); + return pl->AddRequest(r, tag, &v); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, void *tag, const char *k) +{ + return AddRequest(req, (long long)(long)tag, k); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, long long tag, const char *k, int l) +{ + CAST(NCPool, pl); + CAST2(NCRequest, r, req); + + if(r->server == NULL) + return -EC_NOT_INITIALIZED; + if(r->server->keytype != DField::String) + return -EC_BAD_KEY_TYPE; + + DTCValue v(k, l); + return pl->AddRequest(r, tag, &v); +} + +__EXPORT +int ServerPool::AddRequest(Request *req, void *tag, const char *k, int l) +{ + return AddRequest(req, (long long)(long)tag, k, l); +} + +__EXPORT +int ServerPool::Execute(int msec) +{ + CAST(NCPool, pl); + return pl->Execute(msec); +} + +__EXPORT +int ServerPool::ExecuteAll(int msec) +{ + CAST(NCPool, pl); + return pl->ExecuteAll(msec); +} + +__EXPORT +int ServerPool::CancelRequest(int id) +{ + CAST(NCPool, pl); + return pl->CancelRequest(id); +} + +__EXPORT +int ServerPool::CancelAllRequest(int type) +{ + CAST(NCPool, pl); + return pl->CancelAllRequest(type); +} + +__EXPORT +int ServerPool::AbortRequest(int id) +{ + CAST(NCPool, pl); + return pl->AbortRequest(id); +} + +__EXPORT +int ServerPool::AbortAllRequest(int type) +{ + CAST(NCPool, pl); + return pl->AbortAllRequest(type); +} + +__EXPORT +Result * ServerPool::GetResult(void) +{ + return GetResult(0); +} + +__EXPORT +Result * ServerPool::GetResult(int id) +{ + CAST(NCPool, pl); + NCResult *a = pl->GetResult(id); + if( (long)a < 4096 && (long)a > -4095) + return NULL; + Result *s = new Result(); + s->addr = (void *)a; + return s; +} + +__EXPORT +int ServerPool::GetResult(Result &s) +{ + return GetResult(s, 0); +} + +__EXPORT +int ServerPool::GetResult(Result &s, int id) +{ + CAST(NCPool, pl); + s.Reset(); + NCResult *a = pl->GetResult(id); + long iv = (long)a; + if( iv < 4096 && iv > -4095) + { + s.addr = (void *)(new NCResult(iv, "API::executing", iv>0 ? "Request not completed" : id ? "Invalid request" : "No more result")); + return iv; + } + s.addr = (void *)a; + return 0; +} + +__EXPORT +int ServerPool::ServerCount(void) const +{ + CAST(NCPool, pl); + return pl->ServerCount(); +} + +__EXPORT +int ServerPool::RequestCount(int type) const +{ + CAST(NCPool, pl); + if(type == DONE) + return pl->DoneRequestCount(); + if(type == (WAIT|SEND|RECV|DONE)) + return pl->RequestCount(); + return pl->CountRequestState(type); +} + +__EXPORT +int ServerPool::RequestState(int reqId) const +{ + CAST(NCPool, pl); + return pl->RequestState(reqId); +} + +__EXPORT +int ServerPool::GetEpollFD(int size) +{ + CAST(NCPool, pl); + return pl->GetEpollFD(size); +} + +const int ALL_STATE = WAIT|SEND|RECV|DONE; diff --git a/src/search_local/index_storage/api/c_api_cc/example/deletetest.cpp b/src/search_local/index_storage/api/c_api_cc/example/deletetest.cpp new file mode 100644 index 0000000..cf99675 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/example/deletetest.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + DTC::DeleteRequest deleteReq(&stServer); + uid = atoi(argv[1]); + deleteReq.SetKey(uid); + + DTC::Result stResult; + retCode = deleteReq.Execute(stResult); + printf("retCode:%d\n", retCode); + if(retCode == 0) + { + printf("delete success!\n"); + } + + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/example/inserttest.cpp b/src/search_local/index_storage/api/c_api_cc/example/inserttest.cpp new file mode 100644 index 0000000..9c9b740 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/example/inserttest.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + unsigned int age; + std::string name; + std::string city; + std::string descr; + + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + DTC::InsertRequest insertReq(&stServer); + //retCode = insertReq.SetKey(key); + + uid = atoi(argv[1]); + name = std::string(argv[2]); + city = std::string(argv[3]); + descr = std::string(argv[4]); + age = atoi(argv[5]); + + insertReq.SetKey(uid); + //insertReq.Set("key", 100003); + insertReq.Set("uid", uid); + insertReq.Set("name", name.c_str()); + insertReq.Set("city", city.c_str()); + insertReq.Set("descr", descr.c_str()); + insertReq.Set("age", age); + + DTC::Result stResult; + retCode = insertReq.Execute(stResult); + printf("retCode:%d\n", retCode); + if(retCode == 0) + { + DTC::GetRequest getReq(&stServer); + getReq.SetKey(uid); + if(retCode == 0) + retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(retCode == 0) + retCode = getReq.Need("name"); + if(retCode == 0) + retCode = getReq.Need("city"); + if(retCode == 0) + retCode = getReq.Need("descr"); + if(retCode == 0) + retCode = getReq.Need("age"); + if(retCode != 0) + { + printf("get-req set key or need error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + retCode = getReq.Execute(stResult); + retCode = stResult.FetchRow();//开始获取数据 + printf("uid:%d\n", stResult.IntValue("uid")); + printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据 + printf("city: %s\n", stResult.StringValue("city")); + printf("descr: %s\n", stResult.BinaryValue("descr")); + printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据 + } + + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/example/selecttest.cpp b/src/search_local/index_storage/api/c_api_cc/example/selecttest.cpp new file mode 100644 index 0000000..fa4d057 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/example/selecttest.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int iRet; + unsigned int uid; + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与在dtc.jd.com网站上申请dtc时指定的表名即tablename应该一样 + stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + //下面三行代码是Get数据 + unsigned int age; + std::string name; + std::string city; + std::string descr; + + DTC::GetRequest stGetReq(&stServer); + uid = atoi(argv[1]); + iRet = stGetReq.SetKey(uid); + + if(iRet == 0) + iRet = stGetReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(iRet == 0) + iRet = stGetReq.Need("name"); + if(iRet == 0) + iRet = stGetReq.Need("city"); + if(iRet == 0) + iRet = stGetReq.Need("descr"); + if(iRet == 0) + iRet = stGetReq.Need("age"); + if(iRet == 0) + iRet = stGetReq.GT("age", 24); + //iRet = stGetReq.OR("age", 27); + + if(iRet != 0) + { + printf("get-req set key or need error: %d", iRet); + fflush(stdout); + return(-1); + } + + // execute & get result + DTC::Result stResult; + iRet = stGetReq.Execute(stResult); + if(iRet != 0) + //如果出错,则输出错误码、错误阶段，错误信息，stResult.ErrorFrom(), stResult.ErrorMessage() 这两个错误信息很重要，一定要打印出来，方便定位问题 + { + printf ("uin[%u] dtc execute get error: %d, error_from:%s, msg:%s\n", + uid,//出错的key是多少 + iRet,//错误码为多少 + stResult.ErrorFrom(),//返回错误阶段 + stResult.ErrorMessage()//返回错误信息 + ); + fflush(stdout); + return(-2); + } + + if(stResult.NumRows() <= 0) + { + // 数据不存在 + printf("uin[%u] data not exist.\n", uid); + return(0); + } + + printf("nubrow:%d\n", stResult.NumRows()); + for(int i=0;i<=stResult.NumRows();++i) + { + iRet = stResult.FetchRow();//开始获取数据 + if(iRet < 0) + { + printf ("uid[%lu] dtc fetch row error: %d\n", uid, iRet); + fflush(stdout); + return(-3); + } + //如果一切正确，则可以输出数据了 + printf("uid: %d\n", stResult.IntValue("uid"));//输出int类型的数据 + printf("name: %s\n", stResult.StringValue("name"));//输出string类型的数据 + printf("city: %s\n", stResult.StringValue("city"));//输出string类型的数据 + printf("descr: %s\n", stResult.BinaryValue("descr"));//输出binary类型的数据 + printf("age:%d\n",stResult.IntValue("age")); + } + return(0); +} diff --git a/src/search_local/index_storage/api/c_api_cc/example/ttcapi.h b/src/search_local/index_storage/api/c_api_cc/example/ttcapi.h new file mode 100644 index 0000000..312dd49 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/example/ttcapi.h @@ -0,0 +1,890 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DTC { + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int ChangeNodeAddress = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + //for compress + void SetCompressLevel(int); + //get address and tablename set by user + const char * GetAddress(void) const; + const char * GetTableName(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char * GetServerAddress(void) const; + const char * GetServerTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + + void SetAccessKey(const char *token); + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int CompressSet(const char *, const char *, int); + //just compress and set. Don't need compressflag + int CompressSetForce(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *ErrorMessage(void) const; + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest: public Request { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor){} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + long long MemSize() const; + long long DataSize() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + int UnCompressBinaryValue(const char *name,char **buf,int *lenp); + //Uncompress Binary Value without check compressflag + int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp); + const char * UnCompressErrorMessage() const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/example/updatetest.cpp b/src/search_local/index_storage/api/c_api_cc/example/updatetest.cpp new file mode 100644 index 0000000..42a487e --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/example/updatetest.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + unsigned int age; + std::string name; + std::string city; + std::string descr; + + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + uid = atoi(argv[1]); + name = std::string(argv[2]); + city = std::string(argv[3]); + descr = std::string(argv[4]); + age = atoi(argv[5]); + DTC::UpdateRequest UpdateReq(&stServer); + retCode = UpdateReq.SetKey(uid); + if(retCode != 0) + { + printf("update-req set key error: %d", retCode); + fflush(stdout); + return(-1); + } + retCode = UpdateReq.Set("name", name.c_str()); + retCode = UpdateReq.Set("city", city.c_str()); + retCode = UpdateReq.Set("descr", descr.c_str()); + retCode = UpdateReq.Set("age", age); + if(retCode != 0) + { + printf("update-req set field error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + DTC::Result stResult; + retCode = UpdateReq.Execute(stResult); + printf("retCode:%d\n", retCode); + if(retCode == 0) + { + DTC::GetRequest getReq(&stServer); + getReq.SetKey(uid); + if(retCode == 0) + retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(retCode == 0) + retCode = getReq.Need("name"); + if(retCode == 0) + retCode = getReq.Need("city"); + if(retCode == 0) + retCode = getReq.Need("descr"); + if(retCode == 0) + retCode = getReq.Need("age"); + if(retCode != 0) + { + printf("get-req set key or need error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + retCode = getReq.Execute(stResult); + retCode = stResult.FetchRow();//开始获取数据 + printf("uid:%d\n", stResult.IntValue("uid")); + printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据 + printf("city: %s\n", stResult.StringValue("city")); + printf("descr: %s\n", stResult.BinaryValue("descr")); + printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据 + } + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/examples/cache.conf b/src/search_local/index_storage/api/c_api_cc/examples/cache.conf new file mode 100644 index 0000000..806f5cf --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/cache.conf @@ -0,0 +1,32 @@ +LogLevel = debug +BindAddr = 127.0.0.1:9898/tcp + +IdleTimeout = 30 +HelperTimeout = 30 + +CacheShmKey = 892123 +CacheMemorySize = 100M +CacheAverageDataSize = 50 + +MaxFlushSpeed = 1 +MinDirtyNode = 40 +MaxDirtyNode = 60 +MinDirtyTime = 3600 +MaxDirtyTime = 43200 + +DisableDataSource = 0 +DelayUpdate = 0 + +#opening file descritor overflow warning. +OpenningFDAttrID = 41784 + +StartStatReporter = 1 +EnableCoreDump = 1 +DisableWatchDog = 0 + +#ServerRecovery = crash +#[LogApi] +#MessageId = +#CallerId = +#TargetId = +#InterfaceId = diff --git a/src/search_local/index_storage/api/c_api_cc/examples/delete/Makefile b/src/search_local/index_storage/api/c_api_cc/examples/delete/Makefile new file mode 100644 index 0000000..535e6e2 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/delete/Makefile @@ -0,0 +1,39 @@ +LIB = ../../libdtc.so + +OUTPUT = delete + +SOURCES = deletetest.cpp + + +OBJECTS= deletetest.o + CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs + CFLAGS = $(CPPFLAGS) + +all:$(OUTPUT) + +.SUFFIXES: .o .cpp + .cpp.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^) + +.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^ + +$(OUTPUT): $(OBJECTS) + $(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB) + + +clean: + rm -f *.o *.so *.~ *.bak + rm -f $(OUTPUT1) + rm -f $(OUTPUT2) + +install: + cp -f $(OUTPUT) $(INSTALL_PATH) + +dep: + mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend + +ifneq ($(wildcard .depend),) + include .depend +endif + diff --git a/src/search_local/index_storage/api/c_api_cc/examples/delete/deletetest.cpp b/src/search_local/index_storage/api/c_api_cc/examples/delete/deletetest.cpp new file mode 100644 index 0000000..cf99675 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/delete/deletetest.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + DTC::DeleteRequest deleteReq(&stServer); + uid = atoi(argv[1]); + deleteReq.SetKey(uid); + + DTC::Result stResult; + retCode = deleteReq.Execute(stResult); + printf("retCode:%d\n", retCode); + if(retCode == 0) + { + printf("delete success!\n"); + } + + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/examples/delete/ttcapi.h b/src/search_local/index_storage/api/c_api_cc/examples/delete/ttcapi.h new file mode 100644 index 0000000..c113a7b --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/delete/ttcapi.h @@ -0,0 +1,839 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DTC { + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // һٸkey(Ĭ32) + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + const char * GetAddress(void) const; + const char * GetTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/examples/insert/Makefile b/src/search_local/index_storage/api/c_api_cc/examples/insert/Makefile new file mode 100644 index 0000000..c12bfab --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/insert/Makefile @@ -0,0 +1,38 @@ +LIB = ../../libdtc.so + +OUTPUT = insert + +SOURCES = inserttest.cpp + + +OBJECTS= inserttest.o + CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs + CFLAGS = $(CPPFLAGS) + +all:$(OUTPUT) + +.SUFFIXES: .o .cpp + .cpp.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^) + +.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^ + +$(OUTPUT): $(OBJECTS) + $(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB) + + +clean: + rm -f *.o *.so *.~ *.bak + rm -f $(OUTPUT) + +install: + cp -f $(OUTPUT) $(INSTALL_PATH) + +dep: + mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend + +ifneq ($(wildcard .depend),) + include .depend +endif + diff --git a/src/search_local/index_storage/api/c_api_cc/examples/insert/inserttest.cpp b/src/search_local/index_storage/api/c_api_cc/examples/insert/inserttest.cpp new file mode 100644 index 0000000..195d5b3 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/insert/inserttest.cpp @@ -0,0 +1,164 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + unsigned int age; + unsigned int lv = 0; + std::string name; + std::string city; + std::string descr; + unsigned int salary; + + unsigned int uid1; + unsigned int age1; + std::string name1; + std::string city1; + std::string descr1; + unsigned int salary1; + + lv = atoi(argv[1]); + DTC::init_log("dtc_client"); + DTC::set_log_level(lv); + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10901");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000090184d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + DTC::InsertRequest insertReq(&stServer); + //retCode = insertReq.SetKey(key); + + uid = atoi(argv[2]); + name = std::string(argv[3]); + city = std::string(argv[4]); + descr = std::string(argv[5]); + age = atoi(argv[6]); + salary = atoi(argv[7]); + + insertReq.SetKey(uid); + //insertReq.Set("key", 100003); + insertReq.Set("uid", uid); + insertReq.Set("name", name.c_str()); + insertReq.Set("city", city.c_str()); + insertReq.Set("descr", descr.c_str()); + insertReq.Set("age", age); + insertReq.Set("salary", salary); + + DTC::Result stResult; + retCode = insertReq.Execute(stResult); + printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult.ErrorMessage(), stResult.ErrorFrom()); + if(retCode == 0) + { + DTC::GetRequest getReq(&stServer); + getReq.SetKey(uid); + if(retCode == 0) + retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(retCode == 0) + retCode = getReq.Need("name"); + if(retCode == 0) + retCode = getReq.Need("city"); + if(retCode == 0) + retCode = getReq.Need("descr"); + if(retCode == 0) + retCode = getReq.Need("age"); + if(retCode == 0) + retCode = getReq.Need("salary"); + if(retCode != 0) + { + printf("get-req set key or need error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + stResult.Reset(); + retCode = getReq.Execute(stResult); + printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult.ErrorMessage(), stResult.ErrorFrom()); + retCode = stResult.FetchRow();//开始获取数据 + printf("uid:%lu\n", stResult.IntValue("uid")); + printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据 + printf("city: %s\n", stResult.StringValue("city")); + printf("descr: %s\n", stResult.BinaryValue("descr")); + printf("age: %lu\n", stResult.IntValue("age"));//输出int类型的数据 + printf("salary: %lu\n", stResult.IntValue("salary"));//输出int类型的数据 + } + + printf("-------------first request end -----------------\n"); + printf("-------------second request begin---------------\n"); + + DTC::Server stServer1; // 只要server不析构，后台会保持长连接 + stServer1.IntKey(); // 声明key类型 + stServer1.SetTableName("tp1");//设置dtc的表名，与table.conf中tablename应该一样 + stServer1.SetAddress("192.168.214.62", "10201");//设置的dtc的ip和端口 + stServer1.SetTimeout(5); // 设置网络超时时间 + stServer1.SetAccessKey("0000020184d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + DTC::InsertRequest insertReq1(&stServer1); + //retCode = insertReq.SetKey(key); + + uid1 = atoi(argv[8]); + name1 = std::string(argv[9]); + city1 = std::string(argv[10]); + descr1 = std::string(argv[11]); + age1 = atoi(argv[12]); + salary1 = atoi(argv[13]); + + insertReq1.SetKey(uid1); + //insertReq.Set("key", 100003); + insertReq1.Set("uid", uid1); + insertReq1.Set("name", name1.c_str()); + insertReq1.Set("city", city1.c_str()); + insertReq1.Set("descr", descr1.c_str()); + insertReq1.Set("age", age1); + insertReq1.Set("salary", salary1); + + DTC::Result stResult1; + retCode = insertReq1.Execute(stResult1); + printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult1.ErrorMessage(), stResult1.ErrorFrom()); + if(retCode == 0) + { + DTC::GetRequest getReq1(&stServer1); + getReq1.SetKey(uid); + if(retCode == 0) + retCode = getReq1.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(retCode == 0) + retCode = getReq1.Need("name"); + if(retCode == 0) + retCode = getReq1.Need("city"); + if(retCode == 0) + retCode = getReq1.Need("descr"); + if(retCode == 0) + retCode = getReq1.Need("age"); + if(retCode == 0) + retCode = getReq1.Need("salary"); + if(retCode != 0) + { + printf("get-req set key or need error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + stResult1.Reset(); + retCode = getReq1.Execute(stResult1); + printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult1.ErrorMessage(), stResult1.ErrorFrom()); + retCode = stResult1.FetchRow();//开始获取数据 + printf("uid:%lu\n", stResult1.IntValue("uid")); + printf("name: %s\n", stResult1.StringValue("name"));//输出binary类型的数据 + printf("city: %s\n", stResult1.StringValue("city")); + printf("descr: %s\n", stResult1.BinaryValue("descr")); + printf("age: %lu\n", stResult1.IntValue("age"));//输出int类型的数据 + printf("salary: %lu\n", stResult1.IntValue("salary"));//输出int类型的数据 + } + + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/examples/insert/ttcapi.h b/src/search_local/index_storage/api/c_api_cc/examples/insert/ttcapi.h new file mode 100644 index 0000000..312dd49 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/insert/ttcapi.h @@ -0,0 +1,890 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DTC { + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int ChangeNodeAddress = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + //for compress + void SetCompressLevel(int); + //get address and tablename set by user + const char * GetAddress(void) const; + const char * GetTableName(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char * GetServerAddress(void) const; + const char * GetServerTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + + void SetAccessKey(const char *token); + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int CompressSet(const char *, const char *, int); + //just compress and set. Don't need compressflag + int CompressSetForce(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *ErrorMessage(void) const; + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest: public Request { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor){} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + long long MemSize() const; + long long DataSize() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + int UnCompressBinaryValue(const char *name,char **buf,int *lenp); + //Uncompress Binary Value without check compressflag + int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp); + const char * UnCompressErrorMessage() const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/examples/select/Makefile b/src/search_local/index_storage/api/c_api_cc/examples/select/Makefile new file mode 100644 index 0000000..cc95d3a --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/select/Makefile @@ -0,0 +1,47 @@ +LIB = ../../libdtc.so + +OUTPUT1 = selecttest + +SOURCES = selecttest.cpp + +OUTPUT2 = TestServerPool + +SOURCES = TestServerPool.cpp + +OBJECTS1= selecttest.o + CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs + CFLAGS = $(CPPFLAGS) + +OBJECTS2=TestServerPool.o + CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs + CFLAGS = $(CPPFLAGS) +all:$(OUTPUT1) $(OUTPUT2) + +.SUFFIXES: .o .cpp + .cpp.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^) + +.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^ + +$(OUTPUT1): $(OBJECTS1) + $(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB) + +$(OUTPUT2): $(OBJECTS2) + $(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB) + +clean: + rm -f *.o *.so *.~ *.bak + rm -f $(OUTPUT1) + rm -f $(OUTPUT2) + +install: + cp -f $(OUTPUT) $(INSTALL_PATH) + +dep: + mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend + +ifneq ($(wildcard .depend),) + include .depend +endif + diff --git a/src/search_local/index_storage/api/c_api_cc/examples/select/TestServerPool.cpp b/src/search_local/index_storage/api/c_api_cc/examples/select/TestServerPool.cpp new file mode 100644 index 0000000..c466011 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/select/TestServerPool.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +using namespace DTC; + +int main(int argc,char* argv[]) +{ + int ret = 0; + int uid = 0; + int age = 0; + int salary = 0; + std::string name; + std::string city; + std::string descr; + + uid = atoi(argv[1]); + name = std::string(argv[2]); + city = std::string(argv[3]); + descr = std::string(argv[4]); + age = atoi(argv[5]); + salary = atoi(argv[6]); + + ServerPool svp(5, 5); + Server sv1, sv2, sv3, sv4, sv5; + + /* server-request-1 */ + sv1.IntKey(); + sv1.SetTableName("tp1"); + sv1.SetAddress("192.168.214.62", "10201"); + sv1.SetAccessKey("0000020184d9cfc2f395ce883a41d7ffc1bbcf4e"); + sv1.SetTimeout(5); + + InsertRequest req1(&sv1); + req1.SetKey(uid); + //req1.Set("uid", 100001); + req1.Set("name", name.c_str()); + req1.Set("city", city.c_str()); + req1.Set("descr", descr.c_str()); + req1.Set("age", age); + req1.Set("salary", salary); + + /* server-request-2 */ + sv2.IntKey(); + sv2.SetTableName("tp2"); + sv2.SetAddress("192.168.214.62", "10202"); + sv2.SetAccessKey("0000020284d9cfc2f395ce883a41d7ffc1bbcf4e"); + sv2.SetTimeout(5); + + InsertRequest req2(&sv2); + req2.SetKey(uid); + //req2.Set("uid", 100001); + req2.Set("name", name.c_str()); + req2.Set("city", city.c_str()); + req2.Set("descr", descr.c_str()); + req2.Set("age", age); + req2.Set("salary", salary); + /* server-request-3 */ + sv3.IntKey(); + sv3.SetTableName("tp3"); + sv3.SetAddress("192.168.214.62", "10203"); + sv3.SetAccessKey("0000020384d9cfc2f395ce883a41d7ffc1bbcf4e"); + sv3.SetTimeout(5); + + InsertRequest req3(&sv3); + req3.SetKey(uid); + //req2.Set("uid", 100001); + req3.Set("name", name.c_str()); + req3.Set("city", city.c_str()); + req3.Set("descr", descr.c_str()); + req3.Set("age", age); + req3.Set("salary", salary); + /* server-request-4 */ + sv4.IntKey(); + sv4.SetTableName("tp4"); + sv4.SetAddress("192.168.214.62", "10204"); + sv4.SetAccessKey("0000020484d9cfc2f395ce883a41d7ffc1bbcf4e"); + sv4.SetTimeout(5); + + InsertRequest req4(&sv4); + req4.SetKey(uid); + //req4.Set("uid", 100001); + req4.Set("name", name.c_str()); + req4.Set("city", city.c_str()); + req4.Set("descr", descr.c_str()); + req4.Set("age", age); + req4.Set("salary", salary); + /* server-request-5 */ + sv5.IntKey(); + sv5.SetTableName("tp5"); + sv5.SetAddress("192.168.214.62", "10205"); + sv5.SetAccessKey("0000020584d9cfc2f395ce883a41d7ffc1bbcf4e"); + sv5.SetTimeout(5); + + InsertRequest req5(&sv5); + req5.SetKey(uid); + //req5.Set("uid", 100001); + req5.Set("name", name.c_str()); + req5.Set("city", city.c_str()); + req5.Set("descr", descr.c_str()); + req5.Set("age", age); + req5.Set("salary", salary); + + svp.AddRequest(&req1, 1); + svp.AddRequest(&req2, 2); + svp.AddRequest(&req3, 3); + svp.AddRequest(&req4, 4); + svp.AddRequest(&req5, 5); + + ret = svp.Execute(120000); + printf("ret:%d\n", ret); + + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/examples/select/selecttest.cpp b/src/search_local/index_storage/api/c_api_cc/examples/select/selecttest.cpp new file mode 100644 index 0000000..4e84d48 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/select/selecttest.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int iRet; + unsigned int uid; + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + //下面三行代码是Get数据 + unsigned int age; + std::string name; + std::string city; + std::string descr; + + DTC::GetRequest stGetReq(&stServer); + uid = atoi(argv[1]); + iRet = stGetReq.SetKey(uid); + + if(iRet == 0) + iRet = stGetReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(iRet == 0) + iRet = stGetReq.Need("name"); + if(iRet == 0) + iRet = stGetReq.Need("city"); + if(iRet == 0) + iRet = stGetReq.Need("descr"); + if(iRet == 0) + iRet = stGetReq.Need("age"); + if(iRet == 0) + iRet = stGetReq.GT("age", 24); + //iRet = stGetReq.OR("age", 27); + + if(iRet != 0) + { + printf("get-req set key or need error: %d", iRet); + fflush(stdout); + return(-1); + } + + // execute & get result + DTC::Result stResult; + iRet = stGetReq.Execute(stResult); + if(iRet != 0) + //如果出错,则输出错误码、错误阶段，错误信息，stResult.ErrorFrom(), stResult.ErrorMessage() 这两个错误信息很重要，一定要打印出来，方便定位问题 + { + printf ("uin[%u] dtc execute get error: %d, error_from:%s, msg:%s\n", + uid,//出错的key是多少 + iRet,//错误码为多少 + stResult.ErrorFrom(),//返回错误阶段 + stResult.ErrorMessage()//返回错误信息 + ); + fflush(stdout); + return(-2); + } + + if(stResult.NumRows() <= 0) + { + // 数据不存在 + printf("uin[%u] data not exist.\n", uid); + return(0); + } + + printf("nubrow:%d\n", stResult.NumRows()); + for(int i=0;i<=stResult.NumRows();++i) + { + iRet = stResult.FetchRow();//开始获取数据 + if(iRet < 0) + { + printf ("uid[%lu] dtc fetch row error: %d\n", uid, iRet); + fflush(stdout); + return(-3); + } + //如果一切正确，则可以输出数据了 + printf("uid: %d\n", stResult.IntValue("uid"));//输出int类型的数据 + printf("name: %s\n", stResult.StringValue("name"));//输出string类型的数据 + printf("city: %s\n", stResult.StringValue("city"));//输出string类型的数据 + printf("descr: %s\n", stResult.BinaryValue("descr"));//输出binary类型的数据 + printf("age:%d\n",stResult.IntValue("age")); + } + return(0); +} diff --git a/src/search_local/index_storage/api/c_api_cc/examples/select/ttcapi.h b/src/search_local/index_storage/api/c_api_cc/examples/select/ttcapi.h new file mode 100644 index 0000000..312dd49 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/select/ttcapi.h @@ -0,0 +1,890 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DTC { + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int ChangeNodeAddress = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + //for compress + void SetCompressLevel(int); + //get address and tablename set by user + const char * GetAddress(void) const; + const char * GetTableName(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char * GetServerAddress(void) const; + const char * GetServerTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + + void SetAccessKey(const char *token); + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int CompressSet(const char *, const char *, int); + //just compress and set. Don't need compressflag + int CompressSetForce(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *ErrorMessage(void) const; + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest: public Request { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor){} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + long long MemSize() const; + long long DataSize() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + int UnCompressBinaryValue(const char *name,char **buf,int *lenp); + //Uncompress Binary Value without check compressflag + int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp); + const char * UnCompressErrorMessage() const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/examples/table.conf b/src/search_local/index_storage/api/c_api_cc/examples/table.conf new file mode 100644 index 0000000..ed11adb --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/table.conf @@ -0,0 +1,46 @@ +[DB_DEFINE] +DbName = test +DbNum = (1,1) +dbMax = 1 +MachineNum = 1 +Deploy = 0 + +[Machine1] +Procs = 10 +WriteProcs = 10 +CommitProcs = 10 +#DbIdx = [0] +DbAddr = 10.152.23.179:9005 +DbUser = root +DbPass = rootcdb + +[TABLE_DEFINE] +TableName = test +FieldCount = 4 +KeyFieldCount = 1 +#TableNum = (1,100) + +#None=0, Signed=1, Unsigned=2, Float=3, String=4, Binary=5 +[FIELD1] +FieldName = name +FieldType = 4 +FieldSize = 255 +UniqField = 1 + +[FIELD2] +FieldName = age +FieldType = 1 +FieldSize = 4 +Value = 0 + +[FIELD3] +FieldName = sex +FieldType = 1 +FieldSize = 4 +Value = 0 + +[FIELD4] +FieldName = city +FieldType = 1 +FieldSize = 4 +Value = 0 diff --git a/src/search_local/index_storage/api/c_api_cc/examples/update/Makefile b/src/search_local/index_storage/api/c_api_cc/examples/update/Makefile new file mode 100644 index 0000000..5b06014 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/update/Makefile @@ -0,0 +1,38 @@ +LIB = ../../libdtc.so + +OUTPUT = update + +SOURCES = updatetest.cpp + + +OBJECTS= updatetest.o + CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs + CFLAGS = $(CPPFLAGS) + +all:$(OUTPUT) + +.SUFFIXES: .o .cpp + .cpp.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^) + +.o: + $(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^ + +$(OUTPUT): $(OBJECTS) + $(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB) + + +clean: + rm -f *.o *.so *.~ *.bak + rm -f $(OUTPUT) + +install: + cp -f $(OUTPUT) $(INSTALL_PATH) + +dep: + mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend + +ifneq ($(wildcard .depend),) + include .depend +endif + diff --git a/src/search_local/index_storage/api/c_api_cc/examples/update/ttcapi.h b/src/search_local/index_storage/api/c_api_cc/examples/update/ttcapi.h new file mode 100644 index 0000000..312dd49 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/update/ttcapi.h @@ -0,0 +1,890 @@ +#ifndef __DTC_API_H__ +#define __DTC_API_H__ + +#include + +namespace DTC { + class Server; + class ServerPool; + class Request; + class Result; + + const int RequestSvrAdmin = 3; + const int RequestGet = 4; + const int RequestPurge = 5; + const int RequestInsert = 6; + const int RequestUpdate = 7; + const int RequestDelete = 8; + const int RequestReplace = 12; + const int RequestFlush = 13; + const int RequestInvalidate = 14; + + //mem monior Request;2014/06/6;by seanzheng + const int RequestMonitor = 15; + + // sub-code of admin cmd + const int RegisterHB = 1; + const int LogoutHB = 2; + const int GetKeyList = 3; + const int GetUpdateKey = 4; + const int GetRawData = 5; + const int ReplaceRawData = 6; + const int AdjustLRU = 7; + const int VerifyHBT = 8; + const int GetHBTime = 9; + const int SET_READONLY = 10; + const int SET_READWRITE = 11; + const int QueryBinlogID = 12; + const int NodeHandleChange = 13; + const int Migrate = 14; + const int ReloadClusterNodeList = 15; + const int SetClusterNodeState = 16; + const int ChangeNodeAddress = 17; + const int GetClusterState = 18; + const int PurgeForHit = 19; + + const int KeyTypeNone = 0; // undefined + const int KeyTypeInt = 1; // Signed Integer + const int KeyTypeString = 4; // String, case insensitive, null ended + + const int FieldTypeNone = 0; // undefined + const int FieldTypeSigned = 1; // Signed Integer + const int FieldTypeUnsigned = 2; // Unsigned Integer + const int FieldTypeFloat = 3; // float + const int FieldTypeString = 4; // String, case insensitive, null ended + const int FieldTypeBinary = 5; // binary + + void init_log (const char *app, const char *dir = NULL); + void set_log_level(int n); + int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个) + void write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) + __attribute__((format(printf,5,6))); + + class Result; + class Server { + private: + void *addr; + long check; + public: + friend class Request; + friend class Result; + friend class ServerPool; + + Server(void); + ~Server(void); + Server(const Server &); + void CloneTabDef(const Server& source); + int SetAddress(const char *host, const char *port=0); + int SetTableName(const char *); + //for compress + void SetCompressLevel(int); + //get address and tablename set by user + const char * GetAddress(void) const; + const char * GetTableName(void) const; + //get address and tablename set by dtc frame,for plugin only; + const char * GetServerAddress(void) const; + const char * GetServerTableName(void) const; + int IntKey(void); + int BinaryKey(void); + int StringKey(void); + int AddKey(const char* name, int type); + int FieldType(const char* name); + const char *ErrorMessage(void) const; + void SetTimeout(int); + void SetMTimeout(int); + int Connect(void); + void Close(void); + int Ping(void); + void AutoPing(void); + void SetFD(int); // UNSUPPORTED API + void SetAutoUpdateTab(bool autoUpdate); + void SetAutoReconnect(int autoReconnect); + int DecodePacket(Result &, const char *, int); + int CheckPacketSize(const char *, int); + + void SetAccessKey(const char *token); + }; + + class Request { + private: + void *addr; + long check; + Request(const Request &); + public: + friend class Server; + friend class Result; + friend class ServerPool; + + Request(Server *srv, int op); + Request(void); + ~Request(void); + void Reset(void); + void Reset(int); + int AttachServer(Server *srv); + + void SetAdminCode(int code); + void SetHotBackupID(long long); + void SetMasterHBTimestamp(long long); + void SetSlaveHBTimestamp(long long); + +#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); } +#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); } + int Need(const char *); + int Need(const char *, int); + int FieldType(const char*); + void NoCache(void); + void NoNextServer(void); + void Limit(unsigned int, unsigned int); + int EQ(const char *, long long); + int NE(const char *, long long); + int LT(const char *, long long); + int LE(const char *, long long); + int GT(const char *, long long); + int GE(const char *, long long); + int EQ(const char *, const char *); + int NE(const char *, const char *); + int EQ(const char *, const char *, int); + int NE(const char *, const char *, int); + + _REDIR_(EQ, unsigned long long); + _REDIR_(EQ, long); + _REDIR_(EQ, unsigned long); + _REDIR_(EQ, int); + _REDIR_(EQ, unsigned int); + _REDIR_(EQ, short); + _REDIR_(EQ, unsigned short); + _REDIR_(EQ, char); + _REDIR_(EQ, unsigned char); + + _REDIR_(NE, unsigned long long); + _REDIR_(NE, long); + _REDIR_(NE, unsigned long); + _REDIR_(NE, int); + _REDIR_(NE, unsigned int); + _REDIR_(NE, short); + _REDIR_(NE, unsigned short); + _REDIR_(NE, char); + _REDIR_(NE, unsigned char); + + _REDIR_(GT, unsigned long long); + _REDIR_(GT, long); + _REDIR_(GT, unsigned long); + _REDIR_(GT, int); + _REDIR_(GT, unsigned int); + _REDIR_(GT, short); + _REDIR_(GT, unsigned short); + _REDIR_(GT, char); + _REDIR_(GT, unsigned char); + + _REDIR_(GE, unsigned long long); + _REDIR_(GE, long); + _REDIR_(GE, unsigned long); + _REDIR_(GE, int); + _REDIR_(GE, unsigned int); + _REDIR_(GE, short); + _REDIR_(GE, unsigned short); + _REDIR_(GE, char); + _REDIR_(GE, unsigned char); + + _REDIR_(LT, unsigned long long); + _REDIR_(LT, long); + _REDIR_(LT, unsigned long); + _REDIR_(LT, int); + _REDIR_(LT, unsigned int); + _REDIR_(LT, short); + _REDIR_(LT, unsigned short); + _REDIR_(LT, char); + _REDIR_(LT, unsigned char); + + _REDIR_(LE, unsigned long long); + _REDIR_(LE, long); + _REDIR_(LE, unsigned long); + _REDIR_(LE, int); + _REDIR_(LE, unsigned int); + _REDIR_(LE, short); + _REDIR_(LE, unsigned short); + _REDIR_(LE, char); + _REDIR_(LE, unsigned char); + + int Set(const char *, long long); + int OR(const char *, long long); + int Add(const char *, long long); + int Sub(const char *, long long); + int Set(const char *, double); + int Add(const char *, double); + int Sub(const char *, double); + int Set(const char *, const char *); + int Set(const char *, const char *, int); + + //just for compress,only support binary field + int CompressSet(const char *, const char *, int); + //just compress and set. Don't need compressflag + int CompressSetForce(const char *, const char *, int); + + //bits op + int SetMultiBits(const char *, int, int, unsigned int); + int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);} + int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);} + + _REDIR_(Set, unsigned long long); + _REDIR_(Set, long); + _REDIR_(Set, unsigned long); + _REDIR_(Set, int); + _REDIR_(Set, unsigned int); + _REDIR_(Set, short); + _REDIR_(Set, unsigned short); + _REDIR_(Set, char); + _REDIR_(Set, unsigned char); + _REDIRF_(Set, float); + _REDIRF_(Set, long double); + + _REDIR_(OR, unsigned long long); + _REDIR_(OR, long); + _REDIR_(OR, unsigned long); + _REDIR_(OR, int); + _REDIR_(OR, unsigned int); + _REDIR_(OR, short); + _REDIR_(OR, unsigned short); + + _REDIR_(Add, unsigned long long); + _REDIR_(Add, long); + _REDIR_(Add, unsigned long); + _REDIR_(Add, int); + _REDIR_(Add, unsigned int); + _REDIR_(Add, short); + _REDIR_(Add, unsigned short); + _REDIR_(Add, char); + _REDIR_(Add, unsigned char); + _REDIRF_(Add, float); + _REDIRF_(Add, long double); + + _REDIR_(Sub, unsigned long long); + _REDIR_(Sub, long); + _REDIR_(Sub, unsigned long); + _REDIR_(Sub, int); + _REDIR_(Sub, unsigned int); + _REDIR_(Sub, short); + _REDIR_(Sub, unsigned short); + _REDIR_(Sub, char); + _REDIR_(Sub, unsigned char); + _REDIRF_(Sub, float); + _REDIRF_(Sub, long double); +#undef _REDIR_ + + void UnsetKey(void); + int SetKey(long long); + int SetKey(const char *); + int SetKey(const char *, int); +#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int AddKeyValue(const char* name, long long v); + int AddKeyValue(const char* name, const char *str); + int AddKeyValue(const char* name, const char *ptr, int len); +#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + Result *Execute(void); + Result *Execute(long long); + Result *Execute(const char *); + Result *Execute(const char *, int); + +#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int Execute(Result&); + int Execute(Result&, long long); + int Execute(Result&, const char *); + int Execute(Result&, const char *, int); + +#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int EncodePacket(char *&, int&, long long&); + int EncodePacket(char *&, int&, long long&, long long); + int EncodePacket(char *&, int&, long long&, const char *); + int EncodePacket(char *&, int&, long long&, const char *, int); + +#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); } + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + + int SetCacheID(long long); +#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);} + _REDIR_(unsigned long long); + _REDIR_(long); + _REDIR_(unsigned long); + _REDIR_(int); + _REDIR_(unsigned int); + _REDIR_(short); + _REDIR_(unsigned short); + _REDIR_(char); + _REDIR_(unsigned char); +#undef _REDIR_ + const char *ErrorMessage(void) const; + }; + + class GetRequest : public Request { + public: + GetRequest(Server *srv): Request(srv, RequestGet) {} + GetRequest(): Request((Server *)0, RequestGet) {} + }; + + class InsertRequest : public Request { + public: + InsertRequest(Server *srv) : Request(srv, RequestInsert) {} + InsertRequest() : Request((Server *)0, RequestInsert) {} + }; + + class DeleteRequest : public Request { + public: + DeleteRequest(Server *srv) : Request(srv, RequestDelete) {} + DeleteRequest() : Request((Server *)0, RequestDelete) {} + }; + + class UpdateRequest : public Request { + public: + UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {} + UpdateRequest() : Request((Server *)0, RequestUpdate) {} + }; + + class PurgeRequest : public Request { + public: + PurgeRequest(Server *srv) : Request(srv, RequestPurge) {} + PurgeRequest() : Request((Server *)0, RequestPurge) {} + }; + + class ReplaceRequest:public Request { + public: + ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {} + ReplaceRequest() : Request((Server *)0, RequestReplace) {} + }; + + class FlushRequest : public Request { + public: + FlushRequest(Server *srv) : Request(srv, RequestFlush) {} + FlushRequest(void) : Request((Server *)0, RequestFlush) {} + }; + + class InvalidateRequest: public Request { + public: + InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {} + }; + + class SvrAdminRequest: public Request { + public: + SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {} + }; + + class MonitorRequest: public Request { + public: + MonitorRequest(Server *srv) : Request(srv, RequestMonitor){} + }; + + class Result { + private: + void *addr; + long check; + Result(const Result &); + char *ServerInfo() const; + public: + friend class Server; + friend class Request; + friend class ServerPool; + + Result(void); + ~Result(void); + void Reset(void); + + void SetError(int errcode, const char *from, const char *detail); // from will not dupped + int ResultCode(void) const; + const char *ErrorMessage(void) const; + const char *ErrorFrom(void) const; + long long HotBackupID() const; + long long MasterHBTimestamp() const; + long long SlaveHBTimestamp() const; + long long BinlogID() const; + long long BinlogOffset() const; + long long MemSize() const; + long long DataSize() const; + int NumRows(void) const; + int TotalRows(void) const; + int AffectedRows(void) const; + int NumFields(void) const; + const char* FieldName(int n) const; + int FieldPresent(const char* name) const; + int FieldType(int n) const; + long long Tag(void) const; + void *TagPtr(void) const; + long long Magic(void) const; + long long ServerTimestamp(void) const; + long long InsertID(void) const; + long long IntKey(void) const; + const char *BinaryKey(void) const; + const char *BinaryKey(int *) const; + const char *BinaryKey(int &) const; + const char *StringKey(void) const; + const char *StringKey(int *) const; + const char *StringKey(int &) const; + long long IntValue(const char *) const; + double FloatValue(const char *) const; + const char *StringValue(const char *) const; + const char *StringValue(const char *, int*) const; + const char *StringValue(const char *, int&) const; + const char *BinaryValue(const char *) const; + const char *BinaryValue(const char *, int*) const; + const char *BinaryValue(const char *, int&) const; + int UnCompressBinaryValue(const char *name,char **buf,int *lenp); + //Uncompress Binary Value without check compressflag + int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp); + const char * UnCompressErrorMessage() const; + long long IntValue(int) const; + double FloatValue(int) const; + const char *StringValue(int) const; + const char *StringValue(int, int*) const; + const char *StringValue(int, int&) const; + const char *BinaryValue(int) const; + const char *BinaryValue(int, int*) const; + const char *BinaryValue(int, int&) const; + int FetchRow(void); + int Rewind(void); + }; + + class ServerPool { + private: + void *addr; + long check; + ServerPool(ServerPool &); + public: + friend class Server; + friend class Request; + friend class Result; + + ServerPool(int maxServers, int maxRequests); + ~ServerPool(void); + + int GetEpollFD(int size); + int AddServer(Server *srv, int mReq=1, int mConn=0); + int AddRequest(Request *, long long); + int AddRequest(Request *, long long, long long); + int AddRequest(Request *, long long, const char *); + int AddRequest(Request *, long long, const char *, int); + int AddRequest(Request *, void *); + int AddRequest(Request *, void *, long long); + int AddRequest(Request *, void *, const char *); + int AddRequest(Request *, void *, const char *, int); + int Execute(int msec); + int ExecuteAll(int msec); + int CancelRequest(int); + int CancelAllRequest(int type); + int AbortRequest(int); + int AbortAllRequest(int type); + Result *GetResult(void); + Result *GetResult(int); + int GetResult(Result&); + int GetResult(Result&, int); + + int ServerCount(void) const; + int RequestCount(int type) const; + int RequestState(int reqId) const; + }; + + const int WAIT = 1; + const int SEND = 2; + const int RECV = 4; + const int DONE = 8; + const int ALL_STATE = WAIT|SEND|RECV|DONE; + + enum { + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalud field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + }; + + enum { + ER_HASHCHK=1000, + ER_NISAMCHK=1001, + ER_NO=1002, + ER_YES=1003, + ER_CANT_CREATE_FILE=1004, + ER_CANT_CREATE_TABLE=1005, + ER_CANT_CREATE_DB=1006, + ER_DB_CREATE_EXISTS=1007, + ER_DB_DROP_EXISTS=1008, + ER_DB_DROP_DELETE=1009, + ER_DB_DROP_RMDIR=1010, + ER_CANT_DELETE_FILE=1011, + ER_CANT_FIND_SYSTEM_REC=1012, + ER_CANT_GET_STAT=1013, + ER_CANT_GET_WD=1014, + ER_CANT_LOCK=1015, + ER_CANT_OPEN_FILE=1016, + ER_FILE_NOT_FOUND=1017, + ER_CANT_READ_DIR=1018, + ER_CANT_SET_WD=1019, + ER_CHECKREAD=1020, + ER_DISK_FULL=1021, + ER_DUP_KEY=1022, + ER_ERROR_ON_CLOSE=1023, + ER_ERROR_ON_READ=1024, + ER_ERROR_ON_RENAME=1025, + ER_ERROR_ON_WRITE=1026, + ER_FILE_USED=1027, + ER_FILSORT_ABORT=1028, + ER_FORM_NOT_FOUND=1029, + ER_GET_ERRNO=1030, + ER_ILLEGAL_HA=1031, + ER_KEY_NOT_FOUND=1032, + ER_NOT_FORM_FILE=1033, + ER_NOT_KEYFILE=1034, + ER_OLD_KEYFILE=1035, + ER_OPEN_AS_READONLY=1036, + ER_OUTOFMEMORY=1037, + ER_OUT_OF_SORTMEMORY=1038, + ER_UNEXPECTED_EOF=1039, + ER_CON_COUNT_ERROR=1040, + ER_OUT_OF_RESOURCES=1041, + ER_BAD_HOST_ERROR=1042, + ER_HANDSHAKE_ERROR=1043, + ER_DBACCESS_DENIED_ERROR=1044, + ER_ACCESS_DENIED_ERROR=1045, + ER_NO_DB_ERROR=1046, + ER_UNKNOWN_COM_ERROR=1047, + ER_BAD_NULL_ERROR=1048, + ER_BAD_DB_ERROR=1049, + ER_TABLE_EXISTS_ERROR=1050, + ER_BAD_TABLE_ERROR=1051, + ER_NON_UNIQ_ERROR=1052, + ER_SERVER_SHUTDOWN=1053, + ER_BAD_FIELD_ERROR=1054, + ER_WRONG_FIELD_WITH_GROUP=1055, + ER_WRONG_GROUP_FIELD=1056, + ER_WRONG_SUM_SELECT=1057, + ER_WRONG_VALUE_COUNT=1058, + ER_TOO_LONG_IDENT=1059, + ER_DUP_FIELDNAME=1060, + ER_DUP_KEYNAME=1061, + ER_DUP_ENTRY=1062, + ER_WRONG_FIELD_SPEC=1063, + ER_PARSE_ERROR=1064, + ER_EMPTY_QUERY=1065, + ER_NONUNIQ_TABLE=1066, + ER_INVALID_DEFAULT=1067, + ER_MULTIPLE_PRI_KEY=1068, + ER_TOO_MANY_KEYS=1069, + ER_TOO_MANY_KEY_PARTS=1070, + ER_TOO_LONG_KEY=1071, + ER_KEY_COLUMN_DOES_NOT_EXITS=1072, + ER_BLOB_USED_AS_KEY=1073, + ER_TOO_BIG_FIELDLENGTH=1074, + ER_WRONG_AUTO_KEY=1075, + ER_READY=1076, + ER_NORMAL_SHUTDOWN=1077, + ER_GOT_SIGNAL=1078, + ER_SHUTDOWN_COMPLETE=1079, + ER_FORCING_CLOSE=1080, + ER_IPSOCK_ERROR=1081, + ER_NO_SUCH_INDEX=1082, + ER_WRONG_FIELD_TERMINATORS=1083, + ER_BLOBS_AND_NO_TERMINATED=1084, + ER_TEXTFILE_NOT_READABLE=1085, + ER_FILE_EXISTS_ERROR=1086, + ER_LOAD_INFO=1087, + ER_ALTER_INFO=1088, + ER_WRONG_SUB_KEY=1089, + ER_CANT_REMOVE_ALL_FIELDS=1090, + ER_CANT_DROP_FIELD_OR_KEY=1091, + ER_INSERT_INFO=1092, + ER_INSERT_TABLE_USED=1093, + ER_NO_SUCH_THREAD=1094, + ER_KILL_DENIED_ERROR=1095, + ER_NO_TABLES_USED=1096, + ER_TOO_BIG_SET=1097, + ER_NO_UNIQUE_LOGFILE=1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE=1099, + ER_TABLE_NOT_LOCKED=1100, + ER_BLOB_CANT_HAVE_DEFAULT=1101, + ER_WRONG_DB_NAME=1102, + ER_WRONG_TABLE_NAME=1103, + ER_TOO_BIG_SELECT=1104, + ER_UNKNOWN_ERROR=1105, + ER_UNKNOWN_PROCEDURE=1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE=1108, + ER_UNKNOWN_TABLE=1109, + ER_FIELD_SPECIFIED_TWICE=1110, + ER_INVALID_GROUP_FUNC_USE=1111, + ER_UNSUPPORTED_EXTENSION=1112, + ER_TABLE_MUST_HAVE_COLUMNS=1113, + ER_RECORD_FILE_FULL=1114, + ER_UNKNOWN_CHARACTER_SET=1115, + ER_TOO_MANY_TABLES=1116, + ER_TOO_MANY_FIELDS=1117, + ER_TOO_BIG_ROWSIZE=1118, + ER_STACK_OVERRUN=1119, + ER_WRONG_OUTER_JOIN=1120, + ER_NULL_COLUMN_IN_INDEX=1121, + ER_CANT_FIND_UDF=1122, + ER_CANT_INITIALIZE_UDF=1123, + ER_UDF_NO_PATHS=1124, + ER_UDF_EXISTS=1125, + ER_CANT_OPEN_LIBRARY=1126, + ER_CANT_FIND_DL_ENTRY=1127, + ER_FUNCTION_NOT_DEFINED=1128, + ER_HOST_IS_BLOCKED=1129, + ER_HOST_NOT_PRIVILEGED=1130, + ER_PASSWORD_ANONYMOUS_USER=1131, + ER_PASSWORD_NOT_ALLOWED=1132, + ER_PASSWORD_NO_MATCH=1133, + ER_UPDATE_INFO=1134, + ER_CANT_CREATE_THREAD=1135, + ER_WRONG_VALUE_COUNT_ON_ROW=1136, + ER_CANT_REOPEN_TABLE=1137, + ER_INVALID_USE_OF_NULL=1138, + ER_REGEXP_ERROR=1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140, + ER_NONEXISTING_GRANT=1141, + ER_TABLEACCESS_DENIED_ERROR=1142, + ER_COLUMNACCESS_DENIED_ERROR=1143, + ER_ILLEGAL_GRANT_FOR_TABLE=1144, + ER_GRANT_WRONG_HOST_OR_USER=1145, + ER_NO_SUCH_TABLE=1146, + ER_NONEXISTING_TABLE_GRANT=1147, + ER_NOT_ALLOWED_COMMAND=1148, + ER_SYNTAX_ERROR=1149, + ER_DELAYED_CANT_CHANGE_LOCK=1150, + ER_TOO_MANY_DELAYED_THREADS=1151, + ER_ABORTING_CONNECTION=1152, + ER_NET_PACKET_TOO_LARGE=1153, + ER_NET_READ_ERROR_FROM_PIPE=1154, + ER_NET_FCNTL_ERROR=1155, + ER_NET_PACKETS_OUT_OF_ORDER=1156, + ER_NET_UNCOMPRESS_ERROR=1157, + ER_NET_READ_ERROR=1158, + ER_NET_READ_INTERRUPTED=1159, + ER_NET_ERROR_ON_WRITE=1160, + ER_NET_WRITE_INTERRUPTED=1161, + ER_TOO_LONG_STRING=1162, + ER_TABLE_CANT_HANDLE_BLOB=1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164, + ER_DELAYED_INSERT_TABLE_LOCKED=1165, + ER_WRONG_COLUMN_NAME=1166, + ER_WRONG_KEY_COLUMN=1167, + ER_WRONG_MRG_TABLE=1168, + ER_DUP_UNIQUE=1169, + ER_BLOB_KEY_WITHOUT_LENGTH=1170, + ER_PRIMARY_CANT_HAVE_NULL=1171, + ER_TOO_MANY_ROWS=1172, + ER_REQUIRES_PRIMARY_KEY=1173, + ER_NO_RAID_COMPILED=1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175, + ER_KEY_DOES_NOT_EXITS=1176, + ER_CHECK_NO_SUCH_TABLE=1177, + ER_CHECK_NOT_IMPLEMENTED=1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179, + ER_ERROR_DURING_COMMIT=1180, + ER_ERROR_DURING_ROLLBACK=1181, + ER_ERROR_DURING_FLUSH_LOGS=1182, + ER_ERROR_DURING_CHECKPOINT=1183, + ER_NEW_ABORTING_CONNECTION=1184, + ER_DUMP_NOT_IMPLEMENTED= 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED=1186, + ER_INDEX_REBUILD= 1187, + ER_MASTER=1188, + ER_MASTER_NET_READ=1189, + ER_MASTER_NET_WRITE=1190, + ER_FT_MATCHING_KEY_NOT_FOUND=1191, + ER_LOCK_OR_ACTIVE_TRANSACTION=1192, + ER_UNKNOWN_SYSTEM_VARIABLE=1193, + ER_CRASHED_ON_USAGE=1194, + ER_CRASHED_ON_REPAIR=1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK=1196, + ER_TRANS_CACHE_FULL=1197, + ER_SLAVE_MUST_STOP=1198, + ER_SLAVE_NOT_RUNNING=1199, + ER_BAD_SLAVE=1200, + ER_MASTER_INFO=1201, + ER_SLAVE_THREAD=1202, + ER_TOO_MANY_USER_CONNECTIONS=1203, + ER_SET_CONSTANTS_ONLY=1204, + ER_LOCK_WAIT_TIMEOUT=1205, + ER_LOCK_TABLE_FULL=1206, + ER_READ_ONLY_TRANSACTION=1207, + ER_DROP_DB_WITH_READ_LOCK=1208, + ER_CREATE_DB_WITH_READ_LOCK=1209, + ER_WRONG_ARGUMENTS=1210, + ER_NO_PERMISSION_TO_CREATE_USER=1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR=1212, + ER_LOCK_DEADLOCK=1213, + ER_TABLE_CANT_HANDLE_FULLTEXT=1214, + ER_CANNOT_ADD_FOREIGN=1215, + ER_NO_REFERENCED_ROW=1216, + ER_ROW_IS_REFERENCED=1217, + ER_CONNECT_TO_MASTER=1218, + ER_QUERY_ON_MASTER=1219, + ER_ERROR_WHEN_EXECUTING_COMMAND=1220, + ER_WRONG_USAGE=1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222, + ER_CANT_UPDATE_WITH_READLOCK=1223, + ER_MIXING_NOT_ALLOWED=1224, + ER_DUP_ARGUMENT=1225, + ER_USER_LIMIT_REACHED=1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR=1227, + ER_LOCAL_VARIABLE=1228, + ER_GLOBAL_VARIABLE=1229, + ER_NO_DEFAULT=1230, + ER_WRONG_VALUE_FOR_VAR=1231, + ER_WRONG_TYPE_FOR_VAR=1232, + ER_VAR_CANT_BE_READ=1233, + ER_CANT_USE_OPTION_HERE=1234, + ER_NOT_SUPPORTED_YET=1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG=1236, + ER_SLAVE_IGNORED_TABLE=1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR=1238, + CR_UNKNOWN_ERROR=1900, + CR_SOCKET_CREATE_ERROR=1901, + CR_CONNECTION_ERROR=1902, + CR_CONN_HOST_ERROR=1903, + CR_IPSOCK_ERROR =1904, + CR_UNKNOWN_HOST =1905, + CR_SERVER_GONE_ERROR=1906, + CR_VERSION_ERROR=1907, + CR_OUT_OF_MEMORY=1908, + CR_WRONG_HOST_INFO=1909, + CR_LOCALHOST_CONNECTION=1910, + CR_TCP_CONNECTION=1911, + CR_SERVER_HANDSHAKE_ERR=1912, + CR_SERVER_LOST=1913, + CR_COMMANDS_OUT_OF_SYNC=1914, + CR_NAMEDPIPE_CONNECTION=1915, + CR_NAMEDPIPEWAIT_ERROR=1916, + CR_NAMEDPIPEOPEN_ERROR=1917, + CR_NAMEDPIPESETSTATE_ERROR=1918, + CR_CANT_READ_CHARSET=1919, + CR_NET_PACKET_TOO_LARGE=1920, + CR_EMBEDDED_CONNECTION=1921, + CR_PROBE_SLAVE_STATUS=1922, + CR_PROBE_SLAVE_HOSTS=1923, + CR_PROBE_SLAVE_CONNECT=1924, + CR_PROBE_MASTER_CONNECT=1925, + CR_SSL_CONNECTION_ERROR=1926, + CR_MALFORMED_PACKET=1927, + CR_WRONG_LICENSE=1928, + }; +}; + +#endif diff --git a/src/search_local/index_storage/api/c_api_cc/examples/update/updatetest.cpp b/src/search_local/index_storage/api/c_api_cc/examples/update/updatetest.cpp new file mode 100644 index 0000000..42a487e --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/examples/update/updatetest.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include + +#include "dtcapi.h" + +int main(int argc,char* argv[]) +{ + int retCode = 0; + unsigned int uid; + unsigned int age; + std::string name; + std::string city; + std::string descr; + + DTC::Server stServer; // 只要server不析构，后台会保持长连接 + stServer.IntKey(); // 声明key类型 + stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样 + stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口 + stServer.SetTimeout(5); // 设置网络超时时间 + stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成 + + uid = atoi(argv[1]); + name = std::string(argv[2]); + city = std::string(argv[3]); + descr = std::string(argv[4]); + age = atoi(argv[5]); + DTC::UpdateRequest UpdateReq(&stServer); + retCode = UpdateReq.SetKey(uid); + if(retCode != 0) + { + printf("update-req set key error: %d", retCode); + fflush(stdout); + return(-1); + } + retCode = UpdateReq.Set("name", name.c_str()); + retCode = UpdateReq.Set("city", city.c_str()); + retCode = UpdateReq.Set("descr", descr.c_str()); + retCode = UpdateReq.Set("age", age); + if(retCode != 0) + { + printf("update-req set field error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + DTC::Result stResult; + retCode = UpdateReq.Execute(stResult); + printf("retCode:%d\n", retCode); + if(retCode == 0) + { + DTC::GetRequest getReq(&stServer); + getReq.SetKey(uid); + if(retCode == 0) + retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现 + if(retCode == 0) + retCode = getReq.Need("name"); + if(retCode == 0) + retCode = getReq.Need("city"); + if(retCode == 0) + retCode = getReq.Need("descr"); + if(retCode == 0) + retCode = getReq.Need("age"); + if(retCode != 0) + { + printf("get-req set key or need error: %d", retCode); + fflush(stdout); + return(-1); + } + + // execute & get result + retCode = getReq.Execute(stResult); + retCode = stResult.FetchRow();//开始获取数据 + printf("uid:%d\n", stResult.IntValue("uid")); + printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据 + printf("city: %s\n", stResult.StringValue("city")); + printf("descr: %s\n", stResult.BinaryValue("descr")); + printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据 + } + return 0; +} diff --git a/src/search_local/index_storage/api/c_api_cc/keylist.cc b/src/search_local/index_storage/api/c_api_cc/keylist.cc new file mode 100644 index 0000000..c18a8c5 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/keylist.cc @@ -0,0 +1,141 @@ +#include + +#include "memcheck.h" +#include "protocol.h" +#include "keylist.h" +#include "cache_error.h" + +int NCKeyValueList::KeyValueMax = 32; + +NCKeyInfo::~NCKeyInfo() +{ + for(int i = 0; i < 8; ++i) + { + if(keyName[i]) + { + free(keyName[i]); + keyName[i] = NULL; + } + } +} + +int NCKeyInfo::KeyIndex(const char *n) const +{ + namemap_t::const_iterator i = keyMap.find(n); + return i == keyMap.end() ? -1 : i->second; +} + +int NCKeyInfo::AddKey(const char* name, int type) +{ + switch(type){ + case DField::Signed: + case DField::String: + break; + default: + return -EC_BAD_KEY_TYPE; + } + if(KeyIndex(name) >= 0) { + //return -EC_DUPLICATE_FIELD; + + // ignore duplicate key field name + // because NCKeyInfo may be initialized by CheckInternalService() + // add again must be allowed for code compatibility + return 0; + } + int cnt = KeyFields(); + if(cnt >= (int)(sizeof(keyType)/sizeof(keyType[0]))) return -EC_BAD_MULTIKEY; + char* localName = (char*)malloc(strlen(name)+1); + strcpy(localName, name); + keyName[cnt] = localName; + keyMap[localName] = cnt; + keyType[cnt] = type; + keyCount++; + return 0; +} + +int NCKeyInfo::Equal(const NCKeyInfo &other) const { + int n = KeyFields(); + // key field count mis-match + if(other.KeyFields() != n) + return 0; + // key type mis-match + if(memcmp(keyType, other.keyType, n)!=0) + return 0; + for(int i=0; iKeyIndex(name); + if(col < 0 || col >= kn) + return -EC_BAD_KEY_NAME; + + switch(KeyType(col)){ + case DField::Signed: + case DField::Unsigned: + if(type != DField::Signed && type != DField::Unsigned) + return -EC_BAD_VALUE_TYPE; + break; + case DField::String: + case DField::Binary: + if(type != DField::String && type != DField::Binary) + return -EC_BAD_VALUE_TYPE; + if(v.bin.len > 255) + return -EC_KEY_OVERFLOW; + break; + default: + return -EC_BAD_KEY_TYPE; + } + + int row = fcount[col]; + if(row >= KeyValueMax) + return -EC_TOO_MANY_KEY_VALUE; // key值太多 + if(row >= kcount){ + if(REALLOC(val, (kcount+1)*kn*sizeof(DTCValue)) == NULL) + throw std::bad_alloc(); + memset(&Value(row, 0), 0, kn*sizeof(DTCValue)); + kcount++; + } + DTCValue &slot = Value(row, col); + switch(KeyType(col)){ + case DField::Signed: + case DField::Unsigned: + slot = v; + break; + + case DField::String: + case DField::Binary: + slot.bin.len = v.bin.len; + slot.bin.ptr = (char *)MALLOC(v.bin.len+1); + if(slot.bin.ptr==NULL) + throw std::bad_alloc(); + memcpy(slot.bin.ptr, v.bin.ptr, v.bin.len); + slot.bin.ptr[v.bin.len] = '\0'; + break; + } + fcount[col]++; + return 0; +} + diff --git a/src/search_local/index_storage/api/c_api_cc/keylist.h b/src/search_local/index_storage/api/c_api_cc/keylist.h new file mode 100644 index 0000000..58d991a --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/keylist.h @@ -0,0 +1,90 @@ +#ifndef __CHC_KEYLIST_H +#define __CHC_KEYLIST_H + +#include +#include + +#include + +class NCKeyInfo { + private: + struct nocase + { + bool operator()(const char * const & a, const char * const & b) const + { return strcasecmp(a, b) < 0; } + }; + typedef std::map namemap_t; + private: + namemap_t keyMap; +// const char *keyName[8]; + char *keyName[8]; + uint8_t keyType[8]; + int keyCount; + + public: + // zero is KeyField::None + void Clear(void) { + keyCount = 0; + memset(keyType, 0, sizeof(keyType)); + memset(keyName, 0, sizeof(keyName)); + keyMap.clear(); + } + NCKeyInfo() { + keyCount = 0; + memset(keyType, 0, sizeof(keyType)); + memset(keyName, 0, sizeof(keyName)); + } + NCKeyInfo(const NCKeyInfo&that) { + keyCount = that.keyCount; + memcpy(keyType, that.keyType, sizeof(keyType)); + memcpy(keyName, that.keyName, sizeof(keyName)); + for(int i=0; iKeyFields(); } + int KeyType(int id) const { return keyinfo->KeyType(id); } + int KeyCount(void) const { return kcount; } + const char * KeyName(int id) const { return keyinfo->KeyName(id); } + + void Unset(void); + int AddValue(const char *, const DTCValue &, int); + DTCValue &Value(int row, int col) { return val[row*keyinfo->KeyFields()+col]; } + const DTCValue &Value(int row, int col) const { return val[row*keyinfo->KeyFields()+col]; } + DTCValue & operator()(int row, int col) { return Value(row, col); } + const DTCValue & operator()(int row, int col) const { return Value(row, col); } + int IsFlat(void) const { + for(int i=1; iKeyFields(); i++) + if(fcount[0] != fcount[i]) + return 0; + return 1; + } +}; + +#endif + diff --git a/src/search_local/index_storage/api/c_api_cc/log_client.cc b/src/search_local/index_storage/api/c_api_cc/log_client.cc new file mode 100644 index 0000000..5ba7584 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/log_client.cc @@ -0,0 +1,11 @@ +#include "dtcapi.h" +#include + +__EXPORT +void DTC::write_log (int level, + const char*file, const char *func, int lineno, + const char *fmt, ...) +__attribute__((format(printf,5,6))) +__attribute__((__alias__("_write_log_"))) +; + diff --git a/src/search_local/index_storage/api/c_api_cc/readme.txt b/src/search_local/index_storage/api/c_api_cc/readme.txt new file mode 100644 index 0000000..2a11324 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/readme.txt @@ -0,0 +1,3 @@ +ע +1.auto_dtcc++.sh ýűҪrootȨȥִеģǽdtc sdk Ķ̬ + /usr/lib Ŀ¼£ִ ldconfig ʹЧ \ No newline at end of file diff --git a/src/search_local/index_storage/api/c_api_cc/somain.c b/src/search_local/index_storage/api/c_api_cc/somain.c new file mode 100644 index 0000000..97212c6 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/somain.c @@ -0,0 +1,33 @@ +#undef _GNU_SOURCE +#define _GNU_SOURCE +#include +#include +#include +#include + +const char __invoke_dynamic_linker__[] +__attribute__ ((section (".interp"))) +__HIDDEN += +#if __x86_64__ + "/lib64/ld-linux-x86-64.so.2" +#else + "/lib/ld-linux.so.2" +#endif + ; + +__HIDDEN +void _so_start(char *arg1,...) { +#define BANNER "DTC client API v" DTC_VERSION_DETAIL "\n" \ + " - TCP connection supported\n" \ + " - UDP connection supported\n" \ + " - UNIX stream connection supported\n" \ + " - embeded threading connection supported\n" \ + " - protocol packet encode/decode interface supported\n" \ + " - async execute (except embeded threading) supported \n" + + int unused; + unused = write(1, BANNER, sizeof(BANNER)-1); + _exit(0); +} + diff --git a/src/search_local/index_storage/api/c_api_cc/udppool.cc b/src/search_local/index_storage/api/c_api_cc/udppool.cc new file mode 100644 index 0000000..c4dda92 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/udppool.cc @@ -0,0 +1,240 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "udppool.h" +#include "log.h" + +static unsigned int bindip = 0xFFFFFFFF; + +unsigned int GetBindIp(void) +{ + const char *name = getenv("DTCAPI_UDP_INTERFACE"); + + if (name == NULL || name[0] == 0 || strcmp(name, "*") == 0) + { + return 0; + } + + int fd = socket(AF_INET, SOCK_DGRAM, 0); + struct ifconf ifc; + struct ifreq *ifr = NULL; + int n = 0; + int i; + + if (fd < 0) + return 0; + + ifc.ifc_len = 0; + ifc.ifc_req = NULL; + + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + { + ifr = (struct ifreq *) alloca(ifc.ifc_len > 128 ? ifc.ifc_len : 128); + ifc.ifc_req = ifr; + + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + n = ifc.ifc_len / sizeof(struct ifreq); + } + + close(fd); + + for (i = 0; i < n; i++) + { + if (strncmp(ifr[i].ifr_name, name, sizeof(ifr[i].ifr_name)) != 0) + continue; + + if (ifr[i].ifr_addr.sa_family == AF_INET) + return ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr; + } + + return 0; +} + +static int CreatePortIpv4(void) +{ + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (bindip == 0xFFFFFFFF) + bindip = GetBindIp(); + + if (bindip != 0) + { + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = bindip; + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) + { + close(fd); + return -1; + } + } + + return fd; +} + +static int CreatePortIpv6(void) +{ + int fd = socket(AF_INET6, SOCK_DGRAM, 0); + + return fd; +} + +static int CreatePortUnix(void) +{ + int fd = socket(AF_UNIX, SOCK_DGRAM, 0); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), + "@dtcapi-global-%d-%d-%d", getpid(), fd, (int)time(NULL)); + socklen_t len = SUN_LEN(&addr); + addr.sun_path[0] = 0; + if (bind(fd, (const struct sockaddr *)&addr, len) < 0) + { + close(fd); + return -1; + } + + return fd; +} + + +struct NCUdpPortList { +public: + int stopped; + int family; + int (*newport)(void); + pthread_mutex_t lock; + NCUdpPort *list; +public: + NCUdpPort *Get(void); + void Put(NCUdpPort *); +public: + ~NCUdpPortList(void); +}; + +// this destructor only called when unloading libdtc.so +NCUdpPortList::~NCUdpPortList(void) +{ + if (pthread_mutex_lock(&lock) == 0) + { + stopped = 1; + while (list != NULL) + { + NCUdpPort *port = list; + list = port->next; + delete port; + } + pthread_mutex_unlock(&lock); + } +} + +NCUdpPort *NCUdpPortList::Get(void) +{ + NCUdpPort *port = NULL; + + if (pthread_mutex_lock(&lock) == 0) + { + if (list != NULL) + { + port = list; + list = port->next; + } + pthread_mutex_unlock(&lock); + } + else + { + log_error("api mutex_lock error,may have fd leak"); + } + + if (port != NULL) + { + if (getpid() == port->pid) + { + port->sn++; + } + else + { + delete port; + port = NULL; + } + } + + if (port == NULL) + { + int fd = newport(); + + if (fd > 0) + { + port = new NCUdpPort; + port->fd = fd; + unsigned int seed = fd + (long)port + (long) & port + (long)pthread_self() + (long)port; + port->sn = rand_r(&seed); + port->pid = getpid(); + port->timeout = -1; + port->family = family; + } + } + + return port; +} + +void NCUdpPortList::Put(NCUdpPort *port) +{ + if (this != NULL && pthread_mutex_lock(&lock) == 0) + { + if(stopped) { + // always delete port after unloading process + port->Eat(); + } else { + port->next = list; + list = port; + } + pthread_mutex_unlock(&lock); + } + else + { + port->Eat(); + } +} + +static NCUdpPortList ipv4List = { 0, AF_INET, CreatePortIpv4, PTHREAD_MUTEX_INITIALIZER, NULL }; +static NCUdpPortList ipv6List = { 0, AF_INET6, CreatePortIpv6, PTHREAD_MUTEX_INITIALIZER, NULL }; +static NCUdpPortList unixList = { 0, AF_UNIX, CreatePortUnix, PTHREAD_MUTEX_INITIALIZER, NULL }; + +struct NCUdpPortList *GetPortList(int family) { + switch(family) { + case AF_INET: + return &ipv4List; + case AF_INET6: + return &ipv6List; + case AF_UNIX: + return &unixList; + } + return NULL; +} + +NCUdpPort *NCUdpPort::Get(int family) +{ + NCUdpPortList *portList = GetPortList(family); + if(portList == NULL) { + return NULL; + } + + return portList->Get(); +} + +void NCUdpPort::Put() +{ + NCUdpPortList *portList = GetPortList(family); + portList->Put(this); +} diff --git a/src/search_local/index_storage/api/c_api_cc/udppool.h b/src/search_local/index_storage/api/c_api_cc/udppool.h new file mode 100644 index 0000000..cd95638 --- /dev/null +++ b/src/search_local/index_storage/api/c_api_cc/udppool.h @@ -0,0 +1,62 @@ +#include + +class NCPort +{ +public: + uint64_t sn; + int fd; + int timeout;//the timeout this socket is. + +public: + NCPort() + { + sn = 0; + fd = -1; + timeout = -1; + } + + NCPort(const NCPort &that) + { + sn = that.sn; + fd = that.fd; + timeout = that.timeout; + } + + ~NCPort() + { + if (fd >= 0) + { + close(fd); + } + } +}; + +class NCUdpPortList; +class NCUdpPort: public NCPort +{ +public: + int family; + pid_t pid; + +private: + NCUdpPort *next; + +private: + NCUdpPort() + { + pid = -1; + }; + ~NCUdpPort() + { + }; + +public: + friend class NCUdpPortList; + static NCUdpPort *Get(int family); // get from cache + void Put(); // put back cache + void Eat() + { + delete this; + } // eat and delete +}; + diff --git a/src/search_local/index_storage/common/Condition.h b/src/search_local/index_storage/common/Condition.h new file mode 100644 index 0000000..f93d6fe --- /dev/null +++ b/src/search_local/index_storage/common/Condition.h @@ -0,0 +1,101 @@ +/* + * ===================================================================================== + * + * Filename: condition.h + * + * Description: conditional thread operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_CONDITION_H__ +#define __DTC_CONDITION_H__ + +#include "non_copyable.h" + +class condition : private noncopyable +{ +public: + condition(void) + { + pthread_mutex_init(&m_lock, NULL); + pthread_cond_init(&m_cond, NULL); + } + + ~condition(void) + { + pthread_mutex_destroy(&m_lock); + pthread_cond_destroy(&m_cond); + } + + void notify_one(void) + { + pthread_cond_signal(&m_cond); + } + void notify_all(void) + { + pthread_cond_broadcast(&m_cond); + } + + void wait(void) + { + pthread_cond_wait(&m_cond, &m_lock); + } + + void lock(void) + { + pthread_mutex_lock(&m_lock); + } + + void unlock(void) + { + pthread_mutex_unlock(&m_lock); + } + +private: + pthread_cond_t m_cond; + pthread_mutex_t m_lock; +}; + +class copedEnterCritical +{ +public: + copedEnterCritical(condition &c) : m_cond(c) + { + m_cond.lock(); + } + + ~copedEnterCritical(void) + { + m_cond.unlock(); + } + +private: + condition &m_cond; +}; + +class copedLeaveCritical +{ +public: + copedLeaveCritical(condition &c) : m_cond(c) + { + m_cond.unlock(); + } + + ~copedLeaveCritical(void) + { + m_cond.lock(); + } + +private: + condition &m_cond; +}; + +#endif //__DTC_CONDITION_H__ diff --git a/src/search_local/index_storage/common/Makefile b/src/search_local/index_storage/common/Makefile new file mode 100644 index 0000000..5c12b2d --- /dev/null +++ b/src/search_local/index_storage/common/Makefile @@ -0,0 +1,42 @@ +include ../Make.conf + +target = libcommon.a libcommon_nopic.a +CFLAGS+=-I../api/c_api -I. -I../watchdog -I../stat -I../../../3rdlib/attr_api $(ZINC) +LIBS = $(Z_LIB) +VPATH = ../watchdog +#SVN_VERSION = $(shell test -d .svn && (svn info ../ | grep "Last Changed Rev" | cut -d " " -f 4)) +GIT_VERSION=$(shell git log | head -n 4 | grep "commit" | cut -d " " -f 2 | cut -c 1-7) +ifeq "$(GIT_VERSION)a" "a" + GIT_REVISION = "(unknown)" +endif +$(shell sed -i '11s/"[^"]*"/"$(GIT_VERSION)"/' version.h) + +filelist := md5 value decode encode field section table_def \ + packet_base packet_server packet_result \ + task_const task_base task_pkey task_server task_copy task_request task_api \ + poller timer_list config version system_lock log log_stat log_alert \ + shmem mem_check dbconfig dbconfig_tdef admin_tdef \ + daemon daemon_wait proc_title thread poll_thread poll_thread_group thread_cpu_stat \ + buffer localip \ + key_guard bitsop net_addr net_bind blacklist blacklist_unit \ + multi_request task_multiplexer \ + helper_client helper_collect helper_group \ + listener listener_pool decoder_base \ + client_sync client_async client_unit client_dgram \ + plugin_request plugin_mgr plugin_dgram plugin_sync \ + plugin_worker plugin_unit plugin_listener_pool \ + plugin_global plugin_decoder plugin_timer dtc_global\ + compress client_resource_pool \ + new_hash dynamic_helper_collection consistent_hash_selector chash\ + file_backed_key_set markup_stl parse_cluster_config key_route \ + proxy_client proxy_client_unit proxy_listener proxy_listen_pool proxy_process proxy_receiver proxy_sender \ + proxy_multi_request plugin_proxy_mgr plugin_proxy_listener_pool watchdog_fork hotback_task table_def_manager + +libcommon_objs := $(filelist:%=%_pic.o) +%_pic.o: CFLAGS += -pthread -fPIC + +libcommon_nopic_objs := $(filelist:%=%.o) +%.o: CFLAGS += -pthread + +include ../Make.rules + diff --git a/src/search_local/index_storage/common/admin_tdef.cc b/src/search_local/index_storage/common/admin_tdef.cc new file mode 100644 index 0000000..61e26e1 --- /dev/null +++ b/src/search_local/index_storage/common/admin_tdef.cc @@ -0,0 +1,65 @@ +/* + * ===================================================================================== + * + * Filename: admin_tdef.cc + * + * Description: table definition. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "log.h" +#include "admin_tdef.h" +#include "table_def.h" +#include "table_def_manager.h" +#include "protocol.h" +#include "dbconfig.h" + +static struct FieldConfig HBTabField[] = { + {"type", DField::Unsigned, 4, DTCValue::Make(0), 0}, + {"flag", DField::Unsigned, 1, DTCValue::Make(0), 0}, + {"key", DField::Binary, 255, DTCValue::Make(0), 0}, + {"value", DField::Binary, MAXPACKETSIZE, DTCValue::Make(0), 0}, +}; + +DTCTableDefinition *build_hot_backup_table(void) +{ + DTCTableDefinition *tdef = new DTCTableDefinition(4); + tdef->set_table_name("@HOT_BACKUP"); + struct FieldConfig *field = HBTabField; + int field_cnt = sizeof(HBTabField) / sizeof(HBTabField[0]); + + // build hotback table key info base on the actual user definite table + field[0].type = TableDefinitionManager::Instance()->get_cur_table_def()->key_type(); + field[0].size = TableDefinitionManager::Instance()->get_cur_table_def()->field_size(0); + + tdef->set_admin_table(); + for (int i = 0; i < field_cnt; i++) + { + if (tdef->add_field(i, field[i].name, field[i].type, field[i].size) != 0) + { + log_error("add_field failed, name=%s, size=%d, type=%d", + field[i].name, field[i].size, field[i].type); + delete tdef; + return NULL; + } + tdef->set_default_value(i, &field[i].dval); + } + + if (tdef->set_key_fields(1) < 0) + { + log_error("Table key size %d too large, must <= 255", + tdef->key_format() > 0 ? tdef->key_format() : -tdef->key_format()); + delete tdef; + return NULL; + } + tdef->build_info_cache(); + return tdef; +} diff --git a/src/search_local/index_storage/common/admin_tdef.h b/src/search_local/index_storage/common/admin_tdef.h new file mode 100644 index 0000000..1b7f626 --- /dev/null +++ b/src/search_local/index_storage/common/admin_tdef.h @@ -0,0 +1,72 @@ +/* + * ===================================================================================== + * + * Filename: admin_tdef.h + * + * Description: table definition. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ADMIN_TDEF_H__ +#define __ADMIN_TDEF_H__ + +class DTCTableDefinition; +class DTCHotBackup +{ +public: + //type + enum + { + SYNC_LRU = 1, + SYNC_INSERT = 2, + SYNC_UPDATE = 4, + SYNC_PURGE = 8, + SYNC_DELETE = 16, + SYNC_CLEAR = 32, + SYNC_COLEXPAND = 64, + SYNC_COLEXPAND_CMD = 128 + }; + + //flag + enum + { + NON_VALUE = 1, + HAS_VALUE = 2, + EMPTY_NODE = 4, + KEY_NOEXIST = 8, + }; +}; + +class DTCMigrate +{ +public: + //type + enum + { + FROM_CLIENT = 1, + FROM_SERVER = 2 + }; + + //flag + enum + { + NON_VALUE = 1, + HAS_VALUE = 2, + EMPTY_NODE = 4, + KEY_NOEXIST = 8, + }; +}; +extern DTCTableDefinition *build_hot_backup_table(void); + +#define _DTC_HB_COL_EXPAND_ "_dtc_hb_col_expand_" +#define _DTC_HB_COL_EXPAND_DONE_ "_dtc_hb_col_expand_done_" + +#endif diff --git a/src/search_local/index_storage/common/atomic.h b/src/search_local/index_storage/common/atomic.h new file mode 100644 index 0000000..9beb49c --- /dev/null +++ b/src/search_local/index_storage/common/atomic.h @@ -0,0 +1,37 @@ +/* + * ===================================================================================== + * + * Filename: atomic.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ATOMIC__T +#define __ATOMIC__T + +#include + +#if __GNUC__ < 4 +#include "atomic_asm.h" +#else +#include "atomic_gcc.h" +#endif + +#if __WORDSIZE == 64 || __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) +#define HAS_ATOMIC8 1 +#include "atomic_gcc8.h" +#else +#define HAS_ATOMIC8 1 +#include "atomic_asm8.h" +#endif + +#endif diff --git a/src/search_local/index_storage/common/atomic.hpp b/src/search_local/index_storage/common/atomic.hpp new file mode 100644 index 0000000..7f03df6 --- /dev/null +++ b/src/search_local/index_storage/common/atomic.hpp @@ -0,0 +1,97 @@ +#include "atomic.h" + +struct AtomicU32 +{ +private: + typedef uint32_t V; + atomic_t count; + +public: + ~AtomicU32(void) {} + AtomicU32(V v = 0) { set(v); } + + inline V get(void) const { return atomic_read((atomic_t *)&count); } + inline V set(V v) + { + atomic_set(&count, v); + return v; + } + inline V add(V v) { return atomic_add_return(v, &count); } + inline V sub(V v) { return atomic_sub_return(v, &count); } + inline V clear(void) { return atomic_clear(&count); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; + +struct AtomicS32 +{ +private: + typedef int32_t V; + atomic_t count; + +public: + ~AtomicS32(void) {} + AtomicS32(V v = 0) { set(v); } + + inline V get(void) const { return atomic_read((atomic_t *)&count); } + inline V set(V v) + { + atomic_set(&count, v); + return v; + } + inline V add(V v) { return atomic_add_return(v, &count); } + inline V sub(V v) { return atomic_sub_return(v, &count); } + inline V clear(void) { return atomic_clear(&count); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; + +#if HAS_ATOMIC8 +struct AtomicS64 +{ +private: + typedef int64_t V; + atomic8_t count; + +public: + ~AtomicS64(void) {} + AtomicS64(V v = 0) { set(v); } + + inline V get(void) const { return atomic8_read((atomic8_t *)&count); } + inline V set(V v) + { + atomic8_set(&count, v); + return v; + } + inline V add(V v) { return atomic8_add_return(v, &count); } + inline V sub(V v) { return atomic8_sub_return(v, &count); } + inline V clear(void) { return atomic8_clear(&count); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; + +#endif diff --git a/src/search_local/index_storage/common/atomic_asm.h b/src/search_local/index_storage/common/atomic_asm.h new file mode 100644 index 0000000..11f08d1 --- /dev/null +++ b/src/search_local/index_storage/common/atomic_asm.h @@ -0,0 +1,247 @@ +/* + * ===================================================================================== + * + * Filename: atomic_asm.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ARCH_I386_ATOMIC__ +#define __ARCH_I386_ATOMIC__ + +#include +__BEGIN_DECLS + +/* xaddl: require Modern 486+ processor */ + +/* + * Make sure gcc doesn't try to be clever and move things around + * on us. We need to use _exactly_ the address the user gave us, + * not some alias that contains the same information. + */ +typedef struct +{ + volatile int counter; +} atomic_t; + +#define ATOMIC_INIT(i) \ + { \ + (i) \ + } + +/** + * atomic_read - read atomic variable + * @v: pointer of type atomic_t + * + * Atomically reads the value of @v. + */ +#define atomic_read(v) ((v)->counter) + +/** + * atomic_set - set atomic variable + * @v: pointer of type atomic_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +#define atomic_set(v, i) (((v)->counter) = (i)) + +static __inline__ int atomic_clear(atomic_t *v) +{ + int i; + __asm__ __volatile__( + " xorl %0, %0\n" + "lock; xchgl %0, %1;" + : "=r"(i) + : "m"(v->counter)); + return i; +} + +/** + * atomic_add - add integer to atomic variable + * @i: integer value to add + * @v: pointer of type atomic_t + * + * Atomically adds @i to @v. + */ +static __inline__ int atomic_add(int i, atomic_t *v) +{ + int __i = i; + __asm__ __volatile__( + "lock; xaddl %0, %1;" + : "=r"(i) + : "m"(v->counter), "0"(i)); + return __i; +} + +/** + * atomic_sub - subtract the atomic variable + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically subtracts @i from @v. + */ +static __inline__ int atomic_sub(int i, atomic_t *v) +{ + return atomic_add(-i, v); +} + +/** + * atomic_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic_sub_and_test(int i, atomic_t *v) +{ + unsigned char c; + + __asm__ __volatile__( + "lock; subl %2,%0; sete %1" + : "=m"(v->counter), "=qm"(c) + : "ir"(i), "m"(v->counter) + : "memory"); + return c; +} + +/** + * atomic_inc - increment atomic variable + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1. + */ +static __inline__ void atomic_inc(atomic_t *v) +{ + __asm__ __volatile__( + "lock; incl %0" + : "=m"(v->counter) + : "m"(v->counter)); +} + +/** + * atomic_dec - decrement atomic variable + * @v: pointer of type atomic_t + * + * Atomically decrements @v by 1. + */ +static __inline__ void atomic_dec(atomic_t *v) +{ + __asm__ __volatile__( + "lock; decl %0" + : "=m"(v->counter) + : "m"(v->counter)); +} + +/** + * atomic_dec_and_test - decrement and test + * @v: pointer of type atomic_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __inline__ int atomic_dec_and_test(atomic_t *v) +{ + unsigned char c; + + __asm__ __volatile__( + "lock; decl %0; sete %1" + : "=m"(v->counter), "=qm"(c) + : "m"(v->counter) + : "memory"); + return c != 0; +} + +/** + * atomic_inc_and_test - increment and test + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic_inc_and_test(atomic_t *v) +{ + unsigned char c; + + __asm__ __volatile__( + "lock; incl %0; sete %1" + : "=m"(v->counter), "=qm"(c) + : "m"(v->counter) + : "memory"); + return c != 0; +} + +/** + * atomic_add_negative - add and test if negative + * @v: pointer of type atomic_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __inline__ int atomic_add_negative(int i, atomic_t *v) +{ + unsigned char c; + + __asm__ __volatile__( + "lock; addl %2,%0; sets %1" + : "=m"(v->counter), "=qm"(c) + : "ir"(i), "m"(v->counter) + : "memory"); + return c; +} + +/** + * atomic_add_return - add and return + * @v: pointer of type atomic_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns @i + @v + */ +static __inline__ int atomic_add_return(int i, atomic_t *v) +{ + int __i = i; + __asm__ __volatile__( + "lock; xaddl %0, %1;" + : "=r"(i) + : "m"(v->counter), "0"(i)); + return i + __i; +} + +static __inline__ int atomic_sub_return(int i, atomic_t *v) +{ + return atomic_add_return(-i, v); +} + +#define atomic_inc_return(v) (atomic_add_return(1, v)) +#define atomic_dec_return(v) (atomic_sub_return(1, v)) + +/* These are x86-specific, used by some header files */ +#define atomic_clear_mask(mask, addr) \ + __asm__ __volatile__("lock; andl %0,%1" \ + : \ + : "r"(~(mask)), "m"(*addr) \ + : "memory") + +#define atomic_set_mask(mask, addr) \ + __asm__ __volatile__("lock; orl %0,%1" \ + : \ + : "r"(mask), "m"(*(addr)) \ + : "memory") + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/atomic_asm8.h b/src/search_local/index_storage/common/atomic_asm8.h new file mode 100644 index 0000000..36cf73b --- /dev/null +++ b/src/search_local/index_storage/common/atomic_asm8.h @@ -0,0 +1,297 @@ +/* + * ===================================================================================== + * + * Filename: atomic_asm8.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ARCH_I386_ATOMIC8__ +#define __ARCH_I386_ATOMIC8__ + +#include +__BEGIN_DECLS + +/* + * cmpxchg8b: require Pentium or above + * compare edx:eax with target memory, + * if equal, write ecx:ebx to memory + * if non-equal, keep old value in edx:eax and write back + */ + +/* + * Make sure gcc doesn't try to be clever and move things around + * on us. We need to use _exactly_ the address the user gave us, + * not some alias that contains the same information. + */ +typedef struct +{ + volatile long long counter; +} atomic8_t; + +#define ATOMIC8_INIT(i) \ + { \ + (i) \ + } + +/** + * atomic8_read - read atomic variable + * @v: pointer of type atomic8_t + * + * Atomically reads the value of @v. + */ +static __inline__ long long atomic8_read(atomic8_t *v) +{ + register long long r; + __asm__ __volatile__( + /* original value for ebx, ecx doesn't matter */ + " movl %%ecx, %%edx\n" + " movl %%ebx, %%eax\n" + "lock; cmpxchg8b %1\n" + : "=&A"(r), "+o"(v->counter) + : + : "memory", "cc"); + return r; +} + +/** + * atomic8_set - set atomic variable + * @v: pointer of type atomic8_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +static __inline__ long long atomic8_return_set(atomic8_t *v, long long i) +{ + register long long r = v->counter; +#if __pic__ || __PIC__ + __asm__ __volatile__( + " pushl %%ebx\n" + " movl %2, %%ebx\n" + "1: repz; nop\n" // aka pause + "lock; cmpxchg8b (%1)\n" + " jnz 1b\n" + " popl %%ebx\n" + : "+A"(r) + : "ir"(v), "ir"((int)i), "c"((int)((unsigned long long)i >> 32)) + : "memory", "cc"); +#else + __asm__ __volatile__( + "1: repz; nop\n" // aka pause + "lock; cmpxchg8b %1\n" + " jnz 1b\n" + : "+A"(r), "+o"(v->counter) + : "b"((int)i), "c"((int)((unsigned long long)i >> 32)) + : "memory", "cc"); +#endif + return r; +} + +static __inline__ void atomic8_set(atomic8_t *v, long long i) +{ + atomic8_return_set(v, i); +} + +/** + * atomic8_set - set atomic variable + * @v: pointer of type atomic8_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +static __inline__ long long atomic8_clear(atomic8_t *v) +{ + return atomic8_return_set(v, 0); +} + +/** + * atomic8_add - add integer to atomic variable + * @i: integer value to add + * @v: pointer of type atomic8_t + * + * Atomically adds @i to @v. + */ +static __inline__ long long atomic8_add(long long i, atomic8_t *v) +{ + register long long r = v->counter; +#if __pic__ || __PIC__ + __asm__ __volatile__( + " pushl %%ebx\n" + "1: movl %2, %%ebx\n" + " movl %3, %%ecx\n" + " addl %%eax, %%ebx\n" + " adcl %%edx, %%ecx\n" + "repz; nop\n" + "lock; cmpxchg8b (%1)\n" + " jnz 1b\n" + " popl %%ebx\n" + : "+A"(r) + : "ir"(v), "ir"((int)i), "ir"((int)((unsigned long long)i >> 32)) + : "memory", "ecx", "cc"); +#else + __asm__ __volatile__( + "1: movl %2, %%ebx\n" + " movl %3, %%ecx\n" + " addl %%eax, %%ebx\n" + " adcl %%edx, %%ecx\n" + "repz; nop\n" + "lock; cmpxchg8b %1\n" + " jnz 1b\n" + : "+A"(r), "+o"(v->counter) + : "g"((int)i), "g"((int)((unsigned long long)i >> 32)) + : "memory", "ebx", "ecx", "cc"); +#endif + return r; +} + +/** + * atomic8_add_return - add and return + * @v: pointer of type atomic8_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns @i + @v + */ +static __inline__ long long atomic8_add_return(long long i, atomic8_t *v) +{ + register long long r = v->counter; +#if __pic__ || __PIC__ + __asm__ __volatile__( + " pushl %%ebx\n" + "1: movl %2, %%ebx\n" + " movl %3, %%ecx\n" + " addl %%eax, %%ebx\n" + " adcl %%edx, %%ecx\n" + "repz; nop\n" + "lock; cmpxchg8b (%1)\n" + " jnz 1b\n" + " movl %%ebx, %%eax\n" + " movl %%ecx, %%edx\n" + " popl %%ebx\n" + : "+A"(r) + : "ir"(v), "ir"((int)i), "ir"((int)((unsigned long long)i >> 32)) + : "memory", "ecx", "cc"); +#else + __asm__ __volatile__( + "1: movl %2, %%ebx\n" + " movl %3, %%ecx\n" + " addl %%eax, %%ebx\n" + " adcl %%edx, %%ecx\n" + "repz; nop\n" + "lock; cmpxchg8b %1\n" + " jnz 1b\n" + " movl %%ebx, %%eax\n" + " movl %%ecx, %%edx\n" + : "+A"(r), "+o"(v->counter) + : "g"((int)i), "g"((int)((unsigned long long)i >> 32)) + : "memory", "ebx", "ecx", "cc"); +#endif + return r; +} + +/** + * atomic8_sub - subtract the atomic variable + * @i: integer value to subtract + * @v: pointer of type atomic8_t + * + * Atomically subtracts @i from @v. + */ +static __inline__ long long atomic8_sub(long long i, atomic8_t *v) +{ + return atomic8_add(-i, v); +} + +/** + * atomic8_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic8_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic8_sub_and_test(long long i, atomic8_t *v) +{ + return atomic8_add_return(-i, v) == 0; +} + +/** + * atomic8_inc - increment atomic variable + * @v: pointer of type atomic8_t + * + * Atomically increments @v by 1. + */ +static __inline__ long long atomic8_inc(atomic8_t *v) +{ + return atomic8_add(1, v); +} + +/** + * atomic8_dec - decrement atomic variable + * @v: pointer of type atomic8_t + * + * Atomically decrements @v by 1. + */ +static __inline__ long long atomic8_dec(atomic8_t *v) +{ + return atomic8_add(-1, v); +} + +/** + * atomic8_dec_and_test - decrement and test + * @v: pointer of type atomic8_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __inline__ int atomic8_dec_and_test(atomic8_t *v) +{ + return atomic8_add_return(-1, v) == 0; +} + +/** + * atomic8_inc_and_test - increment and test + * @v: pointer of type atomic8_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic8_inc_and_test(atomic8_t *v) +{ + return atomic8_add_return(1, v) == 0; +} + +/** + * atomic8_add_negative - add and test if negative + * @v: pointer of type atomic8_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __inline__ int atomic8_add_negative(long long i, atomic8_t *v) +{ + return atomic8_add_return(i, v) < 0; +} + +static __inline__ long long atomic8_sub_return(long long i, atomic8_t *v) +{ + return atomic8_add_return(-i, v); +} + +#define atomic8_inc_return(v) (atomic8_add_return(1, v)) +#define atomic8_dec_return(v) (atomic8_sub_return(1, v)) + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/atomic_gcc.h b/src/search_local/index_storage/common/atomic_gcc.h new file mode 100644 index 0000000..f8316f5 --- /dev/null +++ b/src/search_local/index_storage/common/atomic_gcc.h @@ -0,0 +1,179 @@ +/* + * ===================================================================================== + * + * Filename: atomic_gcc.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ARCH_I386_ATOMIC__ +#define __ARCH_I386_ATOMIC__ + +#include +__BEGIN_DECLS + +/* + * Make sure gcc doesn't try to be clever and move things around + * on us. We need to use _exactly_ the address the user gave us, + * not some alias that contains the same information. + */ +typedef volatile int atomic_t; + +#define ATOMIC_INIT(i) \ + { \ + (i) \ + } + +/** + * atomic_read - read atomic variable + * @v: pointer of type atomic_t + * + * Atomically reads the value of @v. + */ +#define atomic_read(v) (*(v)) + +/** + * atomic_set - set atomic variable + * @v: pointer of type atomic_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +#define atomic_set(v, i) (*(v) = (i)) + +static __inline__ int atomic_clear(atomic_t *v) +{ + return __sync_fetch_and_and(v, 0); +} + +/** + * atomic_add - add integer to atomic variable + * @i: integer value to add + * @v: pointer of type atomic_t + * + * Atomically adds @i to @v. + */ +static __inline__ int atomic_add(int i, atomic_t *v) +{ + return __sync_fetch_and_add(v, i); +} + +/** + * atomic_sub - subtract the atomic variable + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically subtracts @i from @v. + */ +static __inline__ int atomic_sub(int i, atomic_t *v) +{ + return __sync_fetch_and_sub(v, i); +} + +/** + * atomic_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic_sub_and_test(int i, atomic_t *v) +{ + return __sync_sub_and_fetch(v, i) == 0; +} + +/** + * atomic_inc - increment atomic variable + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1. + */ +static __inline__ int atomic_inc(atomic_t *v) +{ + return __sync_fetch_and_add(v, 1); +} + +/** + * atomic_dec - decrement atomic variable + * @v: pointer of type atomic_t + * + * Atomically decrements @v by 1. + */ +static __inline__ int atomic_dec(atomic_t *v) +{ + return __sync_fetch_and_sub(v, 1); +} + +/** + * atomic_dec_and_test - decrement and test + * @v: pointer of type atomic_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __inline__ int atomic_dec_and_test(atomic_t *v) +{ + return __sync_sub_and_fetch(v, 1) == 0; +} + +/** + * atomic_inc_and_test - increment and test + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic_inc_and_test(atomic_t *v) +{ + return __sync_add_and_fetch(v, 1) == 0; +} + +/** + * atomic_add_negative - add and test if negative + * @v: pointer of type atomic_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __inline__ int atomic_add_negative(int i, atomic_t *v) +{ + return __sync_add_and_fetch(v, i) < 0; +} + +/** + * atomic_add_return - add and return + * @v: pointer of type atomic_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns @i + @v + */ +static __inline__ int atomic_add_return(int i, atomic_t *v) +{ + return __sync_add_and_fetch(v, i); +} + +static __inline__ int atomic_sub_return(int i, atomic_t *v) +{ + return __sync_sub_and_fetch(v, i); +} + +#define atomic_inc_return(v) (atomic_add_return(1, v)) +#define atomic_dec_return(v) (atomic_sub_return(1, v)) + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/atomic_gcc8.h b/src/search_local/index_storage/common/atomic_gcc8.h new file mode 100644 index 0000000..877a63f --- /dev/null +++ b/src/search_local/index_storage/common/atomic_gcc8.h @@ -0,0 +1,186 @@ +/* + * ===================================================================================== + * + * Filename: atomic_gcc8.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ARCH_I386_ATOMIC8__ +#define __ARCH_I386_ATOMIC8__ + +#include +__BEGIN_DECLS + +/* + * Make sure gcc doesn't try to be clever and move things around + * on us. We need to use _exactly_ the address the user gave us, + * not some alias that contains the same information. + */ +typedef volatile int64_t atomic8_t; + +#define ATOMIC_INIT(i) \ + { \ + (i) \ + } + +/** + * atomic8_read - read atomic8 variable + * @v: pointer of type atomic8_t + * + * Atomically reads the value of @v. + */ +#if __WORDSIZE == 64 +#define atomic8_read(v) (*(v)) +#else +#define atomic8_read(v) atomic8_add_return(0, v) +#endif + +/** + * atomic8_set - set atomic8 variable + * @v: pointer of type atomic8_t + * @i: required value + * + * Atomically sets the value of @v to @i. + */ +#if __WORDSIZE == 64 +#define atomic8_set(v, i) (*(v) = (i)) +#else +#define atomic8_set(v, i) (*(v) = (i)) +#endif + +static __inline__ int64_t atomic8_clear(atomic8_t *v) +{ + return __sync_fetch_and_and(v, 0); +} +/** + * atomic8_add - add integer to atomic8 variable + * @i: integer value to add + * @v: pointer of type atomic8_t + * + * Atomically adds @i to @v. + */ +static __inline__ int64_t atomic8_add(int64_t i, atomic8_t *v) +{ + return __sync_fetch_and_add(v, i); +} + +/** + * atomic8_sub - subtract the atomic8 variable + * @i: integer value to subtract + * @v: pointer of type atomic8_t + * + * Atomically subtracts @i from @v. + */ +static __inline__ int64_t atomic8_sub(int64_t i, atomic8_t *v) +{ + return __sync_fetch_and_sub(v, i); +} + +/** + * atomic8_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic8_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic8_sub_and_test(int64_t i, atomic8_t *v) +{ + return __sync_sub_and_fetch(v, i) == 0; +} + +/** + * atomic8_inc - increment atomic8 variable + * @v: pointer of type atomic8_t + * + * Atomically increments @v by 1. + */ +static __inline__ int64_t atomic8_inc(atomic8_t *v) +{ + return __sync_fetch_and_add(v, 1); +} + +/** + * atomic8_dec - decrement atomic8 variable + * @v: pointer of type atomic8_t + * + * Atomically decrements @v by 1. + */ +static __inline__ int64_t atomic8_dec(atomic8_t *v) +{ + return __sync_fetch_and_sub(v, 1); +} + +/** + * atomic8_dec_and_test - decrement and test + * @v: pointer of type atomic8_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __inline__ int atomic8_dec_and_test(atomic8_t *v) +{ + return __sync_sub_and_fetch(v, 1) == 0; +} + +/** + * atomic8_inc_and_test - increment and test + * @v: pointer of type atomic8_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __inline__ int atomic8_inc_and_test(atomic8_t *v) +{ + return __sync_add_and_fetch(v, 1) == 0; +} + +/** + * atomic8_add_negative - add and test if negative + * @v: pointer of type atomic8_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __inline__ int atomic8_add_negative(int64_t i, atomic8_t *v) +{ + return __sync_add_and_fetch(v, i) < 0; +} + +/** + * atomic8_add_return - add and return + * @v: pointer of type atomic8_t + * @i: integer value to add + * + * Atomically adds @i to @v and returns @i + @v + */ +static __inline__ int64_t atomic8_add_return(int64_t i, atomic8_t *v) +{ + return __sync_add_and_fetch(v, i); +} + +static __inline__ int64_t atomic8_sub_return(int64_t i, atomic8_t *v) +{ + return __sync_sub_and_fetch(v, i); +} + +#define atomic8_inc_return(v) (atomic8_add_return(1, v)) +#define atomic8_dec_return(v) (atomic8_sub_return(1, v)) + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/bitsop.c b/src/search_local/index_storage/common/bitsop.c new file mode 100644 index 0000000..5e8d3ae --- /dev/null +++ b/src/search_local/index_storage/common/bitsop.c @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: bitsop.cc + * + * Description: bits op functions + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include "bitsop.h" + +const unsigned char __bitcount[256] = { + /* 00 */ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + /* 10 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + /* 20 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + /* 30 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* 40 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + /* 50 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* 60 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* 70 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + /* 80 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + /* 90 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* a0 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* b0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + /* c0 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + /* d0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + /* e0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + /* f0 */ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + +int CountBits(const char *buf, int sz) +{ + int c; + for (c = 0; sz > 0; buf++, sz--) + { + c += __bitcount[*(unsigned char *)buf]; + } + return c; +} diff --git a/src/search_local/index_storage/common/bitsop.h b/src/search_local/index_storage/common/bitsop.h new file mode 100644 index 0000000..dd2d516 --- /dev/null +++ b/src/search_local/index_storage/common/bitsop.h @@ -0,0 +1,101 @@ +/* + * ===================================================================================== + * + * Filename: bitsop.h + * + * Description: bit operation function. setting location for any platform. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __BITSOP_H__ +#define __BITSOP_H__ +#include +#include +__BEGIN_DECLS +/* + bits操作函数 + */ + +/* + * + * select.h中提供的FD_*宏在32位机器上是按照byte来置位的（汇编实现），但在64位 + * 机器上是按照8 bytes来置位的，所以当碰到mmap文件末尾时，有可能segment fault。 + */ + +#define CHAR_BITS 8 + +extern const unsigned char __bitcount[]; +extern int CountBits(const char *buf, int sz); + +//bits op interface +#define SET_B(bit, addr) __set_b(bit, addr) +#define CLR_B(bit, addr) __clr_b(bit, addr) +#define ISSET_B(bit, addr) __isset_b(bit, addr) +#define COPY_B(dest_bit, dest_addr, src_bit, src_addr, count) __bit_copy(dest_bit, dest_addr, src_bit, src_addr, count) +#define COUNT_B(buf, size) CountBits(buf, size) + +static inline void __set_b(__u32 bit, const volatile void *addr) +{ + volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS); + __u8 shift = bit % CHAR_BITS; + + *begin |= ((__u8)0x1 << shift); + + return; +} + +static inline int __isset_b(__u32 bit, const volatile void *addr) +{ + volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS); + __u8 shift = bit % CHAR_BITS; + + return (*begin & ((__u8)0x1 << shift)) > 0 ? 1 : 0; +} + +static inline void __clr_b(__u32 bit, const volatile void *addr) +{ + volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS); + __u8 shift = bit % CHAR_BITS; + + *begin &= ~((__u8)0x1 << shift); +} + +static inline __u8 __readbyte(const volatile void *addr) +{ + return *(volatile __u8 *)addr; +} +static inline void __writebyte(__u8 val, volatile void *addr) +{ + *(volatile __u8 *)addr = val; +} + +static inline void __bit_copy(__u32 dest_bit, volatile void *dest, + __u32 src_bit, volatile void *src, + __u32 count) +{ + __u32 i; + for (i = 0; i < count; i++) + { + if (__isset_b(src_bit, src)) + __set_b(dest_bit, dest); + else + __clr_b(dest_bit, dest); + + dest_bit++; + src_bit++; + } + + return; +} + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/blacklist.cc b/src/search_local/index_storage/common/blacklist.cc new file mode 100644 index 0000000..93621f1 --- /dev/null +++ b/src/search_local/index_storage/common/blacklist.cc @@ -0,0 +1,193 @@ +/* + * ===================================================================================== + * + * Filename: blacklist.cc + * + * Description: blacklist in order to block super large node into cache. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "log.h" +#include "blacklist.h" +#include "keycodec.h" + +DTC_USING_NAMESPACE + +BlackList::BlackList() : current_blslot_count(0), + max_blslot_count(0), + blslot_expired_time(0) +{ +} + +BlackList::~BlackList() +{ +} + +int BlackList::init_blacklist(const unsigned max, const unsigned keytype, const unsigned expired) +{ + for (int i = 0; i < MAX_HASH_DEPTH; ++i) + { + INIT_HLIST_HEAD(hash_list + i); + } + + INIT_LIST_HEAD(&time_list); + + key_type = keytype; + current_blslot_count = 0; + max_blslot_count = max < 1000000 ? max : 1000000; /* max:1000000 slots */ + blslot_expired_time = expired; + + return 0; +} + +int BlackList::add_blacklist(const char *ptr, const unsigned vsize) +{ + if (!ptr) + return 0; + + KeyCodec key_codec(key_type); + + struct hlist_head *h = hash_list + key_codec.key_hash(ptr) % MAX_HASH_DEPTH; + struct hlist_node *pos = 0; + struct blslot *tpos = 0; + + hlist_for_each_entry(tpos, pos, h, hash) + { + /* hit */ + if (0 == key_codec.key_compare(ptr, tpos->key)) + { + tpos->vsize = vsize; + tpos->expired = time(NULL) + blslot_expired_time; + /* adjust time list */ + _list_move_tail(&tpos->time, &time_list); + return 0; + } + } + + if (current_blslot_count >= max_blslot_count) + /* overflow */ + return -1; + + tpos = (struct blslot *)malloc(offsetof(struct blslot, key) + key_codec.total_length(ptr)); + if (NULL == tpos) + { + log_notice("allocate blacklist slot failed"); + return -1; + } + + tpos->vsize = vsize; + tpos->expired = time(NULL) + blslot_expired_time; + memcpy(tpos->key, ptr, key_codec.total_length(ptr)); + + list_add_tail(&tpos->time, &time_list); + hlist_add_head(&tpos->hash, h); + + stat_everything(tpos, 1); + return 0; +} + +int BlackList::in_blacklist(const char *ptr) +{ + if (!ptr) + return 0; + + KeyCodec key_codec(key_type); + + struct hlist_head *h = hash_list + key_codec.key_hash(ptr) % MAX_HASH_DEPTH; + struct hlist_node *pos = 0; + struct blslot *tpos = 0; + + hlist_for_each_entry(tpos, pos, h, hash) + { + /* found */ + if (0 == key_codec.key_compare(ptr, tpos->key)) + return 1; + } + + return 0; +} + +int BlackList::try_expired_blacklist(void) +{ + unsigned now = time(NULL); + struct blslot *pos = 0; + + /* time->next is the oldest slot */ + while (!list_empty(&time_list)) + { + pos = list_entry(time_list.next, struct blslot, time); + if (pos->expired > now) + break; + + list_del(&pos->time); + hlist_del(&pos->hash); + stat_everything(pos, 0); + free(pos); + } + + return 0; +} + +/* TODO: 统计top10 */ +void BlackList::stat_everything(const struct blslot *slot, const int add) +{ + /* add */ + if (add) + ++current_blslot_count; + /* delete */ + else + --current_blslot_count; + return; +} + +void BlackList::dump_all_blslot(void) +{ + KeyCodec key_codec(key_type); + + struct blslot *pos = 0; + list_for_each_entry(pos, &time_list, time) + { + switch (key_type) + { + case 1: + log_debug("key: %u size: %u expired: %u hash: %u", + *(uint8_t *)key_codec.key_pointer(pos->key), + pos->vsize, + pos->expired, + key_codec.key_hash(pos->key) % MAX_HASH_DEPTH); + break; + case 2: + log_debug("key: %u size: %u expired: %u hash: %u", + *(uint16_t *)key_codec.key_pointer(pos->key), + pos->vsize, + pos->expired, + key_codec.key_hash(pos->key) % MAX_HASH_DEPTH); + break; + case 4: + log_debug("key: %u size: %u expired: %u hash: %u", + *(uint32_t *)key_codec.key_pointer(pos->key), + pos->vsize, + pos->expired, + key_codec.key_hash(pos->key) % MAX_HASH_DEPTH); + break; + default: + log_debug("key: %10s size: %u expired: %u hash: %u", + key_codec.key_pointer(pos->key), + pos->vsize, + pos->expired, + key_codec.key_hash(pos->key) % MAX_HASH_DEPTH); + break; + } + } + + return; +} diff --git a/src/search_local/index_storage/common/blacklist.h b/src/search_local/index_storage/common/blacklist.h new file mode 100644 index 0000000..9855bce --- /dev/null +++ b/src/search_local/index_storage/common/blacklist.h @@ -0,0 +1,73 @@ +/* + * ===================================================================================== + * + * Filename: blacklist.h + * + * Description: blacklist in order to block super large node into cache. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __DTC_BLACKLIST_H +#define __DTC_BLACKLIST_H + +#include "list.h" +#include "hlist.h" +#include +#include "namespace.h" + +#define MAX_HASH_DEPTH 65535 + +DTC_BEGIN_NAMESPACE + +struct blslot +{ + struct hlist_node hash; /* link to hash_list */ + struct list_head time; /* link to time_list */ + unsigned int vsize; + unsigned int expired; + char key[0]; +} __attribute__((__aligned__(1))); + +class BlackList +{ +public: + BlackList(); + ~BlackList(); + + int init_blacklist(const unsigned max, const unsigned type, const unsigned expired); + int add_blacklist(const char *packedkey, const unsigned vsize); + int in_blacklist(const char *packedkey); + + /* dump all blslot in blacklist, debug only */ + void dump_all_blslot(void); + +protected: + /* try expire all expired slot */ + int try_expired_blacklist(void); + + unsigned current_blslot_count; + +private: + /* double linked hash list with single pointer list head */ + struct hlist_head hash_list[MAX_HASH_DEPTH]; + + /* time list */ + struct list_head time_list; + + unsigned max_blslot_count; + unsigned blslot_expired_time; + unsigned key_type; + void stat_everything(const struct blslot *, const int add); +}; + +DTC_END_NAMESPACE +#endif diff --git a/src/search_local/index_storage/common/blacklist_unit.cc b/src/search_local/index_storage/common/blacklist_unit.cc new file mode 100644 index 0000000..4d6b380 --- /dev/null +++ b/src/search_local/index_storage/common/blacklist_unit.cc @@ -0,0 +1,43 @@ +/* + * ===================================================================================== + * + * Filename: blacklist_unit.cc + * + * Description: balcklist attach excute unit, arun expired task and output stastic result at regular intervals. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +DTC_USING_NAMESPACE + +BlackListUnit::BlackListUnit(TimerList *t) : timer(t) +{ +} + +BlackListUnit::~BlackListUnit() +{ +} + +void BlackListUnit::timer_notify(void) +{ + log_debug("sched blacklist-expired task"); + + /* expire all expired eslot */ + try_expired_blacklist(); + + BlackList::dump_all_blslot(); + + /* set timer agagin */ + attach_timer(timer); + + return; +} diff --git a/src/search_local/index_storage/common/blacklist_unit.h b/src/search_local/index_storage/common/blacklist_unit.h new file mode 100644 index 0000000..0c74bf4 --- /dev/null +++ b/src/search_local/index_storage/common/blacklist_unit.h @@ -0,0 +1,93 @@ +/* + * ===================================================================================== + * + * Filename: blacklist_unit.h + * + * Description: balcklist attach excute unit, arun expired task and output stastic result at regular intervals. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __DTC_BLACKLIST_UNIT_H +#define __DTC_BLACKLIST_UNIT_H + +#include "namespace.h" +#include "timer_list.h" +#include "blacklist.h" +#include "log.h" +#include "stat_dtc.h" + +DTC_BEGIN_NAMESPACE + +class TimerObject; +class BlackListUnit : public BlackList, private TimerObject +{ +public: + BlackListUnit(TimerList *timer); + virtual ~BlackListUnit(void); + + int init_blacklist(const unsigned max, const unsigned type, const unsigned expired = 10 * 60 /*10 mins*/) + { + /* init statisitc item */ + stat_blacksize = statmgr.get_sample(BLACKLIST_SIZE); + stat_blslot_count = statmgr.get_item_u32(BLACKLIST_CURRENT_SLOT); + + return BlackList::init_blacklist(max, type, expired); + } + + int add_blacklist(const char *key, const unsigned vsize) + { + int ret = BlackList::add_blacklist(key, vsize); + if (0 == ret) + { + /* statistic */ + stat_blacksize.push(vsize); + stat_blslot_count = current_blslot_count; + } + + return ret; + } + + int try_expired_blacklist(void) + { + int ret = BlackList::try_expired_blacklist(); + if (0 == ret) + { + /* statistic */ + stat_blslot_count = current_blslot_count; + } + + return ret; + } + + void start_blacklist_expired_task(void) + { + log_info("start blacklist-expired task"); + + attach_timer(timer); + + return; + } + +public: + virtual void timer_notify(void); + +private: + TimerList *timer; + + /* for statistic */ + StatSample stat_blacksize; /* black size distribution */ + StatItemU32 stat_blslot_count; +}; + +DTC_END_NAMESPACE + +#endif diff --git a/src/search_local/index_storage/common/buffer.cc b/src/search_local/index_storage/common/buffer.cc new file mode 100644 index 0000000..9263234 --- /dev/null +++ b/src/search_local/index_storage/common/buffer.cc @@ -0,0 +1,86 @@ +/* + * ===================================================================================== + * + * Filename: buffer.cc + * + * Description: buffer block relevant operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "buffer.h" + +void CreateBuff(int inlen, int &len, char **buff) +{ + if (*buff == NULL) + { + *buff = (char *)CALLOC(inlen, sizeof(char)); + if (*buff == NULL) + len = 0; + else + len = inlen; + } + else + { + if (len < inlen) + { + FREE_IF(*buff); + *buff = (char *)CALLOC(inlen, sizeof(char)); + if (*buff == NULL) + len = 0; + else + len = inlen; + } + else + { + memset(*buff, 0x0, len); + } + } +} + +int buffer::bprintf(const char *format, ...) +{ + va_list ap; + long len; + + va_start(ap, format); + len = bprintf(format, ap); + va_end(ap); + return len; +} + +int buffer::vbprintf(const char *format, va_list ap) +{ + long len; + va_list ap2; + +#ifdef __va_copy + __va_copy(ap2, ap); +#else + va_copy(ap2, ap); +#endif + len = vsnprintf(cursor(), remain(), format, ap); + if (len < 0) + return len; + + if ((unsigned long)len >= remain()) + { + if (expand(len) < 0) + return -1; + len = vsnprintf(cursor(), remain(), format, ap2); + if (len < 0) + return len; + } + dataSize += len; + return len; +} diff --git a/src/search_local/index_storage/common/buffer.h b/src/search_local/index_storage/common/buffer.h new file mode 100644 index 0000000..7551c47 --- /dev/null +++ b/src/search_local/index_storage/common/buffer.h @@ -0,0 +1,200 @@ +/* + * ===================================================================================== + * + * Filename: buffer.h + * + * Description: buffer block relevant operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __BUFFER_H +#define __BUFFER_H + +#include "mem_check.h" +#include +#include + +extern void CreateBuff(int inlen, int &len, char **buff); + +class buffer +{ +private: + char *buf; + unsigned int bufSize; + unsigned int dataSize; + unsigned int needSize; + +public: + buffer(void) + { + buf = NULL; + bufSize = dataSize = needSize = 0; + } + ~buffer(void) + { + FREE_IF(buf); + bufSize = dataSize = needSize = 0; + } + inline int expand(int size) + { + needSize = dataSize + size + 1; + if (needSize > bufSize) + { + int sz = needSize + (needSize >> 3); + sz = (sz | 0xf) + 0x10; // align to 16 bytes + if (REALLOC(buf, sz) == NULL) + return -1; + bufSize = sz; + } + return 0; + } + inline void trunc(int n) + { + if (n >= (long long)dataSize) + return; + if (n >= 0) + { + buf[dataSize = n] = 0; + return; + } + n += dataSize; + if (n < 0) + return; + buf[dataSize = n] = 0; + } + inline int resize(int n) + { + if (n > (long long)dataSize && expand(n - dataSize) < 0) + return -1; + buf[dataSize = n] = 0; + return 0; + } + inline void clear(void) { trunc(0); } + inline void release(void) + { + trunc(0); + FREE_CLEAR(buf); + bufSize = 0; + needSize = 0; + } + inline char &operator[](int n) const { return buf[n]; } + inline char at(int n) const + { + if (n > (long long)dataSize) + return 0; + if (n >= 0) + return buf[n]; + n += dataSize; + if (n < 0) + return 0; + return buf[n]; + } + inline char *c_str(void) { return buf; } + inline const char *c_str(void) const { return buf; } + inline char *cursor(void) { return buf + dataSize; } + inline const char *cursor(void) const { return buf + dataSize; } + inline unsigned int size(void) const { return dataSize; } + inline unsigned int needed(void) const { return needSize; } + inline unsigned int remain(void) const { return bufSize - dataSize; } + inline unsigned int margin(void) const { return needSize < bufSize ? 0 : needSize - bufSize; } + inline int append(const char *data, int len) + { + if (expand(len) < 0) + return -1; + memcpy(buf + dataSize, data, len); + dataSize += len; + buf[dataSize] = 0; + return 0; + } + inline int append(const char *data) + { + return append(data, strlen(data)); + } + inline int append(const buffer &data) + { + return append(data.c_str(), data.size()); + } + inline int append(const buffer *data) + { + return append(data->c_str(), data->size()); + } + +#define __TEMPLATE_FUNC__(t) \ + inline int append(t v) \ + { \ + if (expand(sizeof(t)) < 0) \ + return -1; \ + *(t *)(buf + dataSize) = v; \ + dataSize += sizeof(t); \ + buf[dataSize] = 0; \ + return 0; \ + } + __TEMPLATE_FUNC__(char); + __TEMPLATE_FUNC__(signed char); + __TEMPLATE_FUNC__(unsigned char); + __TEMPLATE_FUNC__(short); + __TEMPLATE_FUNC__(unsigned short); + __TEMPLATE_FUNC__(int); + __TEMPLATE_FUNC__(unsigned int); + __TEMPLATE_FUNC__(long); + __TEMPLATE_FUNC__(unsigned long); + __TEMPLATE_FUNC__(long long); + __TEMPLATE_FUNC__(unsigned long long); + __TEMPLATE_FUNC__(float); + __TEMPLATE_FUNC__(double); +#undef __TEMPLATE_FUNC__ + + // unsafe + inline buffer &add(const char *data, int len) + { + memcpy(buf + dataSize, data, len); + dataSize += len; + return *this; + } + inline buffer &operator<<(const char *data) + { + return add(data, strlen(data)); + } + inline buffer &operator<<(const buffer &data) + { + return add(data.c_str(), data.size()); + } + inline buffer &operator<<(const buffer *data) + { + return add(data->c_str(), data->size()); + } + +#define __TEMPLATE_FUNC__(t) \ + inline buffer &operator<<(t v) \ + { \ + *(t *)(buf + dataSize) = v; \ + dataSize += sizeof(t); \ + return *this; \ + } + __TEMPLATE_FUNC__(char); + __TEMPLATE_FUNC__(signed char); + __TEMPLATE_FUNC__(unsigned char); + __TEMPLATE_FUNC__(short); + __TEMPLATE_FUNC__(unsigned short); + __TEMPLATE_FUNC__(int); + __TEMPLATE_FUNC__(unsigned int); + __TEMPLATE_FUNC__(long); + __TEMPLATE_FUNC__(unsigned long); + __TEMPLATE_FUNC__(long long); + __TEMPLATE_FUNC__(unsigned long long); + __TEMPLATE_FUNC__(float); + __TEMPLATE_FUNC__(double); +#undef __TEMPLATE_FUNC__ + int bprintf(const char *format, ...); + int vbprintf(const char *format, va_list ap); +}; + +#endif diff --git a/src/search_local/index_storage/common/buffer_error.h b/src/search_local/index_storage/common/buffer_error.h new file mode 100644 index 0000000..5663a4c --- /dev/null +++ b/src/search_local/index_storage/common/buffer_error.h @@ -0,0 +1,135 @@ +/* + * ===================================================================================== + * + * Filename: buffer_error.h + * + * Description: cache error code list. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_CACHE_ERRR_H__ +#define __CH_CACHE_ERRR_H__ + +enum +{ + EC_ERROR_BASE = 2000, + EC_BAD_COMMAND, // unsupported command + EC_MISSING_SECTION, // missing mandatory section + EC_EXTRA_SECTION, // incompatible section present + EC_DUPLICATE_TAG, // same tag appear twice + + EC_DUPLICATE_FIELD, //5: same field appear twice in .Need() + EC_BAD_SECTION_LENGTH, // section length too short + EC_BAD_VALUE_LENGTH, // value length not allow + EC_BAD_STRING_VALUE, // string value w/o NULL + EC_BAD_FLOAT_VALUE, // invalid float format + + EC_BAD_FIELD_NUM, //10: invalid total field# + EC_EXTRA_SECTION_DATA, // section length too large + EC_BAD_VALUE_TYPE, // incompatible value type + EC_BAD_OPERATOR, // incompatible operator/comparison + EC_BAD_FIELD_ID, // invalid field ID + + EC_BAD_FIELD_NAME, //15: invalid field name + EC_BAD_FIELD_TYPE, // invalid field type + EC_BAD_FIELD_SIZE, // invalid field size + EC_TABLE_REDEFINED, // table defined twice + EC_TABLE_MISMATCH, // request table != server table + + EC_VERSION_MISMATCH, //20: unsupported protocol version + EC_CHECKSUM_MISMATCH, // table hash not equal + EC_NO_MORE_DATA, // End of Result + EC_NEED_FULL_FIELDSET, // only full field set accepted by helper + EC_BAD_KEY_TYPE, // key type incompatible + + EC_BAD_KEY_SIZE, // 25: key size incompatible + EC_SERVER_BUSY, //server error + EC_BAD_SOCKET, // network failed + EC_NOT_INITIALIZED, // object didn't initialized + EC_BAD_HOST_STRING, + + EC_BAD_TABLE_NAME, // 30 + EC_TASK_NEED_DELETE, + EC_KEY_NEEDED, + EC_SERVER_ERROR, + EC_UPSTREAM_ERROR, + + EC_KEY_OVERFLOW, // 35 + EC_BAD_MULTIKEY, + EC_READONLY_FIELD, + EC_BAD_ASYNC_CMD, + EC_OUT_OF_KEY_RANGE, + + EC_REQUEST_ABORTED, // 40 + EC_PARALLEL_MODE, + EC_KEY_NOTEXIST, + EC_SERVER_READONLY, + EC_BAD_INVALID_FIELD, + + EC_DUPLICATE_KEY, // 45 + EC_TOO_MANY_KEY_VALUE, + EC_BAD_KEY_NAME, + EC_BAD_RAW_DATA, + EC_BAD_HOTBACKUP_JID, + + EC_FULL_SYNC_COMPLETE, //50 + EC_FULL_SYNC_STAGE, + EC_INC_SYNC_STAGE, + EC_ERR_SYNC_STAGE, + EC_NOT_ALLOWED_INSERT, + + EC_COMPRESS_ERROR, //55 + EC_UNCOMPRESS_ERROR, + EC_TASKPOOL, + EC_STATE_ERROR, + EC_DATA_NEEDED, + + EC_BAD_ACCESS_KEY, //60 + EC_TASK_TIMEOUT, + EC_BUSINESS_WITHOUT_EXPIRETIME, //62 + EC_EMPTY_TBDEF, //63 + EC_INVALID_KEY_VALUE, //64 + + EC_INVALID_EXPIRETIME, //65 + EC_GET_EXPIRETIME_END_OF_RESULT, //66 + EC_ERR_MIGRATEDB_ILLEGAL, + EC_ERR_MIGRATEDB_DUPLICATE, + EC_ERR_MIGRATEDB_HELPER, + + EC_ERR_MIGRATEDB_MIGRATING, // 70 + EC_ERR_MIGRATEDB_NOT_START, + EC_ERR_MIGRATEDB_DISTINCT, + EC_ERR_HOTBACK_WRITELRU, + EC_ERR_HOTBACK_WRITEUPDATE, + + EC_ERR_COL_EXPANDING, // 75 + EC_ERR_COL_EXPAND_DUPLICATE, + EC_ERR_COL_EXPAND_DONE_DUPLICATE, + EC_ERR_COL_EXPAND_DONE_DISTINCT, + EC_ERR_COL_EXPAND_NO_MEM, + + EC_ERR_COL_EXPAND_KEY_ERROR, // 80 + EC_ERR_COL_NOT_EXPANDING, + EC_ERR_COL_NO_KEY, + EC_GET_EXPIRETIME_RESULT_NULL, + EC_GET_EXPIRETIME_FIELD_EXPIRETIME_NULL, + + EC_BAD_FIELD_SIZE_ON_RESOLVE, // 85 + EC_EXCEPTION_ON_RESOLVE, + EC_BAD_FIELD_SIZE_ON_CHECKKEY, + EC_EXCEPTION_ON_CHECKKEY, + EC_NOT_HAVE_AVAILABLE_HELPERGROUP, + + EC_NOT_HAVE_AVAILABLE_HELPERCLIENT, // 90 + EC_RELOAD_CONFIG_FAILED, +}; + +#endif diff --git a/src/search_local/index_storage/common/chash.cc b/src/search_local/index_storage/common/chash.cc new file mode 100644 index 0000000..78d20d8 --- /dev/null +++ b/src/search_local/index_storage/common/chash.cc @@ -0,0 +1,107 @@ +/* + * ===================================================================================== + * + * Filename: chash.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "chash.h" + +#define mix(a, b, c) \ + { \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 13); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 8); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 13); \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 12); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 16); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 5); \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 3); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 10); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 15); \ + } + +typedef unsigned int u4; + +unsigned int chash(const char *k, int length) +{ + unsigned int a, b, c; /* the internal state */ + u4 len; /* how many key bytes still need mixing */ + + /* Set up the internal state */ + len = length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + // 'TMJR' + c = 0x544D4A52; /* variable initialization of internal state */ + + /*---------------------------------------- handle most of the key */ + while (len >= 12) + { + a = a + (k[0] + ((u4)k[1] << 8) + ((u4)k[2] << 16) + ((u4)k[3] << 24)); + b = b + (k[4] + ((u4)k[5] << 8) + ((u4)k[6] << 16) + ((u4)k[7] << 24)); + c = c + (k[8] + ((u4)k[9] << 8) + ((u4)k[10] << 16) + ((u4)k[11] << 24)); + mix(a, b, c); + k = k + 12; + len = len - 12; + } + + /*------------------------------------- handle the last 11 bytes */ + c = c + length; + switch (len) /* all the case statements fall through */ + { + case 11: + c = c + ((u4)k[10] << 24); + case 10: + c = c + ((u4)k[9] << 16); + case 9: + c = c + ((u4)k[8] << 8); + /* the first byte of c is reserved for the length */ + case 8: + b = b + ((u4)k[7] << 24); + case 7: + b = b + ((u4)k[6] << 16); + case 6: + b = b + ((u4)k[5] << 8); + case 5: + b = b + k[4]; + case 4: + a = a + ((u4)k[3] << 24); + case 3: + a = a + ((u4)k[2] << 16); + case 2: + a = a + ((u4)k[1] << 8); + case 1: + a = a + k[0]; + /* case 0: nothing left to add */ + } + mix(a, b, c); + /*-------------------------------------------- report the result */ + return c; +} diff --git a/src/search_local/index_storage/common/chash.h b/src/search_local/index_storage/common/chash.h new file mode 100644 index 0000000..da5123a --- /dev/null +++ b/src/search_local/index_storage/common/chash.h @@ -0,0 +1,27 @@ +/* + * ===================================================================================== + * + * Filename: chash.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef BITMAP_C_HASH_H__ +#define BITMAP_C_HASH_H__ + +#include + +//hash function for consistent hash +//the algorithm is the same as new_hash, but use another initial value +uint32_t chash(const char *data, int len); + +#endif diff --git a/src/search_local/index_storage/common/client_async.cc b/src/search_local/index_storage/common/client_async.cc new file mode 100644 index 0000000..c433868 --- /dev/null +++ b/src/search_local/index_storage/common/client_async.cc @@ -0,0 +1,294 @@ +/* + * ===================================================================================== + * + * Filename: client_async.cc + * + * Description: client async operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "client_async.h" +#include "client_unit.h" +#include "poll_thread.h" +#include "log.h" + +static int statEnable = 0; +static StatItemU32 statAcceptCount; +static StatItemU32 statCurConnCount; + +inline AsyncInfo::AsyncInfo(ClientAsync *c, TaskRequest *r) + : cli(c), + req(r), + pkt(NULL) + +{ + c->numReq++; + /* move to waiting list */ + ListAddTail(&c->waitList); +} + +inline AsyncInfo::~AsyncInfo() +{ + if (req) + req->clear_owner_info(); + DELETE(pkt); + if (cli) + cli->numReq--; +} + +class CReplyAsync : public ReplyDispatcher +{ +public: + CReplyAsync(void) {} + virtual ~CReplyAsync(void); + virtual void reply_notify(TaskRequest *task); +}; + +CReplyAsync::~CReplyAsync(void) {} + +void CReplyAsync::reply_notify(TaskRequest *task) +{ + AsyncInfo *info = task->OwnerInfo(); + if (info == NULL) + { + delete task; + } + else if (info->cli == NULL) + { + log_error("info->cli is NULL, possible memory corrupted"); + delete task; + } + else + { + info->cli->queue_result(info); + } +} + +static CReplyAsync replyAsync; + +ClientAsync::ClientAsync(DTCDecoderUnit *o, int fd, int m, void *peer, int ps) + : PollerObject(o->owner_thread(), fd), + owner(o), + receiver(fd), + curReq(NULL), + curRes(NULL), + maxReq(m), + numReq(0) +{ + addrLen = ps; + addr = MALLOC(ps); + memcpy((char *)addr, (char *)peer, ps); + + if (!statEnable) + { + statAcceptCount = statmgr.get_item_u32(ACCEPT_COUNT); + statCurConnCount = statmgr.get_item_u32(CONN_COUNT); + statEnable = 1; + } + ++statAcceptCount; + ++statCurConnCount; +} + +ClientAsync::~ClientAsync() +{ + while (!doneList.ListEmpty()) + { + AsyncInfo *info = doneList.NextOwner(); + delete info; + } + while (!waitList.ListEmpty()) + { + AsyncInfo *info = waitList.NextOwner(); + delete info; + } + --statCurConnCount; + + FREE_IF(addr); +} + +int ClientAsync::Attach() +{ + enable_input(); + if (attach_poller() == -1) + return -1; + + return 0; +} + +//server peer +int ClientAsync::recv_request() +{ + if (curReq == NULL) + { + curReq = new TaskRequest(owner->owner_table()); + if (NULL == curReq) + { + log_error("%s", "create TaskRequest object failed, msg[no enough memory]"); + return -1; + } + curReq->set_hotbackup_table(owner->admin_table()); + receiver.erase(); + } + + int ret = curReq->Decode(receiver); + switch (ret) + { + default: + case DecodeFatalError: + if (errno != 0) + log_notice("decode fatal error, ret = %d msg = %m", ret); + DELETE(curReq); + return -1; + + case DecodeDataError: + curReq->response_timer_start(); + curReq->mark_as_hit(); + if (curReq->result_code() < 0) + log_notice("DecodeDataError, role=%d, fd=%d, result=%d", + curReq->Role(), netfd, curReq->result_code()); + return queue_error(); + + case DecodeDone: + if (curReq->prepare_process() < 0) + return queue_error(); + curReq->set_owner_info(new AsyncInfo(this, curReq), 0, (struct sockaddr *)addr); + curReq->push_reply_dispatcher(&replyAsync); + owner->task_dispatcher(curReq); + curReq = NULL; + } + return 0; +} + +int ClientAsync::queue_error(void) +{ + AsyncInfo *info = new AsyncInfo(this, curReq); + curReq = NULL; + return queue_result(info); +} + +int ClientAsync::queue_result(AsyncInfo *info) +{ + if (info->req == NULL) + { + delete info; + return 0; + } + + TaskRequest *const cur = info->req; + owner->record_request_time(curReq); + + /* move to sending list */ + info->list_move_tail(&doneList); + /* convert request to result */ + info->pkt = new Packet; + if (info->pkt == NULL) + { + delete info->req; + info->req = NULL; + delete info; + log_error("create Packet object failed"); + return 0; + } + + cur->versionInfo.set_keep_alive_timeout(owner->idle_time()); + info->pkt->encode_result(info->req); + DELETE(info->req); + + Response(); + adjust_events(); + return 0; +} + +int ClientAsync::response_one(void) +{ + if (curRes == NULL) + return 0; + + int ret = curRes->Send(netfd); + + switch (ret) + { + case SendResultMoreData: + return 0; + + case SendResultDone: + DELETE(curRes); + numReq--; + return 0; + + default: + log_notice("send failed, return = %d, error = %m", ret); + return -1; + } + return 0; +} + +int ClientAsync::Response(void) +{ + do + { + int ret; + if (curRes == NULL) + { + // All result sent + if (doneList.ListEmpty()) + break; + + AsyncInfo *info = doneList.NextOwner(); + curRes = info->pkt; + info->pkt = NULL; + delete info; + numReq++; + if (curRes == NULL) + continue; + } + ret = response_one(); + if (ret < 0) + return ret; + } while (curRes == NULL); + return 0; +} + +void ClientAsync::input_notify(void) +{ + if (recv_request() < 0) + delete this; + else + adjust_events(); +} + +void ClientAsync::output_notify(void) +{ + if (Response() < 0) + delete this; + else + adjust_events(); +} + +int ClientAsync::adjust_events(void) +{ + if (curRes == NULL && doneList.ListEmpty()) + disable_output(); + else + enable_output(); + if (numReq >= maxReq) + DisableInput(); + else + enable_input(); + return apply_events(); +} diff --git a/src/search_local/index_storage/common/client_async.h b/src/search_local/index_storage/common/client_async.h new file mode 100644 index 0000000..6c2f916 --- /dev/null +++ b/src/search_local/index_storage/common/client_async.h @@ -0,0 +1,83 @@ +/* + * ===================================================================================== + * + * Filename: client_async.h + * + * Description: client async operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CLIENT__ASYNC_H__ +#define __CLIENT__ASYNC_H__ + +#include "poller.h" +#include "task_request.h" +#include "packet.h" +#include "timer_list.h" +#include "list.h" + +class DTCDecoderUnit; +class ClientAsync; + +class AsyncInfo : private ListObject +{ +public: + AsyncInfo(ClientAsync *c, TaskRequest *r); + ~AsyncInfo(); + void list_move_tail(ListObject *a) + { + ListObject::list_move_tail(a); + } + + ClientAsync *cli; + TaskRequest *req; + Packet *pkt; +}; + +class ClientAsync : public PollerObject +{ +public: + friend class AsyncInfo; + DTCDecoderUnit *owner; + + ClientAsync(DTCDecoderUnit *, int fd, int depth, void *peer, int ps); + virtual ~ClientAsync(); + + virtual int Attach(void); + int queue_result(AsyncInfo *); + int queue_error(void); + int flush_result(void); + int adjust_events(void); + + virtual void input_notify(void); + +private: + virtual void output_notify(void); + + int recv_request(void); + int Response(void); + int response_one(void); + +protected: + SimpleReceiver receiver; + TaskRequest *curReq; // decoding + Packet *curRes; // sending + + ListObject waitList; + ListObject doneList; + + void *addr; + int addrLen; + int maxReq; + int numReq; +}; + +#endif diff --git a/src/search_local/index_storage/common/client_dgram.cc b/src/search_local/index_storage/common/client_dgram.cc new file mode 100644 index 0000000..22ff25f --- /dev/null +++ b/src/search_local/index_storage/common/client_dgram.cc @@ -0,0 +1,299 @@ +/* + * ===================================================================================== + * + * Filename: client_dgram.cc + * + * Description: client dgram. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client_dgram.h" +#include "task_request.h" +#include "client_unit.h" +#include "protocol.h" +#include "log.h" + +class PollThread; +static int GetSocketFamily(int fd) +{ + struct sockaddr addr; + bzero(&addr, sizeof(addr)); + socklen_t alen = sizeof(addr); + getsockname(fd, &addr, &alen); + return addr.sa_family; +} + +class ReplyDgram : public ReplyDispatcher +{ +public: + ReplyDgram(void) {} + virtual ~ReplyDgram(void); + virtual void reply_notify(TaskRequest *task); +}; + +ReplyDgram::~ReplyDgram(void) {} + +void ReplyDgram::reply_notify(TaskRequest *task) +{ + DgramInfo *info = task->OwnerInfo(); + if (info == NULL) + { + delete task; + } + else if (info->cli == NULL) + { + log_error("info->cli is NULL, possible memory corrupted"); + FREE(info); + delete task; + } + else + { + info->cli->send_result(task, info->addr, info->len); + FREE(info); + } +} + +static ReplyDgram replyDgram; + +ClientDgram::ClientDgram(DTCDecoderUnit *o, int fd) + : PollerObject(o->owner_thread(), fd), + owner(o), + hastrunc(0), + mru(0), + mtu(0), + alen(0), + abuf(NULL) +{ +} + +ClientDgram::~ClientDgram() +{ + FREE_IF(abuf); +} + +int ClientDgram::init_socket_info(void) +{ + switch (GetSocketFamily(netfd)) + { + default: + mru = 65508; + mtu = 65507; + alen = sizeof(struct sockaddr); + break; + case AF_UNIX: + mru = 16 << 20; + mtu = 16 << 20; + alen = sizeof(struct sockaddr_un); + break; + case AF_INET: + hastrunc = 1; + mru = 65508; + mtu = 65507; + alen = sizeof(struct sockaddr_in); + break; + case AF_INET6: + hastrunc = 1; + mru = 65508; + mtu = 65507; + alen = sizeof(struct sockaddr_in6); + break; + } + return 0; +} + +int ClientDgram::allocate_dgram_info(void) +{ + if (abuf != NULL) + return 0; + + abuf = (DgramInfo *)MALLOC(sizeof(DgramInfo) + alen); + if (abuf == NULL) + return -1; + + abuf->cli = this; + return 0; +} + +int ClientDgram::Attach() +{ + init_socket_info(); + + enable_input(); + if (attach_poller() == -1) + return -1; + + return 0; +} + +// server peer +// return value: -1, fatal error or no more packets +// return value: 0, more packet may be pending +int ClientDgram::recv_request(int noempty) +{ + if (allocate_dgram_info() < 0) + { + log_error("%s", "create DgramInfo object failed, msg[no enough memory]"); + return -1; + } + + int dataLen; + if (hastrunc) + { + char dummy[1]; + abuf->len = alen; + dataLen = recvfrom(netfd, dummy, 1, MSG_DONTWAIT | MSG_TRUNC | MSG_PEEK, (sockaddr *)abuf->addr, &abuf->len); + if (dataLen < 0) + { + // NO ERROR, and assume NO packet pending + return -1; + } + } + else + { + dataLen = -1; + ioctl(netfd, SIOCINQ, &dataLen); + if (dataLen < 0) + { + log_error("%s", "next packet size unknown"); + return -1; + } + + if (dataLen == 0) + { + if (noempty) + { + // NO ERROR, and assume NO packet pending + return -1; + } + /* treat 0 as max packet, because we can't ditiguish the no-packet & zero-packet */ + dataLen = mru; + } + } + + char *buf = (char *)MALLOC(dataLen); + if (buf == NULL) + { + log_error("allocate packet buffer[%d] failed, msg[no enough memory]", dataLen); + return -1; + } + + abuf->len = alen; + dataLen = recvfrom(netfd, buf, dataLen, MSG_DONTWAIT, (sockaddr *)abuf->addr, &abuf->len); + if (abuf->len <= 1) + { + log_notice("recvfrom error: no source address"); + free(buf); + // more packet pending + return 0; + } + + if (dataLen <= (int)sizeof(PacketHeader)) + { + int err = dataLen == -1 ? errno : 0; + if (err != EAGAIN) + log_notice("recvfrom error: size=%d errno=%d", dataLen, err); + free(buf); + // more packet pending + return 0; + } + + TaskRequest *task = new TaskRequest(owner->owner_table()); + if (NULL == task) + { + log_error("%s", "create TaskRequest object failed, msg[no enough memory]"); + return -1; + } + + task->set_hotbackup_table(owner->admin_table()); + + int ret = task->Decode(buf, dataLen, 1 /*eat*/); + switch (ret) + { + default: + case DecodeFatalError: + if (errno != 0) + log_notice("decode fatal error, ret = %d msg = %m", ret); + free(buf); // buf not eatten + break; + + case DecodeDataError: + task->response_timer_start(); + if (task->result_code() < 0) + log_notice("DecodeDataError, role=%d, fd=%d, result=%d", + task->Role(), netfd, task->result_code()); + send_result(task, (void *)abuf->addr, abuf->len); + break; + + case DecodeDone: + if (task->prepare_process() < 0) + { + send_result(task, (void *)abuf->addr, abuf->len); + break; + } + + task->set_owner_info(abuf, 0, (sockaddr *)abuf->addr); + abuf = NULL; // eat abuf + + task->push_reply_dispatcher(&replyDgram); + owner->task_dispatcher(task); + } + return 0; +} + +int ClientDgram::send_result(TaskRequest *task, void *addr, int len) +{ + if (task == NULL) + return 0; + + owner->record_request_time(task); + Packet *reply = new Packet; + if (reply == NULL) + { + delete task; + log_error("create Packet object failed"); + return 0; + } + + task->versionInfo.set_keep_alive_timeout(owner->idle_time()); + reply->encode_result(task, mtu); + delete task; + + int ret = reply->send_to(netfd, (struct sockaddr *)addr, len); + + delete reply; + + if (ret != SendResultDone) + { + log_notice("send failed, return = %d, error = %m", ret); + return 0; + } + return 0; +} + +void ClientDgram::input_notify(void) +{ + const int batchsize = 64; + for (int i = 0; i < batchsize; ++i) + { + if (recv_request(i) < 0) + break; + } +} diff --git a/src/search_local/index_storage/common/client_dgram.h b/src/search_local/index_storage/common/client_dgram.h new file mode 100644 index 0000000..133bf13 --- /dev/null +++ b/src/search_local/index_storage/common/client_dgram.h @@ -0,0 +1,64 @@ +/* + * ===================================================================================== + * + * Filename: client_dgram.h + * + * Description: client dgram. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CLIENT__DGRAM_H__ +#define __CLIENT__DGRAM_H__ + +#include +#include "poller.h" +#include "task_request.h" +#include "packet.h" +#include "timer_list.h" + +class DTCDecoderUnit; +class ClientDgram; + +struct DgramInfo +{ + ClientDgram *cli; + socklen_t len; + char addr[0]; +}; + +class ClientDgram : public PollerObject +{ +public: + DTCDecoderUnit *owner; + + ClientDgram(DTCDecoderUnit *, int fd); + virtual ~ClientDgram(); + + virtual int Attach(void); + int send_result(TaskRequest *, void *, int); + +protected: + // recv on empty&no packets + int recv_request(int noempty); + +private: + int hastrunc; + int mru; + int mtu; + int alen; // address length + DgramInfo *abuf; // current packet address + + virtual void input_notify(void); + int allocate_dgram_info(void); + int init_socket_info(void); +}; + +#endif diff --git a/src/search_local/index_storage/common/client_resource_pool.cc b/src/search_local/index_storage/common/client_resource_pool.cc new file mode 100644 index 0000000..a4c5ace --- /dev/null +++ b/src/search_local/index_storage/common/client_resource_pool.cc @@ -0,0 +1,263 @@ +/* + * ===================================================================================== + * + * Filename: client_resource_pool.cc + * + * Description: client resource pool. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "client_resource_pool.h" +#include "packet.h" +#include "task_request.h" +#include "log.h" + +ClientResourceSlot::~ClientResourceSlot() +{ + DELETE(task); + DELETE(packet); +} + +ClientResourcePool::~ClientResourcePool() +{ +} + +/* rules: */ +/* head free slot: freeprev == 0 + * tail free slot: freenext == 0 + * used slot: freeprev == 0, freenext == 0 + * non-empty list: freehead, freetail not 0 + * empty list: freehead, freetail == 0 + * */ +int ClientResourcePool::Init() +{ + taskslot = (ClientResourceSlot *)CALLOC(256, sizeof(ClientResourceSlot)); + if (taskslot == NULL) + return -1; + + total = 256; + used = 1; + freehead = 1; + freetail = total - 1; + + /* 1(freehead), 2..., total - 1(freetail) */ + taskslot[1].freenext = 2; + taskslot[1].freeprev = 0; + if ((taskslot[1].task = new TaskRequest) == NULL) + return -1; + if ((taskslot[1].packet = new Packet) == NULL) + return -1; + taskslot[1].seq = 0; + + for (unsigned int i = 2; i < total - 1; i++) + { + taskslot[i].freenext = i + 1; + taskslot[i].freeprev = i - 1; + if ((taskslot[i].task = new TaskRequest) == NULL) + return -1; + if ((taskslot[i].packet = new Packet) == NULL) + return -1; + taskslot[i].seq = 0; + } + + taskslot[total - 1].freenext = 0; + taskslot[total - 1].freeprev = total - 2; + if ((taskslot[total - 1].task = new TaskRequest) == NULL) + return -1; + if ((taskslot[total - 1].packet = new Packet) == NULL) + return -1; + taskslot[total - 1].seq = 0; + + return 0; +} + +int ClientResourcePool::Alloc(unsigned int &id, uint32_t &seq) +{ + if (freehead == 0) + { + if (Enlarge() < 0) + return -1; + } + + used++; + id = freehead; + freehead = taskslot[id].freenext; + if (freehead == 0) + freetail = 0; + taskslot[id].freenext = 0; + if (freetail != 0) + taskslot[freehead].freeprev = 0; + + if (taskslot[id].task == NULL) + { + if ((taskslot[id].task = new TaskRequest) == NULL) + { + used--; + if (freetail == 0) + freetail = id; + taskslot[id].freenext = freehead; + taskslot[freehead].freeprev = id; + freehead = id; + return -1; + } + } + if (taskslot[id].packet == NULL) + { + if ((taskslot[id].packet = new Packet) == NULL) + { + used--; + if (freetail == 0) + freetail = id; + taskslot[id].freenext = freehead; + taskslot[freehead].freeprev = id; + freehead = id; + return -1; + } + } + + taskslot[id].seq++; + seq = taskslot[id].seq; + + return 0; +} + +void ClientResourcePool::Clean(unsigned int id) +{ + if (taskslot[id].task) + taskslot[id].task->Clean(); + if (taskslot[id].packet) + taskslot[id].packet->Clean(); +} + +void ClientResourcePool::Free(unsigned int id, uint32_t seq) +{ + if (taskslot[id].seq != seq) + { + log_crit("client resource seq not right: %u, %u, not free", taskslot[id].seq, seq); + return; + } + + used--; + taskslot[freehead].freeprev = id; + taskslot[id].freenext = freehead; + freehead = id; + if (freetail == 0) + freetail = id; + +} + +int ClientResourcePool::Fill(unsigned int id) +{ + if (taskslot[id].task && taskslot[id].packet) + return 0; + + if ((taskslot[id].task == NULL) && ((taskslot[id].task = new TaskRequest) == NULL)) + return -1; + if ((taskslot[id].packet == NULL) && ((taskslot[id].packet = new Packet) == NULL)) + return -1; + + return 0; +} + +/* enlarge when freehead == 0, empty list */ +int ClientResourcePool::Enlarge() +{ + void *p; + unsigned int old = total; + + p = REALLOC(taskslot, sizeof(ClientResourceSlot) * 2 * total); + if (!p) + { + log_warning("enlarge taskslot failed, current CTaskRequestSlot %u", total); + return -1; + } + + total *= 2; + freehead = old; + freetail = total - 1; + + taskslot[old].freenext = old + 1; + taskslot[old].freeprev = 0; + if ((taskslot[old].task = new TaskRequest) == NULL) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + if ((taskslot[old].packet = new Packet) == NULL) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + taskslot[old].seq = 0; + + for (unsigned int i = old + 1; i < total - 1; i++) + { + taskslot[i].freenext = i + 1; + taskslot[i].freeprev = i - 1; + if ((taskslot[i].task = new TaskRequest) == NULL) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + if ((taskslot[i].packet = new Packet) == NULL) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + taskslot[i].seq = 0; + } + + taskslot[total - 1].freenext = 0; + taskslot[total - 1].freeprev = total - 2; + if (!(taskslot[total - 1].task = new TaskRequest)) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + if (!(taskslot[total - 1].packet = new Packet)) + { + log_warning("no mem for new task slot, current CTaskRequestSlot %u", total); + return -1; + } + taskslot[total - 1].seq = 0; + + return 0; +} + +void ClientResourcePool::Shrink() +{ + int shrink_cnt = (total - used) / 2; + int slot = freetail; + + if (freetail == 0) + return; + + for (int i = 0; i < shrink_cnt; i++) + { + if (taskslot[slot].task) + { + delete taskslot[slot].task; + taskslot[slot].task = NULL; + } + if (taskslot[slot].packet) + { + delete taskslot[slot].packet; + taskslot[slot].packet = NULL; + } + + slot = taskslot[slot].freeprev; + if (slot == 0) + break; + } +} diff --git a/src/search_local/index_storage/common/client_resource_pool.h b/src/search_local/index_storage/common/client_resource_pool.h new file mode 100644 index 0000000..b5598a2 --- /dev/null +++ b/src/search_local/index_storage/common/client_resource_pool.h @@ -0,0 +1,81 @@ +/* + * ===================================================================================== + * + * Filename: client_resource_pool.h + * + * Description: client resource pool. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef CLIENT_RESOURCE_POOL_H +#define CLIENT_RESOURCE_POOL_H + +#include + +class TaskRequest; +class Packet; +class ClientResourceSlot +{ +public: + ClientResourceSlot() : freenext(0), + freeprev(0), + task(NULL), + packet(NULL), + seq(0) + { + } + ~ClientResourceSlot(); + int freenext; + /*only used if prev slot free too*/ + int freeprev; + TaskRequest *task; + Packet *packet; + uint32_t seq; +}; + +/* automaticaly change size according to usage status */ +/* resource from pool need */ +class ClientResourcePool +{ +public: + ClientResourcePool() : total(0), + used(0), + freehead(0), + freetail(0), + taskslot(NULL) + { + } + ~ClientResourcePool(); + int Init(); + inline ClientResourceSlot *Slot(unsigned int id) { return &taskslot[id]; } + /* clean resource allocated */ + int Alloc(unsigned int &id, uint32_t &seq); + int Fill(unsigned int id); + void Clean(unsigned int id); + /* free, should clean first */ + void Free(unsigned int id, uint32_t seq); + +private: + unsigned int total; + unsigned int used; + unsigned int freehead; + unsigned int freetail; + ClientResourceSlot *taskslot; + + int Enlarge(); + void Shrink(); + inline int half_use() + { + return used <= total / 2 ? 1 : 0; + } +}; + +#endif diff --git a/src/search_local/index_storage/common/client_sync.cc b/src/search_local/index_storage/common/client_sync.cc new file mode 100644 index 0000000..8c5c8b8 --- /dev/null +++ b/src/search_local/index_storage/common/client_sync.cc @@ -0,0 +1,318 @@ +/* + * ===================================================================================== + * + * Filename: client_sync.cc + * + * Description: client sync. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "client_sync.h" +#include "task_request.h" +#include "packet.h" +#include "client_resource_pool.h" +#include "client_unit.h" +#include "log.h" + +#include + +class PollThread; +static int statEnable = 0; +static StatItemU32 statAcceptCount; +static StatItemU32 statCurConnCount; + +class ReplySync : public ReplyDispatcher +{ +public: + ReplySync(void) {} + virtual ~ReplySync(void) {} + virtual void reply_notify(TaskRequest *task); +}; + +void ReplySync::reply_notify(TaskRequest *task) +{ + ClientSync *cli = task->OwnerInfo(); + DTCDecoderUnit *resourceOwner = (DTCDecoderUnit *)task->resourceOwner; + + /* cli not exist, clean&free resource */ + if (cli == NULL) + { + if (task->resourceId != 0) + { + resourceOwner->clean_resource(task->resourceId); + resourceOwner->unregist_resource(task->resourceId, task->resourceSeq); + } + } + /* normal case */ + else if (cli->send_result() == 0) + { + cli->delay_apply_events(); + } + else /* send error, delete client, resource will be clean&free there */ + { + // delete client apply delete task + delete cli; + } +} + +static ReplySync syncReplyObject; + +ClientSync::ClientSync(DTCDecoderUnit *o, int fd, void *peer, int ps) : PollerObject(o->owner_thread(), fd), + owner(o), + receiver(fd), + stage(IdleState), + task(NULL), + reply(NULL) +{ + + addrLen = ps; + addr = MALLOC(ps); + resourceId = 0; + resource = NULL; + + if (addr == NULL) + throw(int) - ENOMEM; + + memcpy((char *)addr, (char *)peer, ps); + + if (!statEnable) + { + statAcceptCount = statmgr.get_item_u32(ACCEPT_COUNT); + statCurConnCount = statmgr.get_item_u32(CONN_COUNT); + statEnable = 1; + } + + /* ClientSync deleted if allocate resource failed. clean resource allocated */ + get_resource(); + if (resource) + { + task = resource->task; + reply = resource->packet; + } + else + { + throw(int) - ENOMEM; + } + rscStatus = RscClean; + + ++statAcceptCount; + ++statCurConnCount; +} + +ClientSync::~ClientSync() +{ + if (task) + { + if (stage == ProcReqState) + { + /* task in use, save resource to reply */ + task->clear_owner_info(); + task->resourceId = resourceId; + task->resourceOwner = owner; + task->resourceSeq = resourceSeq; + } + else + { + /* task not in use, clean resource, free resource */ + if (resource) + { + clean_resource(); + rscStatus = RscClean; + free_resource(); + } + } + } + + //DELETE(reply); + FREE_IF(addr); + --statCurConnCount; +} + +int ClientSync::Attach() +{ + enable_input(); + if (attach_poller() == -1) + return -1; + + attach_timer(owner->idle_list()); + + stage = IdleState; + return 0; +} + +//server peer +int ClientSync::recv_request() +{ + /* clean task from pool, basic init take place of old construction */ + if (RscClean == rscStatus) + { + task->set_data_table(owner->owner_table()); + task->set_hotbackup_table(owner->admin_table()); + task->set_role_as_server(); + task->BeginStage(); + receiver.erase(); + rscStatus = RscDirty; + } + + disable_timer(); + + int ret = task->Decode(receiver); + switch (ret) + { + default: + case DecodeFatalError: + if (errno != 0) + log_notice("decode fatal error, ret = %d msg = %m", ret); + return -1; + + case DecodeDataError: + task->response_timer_start(); + task->mark_as_hit(); + if (task->result_code() < 0) + log_notice("DecodeDataError, role=%d, fd=%d, result=%d", task->Role(), netfd, task->result_code()); + return send_result(); + + case DecodeIdle: + attach_timer(owner->idle_list()); + stage = IdleState; + rscStatus = RscClean; + break; + + case DecodeWaitData: + stage = RecvReqState; + break; + + case DecodeDone: + if ((ret = task->prepare_process()) < 0) + { + log_debug("build packed key error: %d", ret); + return send_result(); + } +#if 0 + /* 处理任务时，如果client关闭连接，server也应该关闭连接, 所以依然enable_input */ + DisableInput(); +#endif + disable_output(); + task->set_owner_info(this, 0, (struct sockaddr *)addr); + stage = ProcReqState; + task->push_reply_dispatcher(&syncReplyObject); + owner->task_dispatcher(task); + } + return 0; +} + +/* keep task in resource slot */ +int ClientSync::send_result(void) +{ + stage = SendRepState; + + owner->record_request_time(task); + + if (task->flag_keep_alive()) + task->versionInfo.set_keep_alive_timeout(owner->idle_time()); + reply->encode_result(task); + + return Response(); +} + +int ClientSync::Response(void) +{ + int ret = reply->Send(netfd); + + switch (ret) + { + case SendResultMoreData: + enable_output(); + return 0; + + case SendResultDone: + clean_resource(); + rscStatus = RscClean; + stage = IdleState; + disable_output(); + enable_input(); + attach_timer(owner->idle_list()); + return 0; + + default: + log_notice("send failed, return = %d, error = %m", ret); + return -1; + } + return 0; +} + +void ClientSync::input_notify(void) +{ + if (stage == IdleState || stage == RecvReqState) + { + if (recv_request() < 0) + { + + delete this; + return; + } + } + + /* receive input events again. */ + else + { + /* check whether client close connection. */ + if (check_link_status()) + { + log_debug("client close connection, delete ClientSync obj, stage=%d", stage); + delete this; + return; + } + else + { + DisableInput(); + } + } + + delay_apply_events(); +} + +void ClientSync::output_notify(void) +{ + if (stage == SendRepState) + { + if (Response() < 0) + delete this; + } + else + { + disable_output(); + log_info("Spurious ClientSync::output_notify, stage=%d", stage); + } +} + +int ClientSync::get_resource() +{ + return owner->regist_resource(&resource, resourceId, resourceSeq); +} + +void ClientSync::free_resource() +{ + task = NULL; + reply = NULL; + owner->unregist_resource(resourceId, resourceSeq); +} + +void ClientSync::clean_resource() +{ + owner->clean_resource(resourceId); +} diff --git a/src/search_local/index_storage/common/client_sync.h b/src/search_local/index_storage/common/client_sync.h new file mode 100644 index 0000000..f6cad3a --- /dev/null +++ b/src/search_local/index_storage/common/client_sync.h @@ -0,0 +1,80 @@ +/* + * ===================================================================================== + * + * Filename: client_sync.h + * + * Description: client sync. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CLIENT_SYNC_H__ +#define __CLIENT_SYNC_H__ + +#include "poller.h" +#include "timer_list.h" +#include "receiver.h" + +class DTCDecoderUnit; +class TaskRequest; +class Packet; +class ClientResourceSlot; + +class ClientSync : public PollerObject, private TimerObject +{ +public: + DTCDecoderUnit *owner; + void *addr; + int addrLen; + unsigned int resourceId; + ClientResourceSlot *resource; + uint32_t resourceSeq; + enum RscStatus + { + RscClean, + RscDirty, + }; + RscStatus rscStatus; + + ClientSync(DTCDecoderUnit *, int fd, void *, int); + virtual ~ClientSync(); + + virtual int Attach(void); + int send_result(void); + + virtual void input_notify(void); + +private: + virtual void output_notify(void); + + int get_resource(); + void free_resource(); + void clean_resource(); + +protected: + enum ClientState + { + IdleState, + RecvReqState, //wait for recv request, server side + SendRepState, //wait for send response, server side + ProcReqState, // IN processing + }; + + SimpleReceiver receiver; + ClientState stage; + TaskRequest *task; + Packet *reply; + + int recv_request(void); + int Response(void); + void adjust_events(void); +}; + +#endif diff --git a/src/search_local/index_storage/common/client_unit.cc b/src/search_local/index_storage/common/client_unit.cc new file mode 100644 index 0000000..3b974f2 --- /dev/null +++ b/src/search_local/index_storage/common/client_unit.cc @@ -0,0 +1,177 @@ +/* + * ===================================================================================== + * + * Filename: client_unit.cc + * + * Description: client uint class definition. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "client_unit.h" +#include "client_sync.h" +#include "client_async.h" +#include "client_dgram.h" +#include "poll_thread.h" +#include "task_request.h" +#include "log.h" + +DTCDecoderUnit::DTCDecoderUnit(PollThread *o, DTCTableDefinition **tdef, int it) : DecoderUnit(o, it), + tableDef(tdef), + output(o) +{ + + statRequestTime[0] = statmgr.get_sample(REQ_USEC_ALL); + statRequestTime[1] = statmgr.get_sample(REQ_USEC_GET); + statRequestTime[2] = statmgr.get_sample(REQ_USEC_INS); + statRequestTime[3] = statmgr.get_sample(REQ_USEC_UPD); + statRequestTime[4] = statmgr.get_sample(REQ_USEC_DEL); + statRequestTime[5] = statmgr.get_sample(REQ_USEC_FLUSH); + statRequestTime[6] = statmgr.get_sample(REQ_USEC_HIT); + statRequestTime[7] = statmgr.get_sample(REQ_USEC_REPLACE); + + if (clientResourcePool.Init() < 0) + throw(int) - ENOMEM; +} + +DTCDecoderUnit::~DTCDecoderUnit() +{ +} + +void DTCDecoderUnit::record_request_time(int hit, int type, unsigned int usec) +{ + static const unsigned char cmd2type[] = + { + /*Nop*/ 0, + /*result_code*/ 0, + /*DTCResultSet*/ 0, + /*HelperAdmin*/ 0, + /*Get*/ 1, + /*Purge*/ 5, + /*Insert*/ 2, + /*Update*/ 3, + /*Delete*/ 4, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Replace*/ 7, + /*Flush*/ 5, + /*Other*/ 0, + /*Other*/ 0, + }; + statRequestTime[0].push(usec); + unsigned int t = hit ? 6 : cmd2type[type]; + if (t) + statRequestTime[t].push(usec); +} + +void DTCDecoderUnit::record_request_time(TaskRequest *req) +{ + record_request_time(req->flag_is_hit(), req->request_code(), req->responseTimer.live()); +} + +int DTCDecoderUnit::process_stream(int newfd, int req, void *peer, int peerSize) +{ + if (req <= 1) + { + ClientSync *cli = NULL; + try + { + cli = new ClientSync(this, newfd, peer, peerSize); + } + catch (int err) + { + DELETE(cli); + return -1; + } + + if (0 == cli) + { + log_error("create CClient object failed, errno[%d], msg[%m]", errno); + return -1; + } + + if (cli->Attach() == -1) + { + log_error("Invoke CClient::Attach() failed"); + delete cli; + return -1; + } + + /* accept唤醒后立即recv */ + cli->input_notify(); + } + else + { + ClientAsync *cli = new ClientAsync(this, newfd, req, peer, peerSize); + + if (0 == cli) + { + log_error("create CClient object failed, errno[%d], msg[%m]", errno); + return -1; + } + + if (cli->Attach() == -1) + { + log_error("Invoke CClient::Attach() failed"); + delete cli; + return -1; + } + + /* accept唤醒后立即recv */ + cli->input_notify(); + } + return 0; +} + +int DTCDecoderUnit::process_dgram(int newfd) +{ + ClientDgram *cli = new ClientDgram(this, newfd); + + if (0 == cli) + { + log_error("create CClient object failed, errno[%d], msg[%m]", errno); + return -1; + } + + if (cli->Attach() == -1) + { + log_error("Invoke CClient::Attach() failed"); + delete cli; + return -1; + } + return 0; +} + +int DTCDecoderUnit::regist_resource(ClientResourceSlot **res, unsigned int &id, uint32_t &seq) +{ + if (clientResourcePool.Alloc(id, seq) < 0) + { + id = 0; + *res = NULL; + return -1; + } + + *res = clientResourcePool.Slot(id); + return 0; +} + +void DTCDecoderUnit::unregist_resource(unsigned int id, uint32_t seq) +{ + clientResourcePool.Free(id, seq); +} + +void DTCDecoderUnit::clean_resource(unsigned int id) +{ + clientResourcePool.Clean(id); +} diff --git a/src/search_local/index_storage/common/client_unit.h b/src/search_local/index_storage/common/client_unit.h new file mode 100644 index 0000000..b515ca8 --- /dev/null +++ b/src/search_local/index_storage/common/client_unit.h @@ -0,0 +1,61 @@ +/* + * ===================================================================================== + * + * Filename: client_unit.h + * + * Description: client uint class definition. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_CLIENT_UNIT_H__ +#define __H_DTC_CLIENT_UNIT_H__ + +#include "request_base_all.h" +#include "stat_dtc.h" +#include "decoder_base.h" +#include "stop_watch.h" +#include "client_resource_pool.h" + +class TaskRequest; +class DTCTableDefinition; + +class DTCDecoderUnit : public DecoderUnit +{ +public: + DTCDecoderUnit(PollThread *, DTCTableDefinition **, int); + virtual ~DTCDecoderUnit(); + + virtual int process_stream(int fd, int req, void *, int); + virtual int process_dgram(int fd); + + void bind_dispatcher(TaskDispatcher *p) { output.bind_dispatcher(p); } + void task_dispatcher(TaskRequest *task) { output.task_notify(task); } + DTCTableDefinition *owner_table(void) const { return tableDef[0]; } + DTCTableDefinition *admin_table(void) const { return tableDef[1]; } + + void record_request_time(int hit, int type, unsigned int usec); + // TaskRequest *p must nonnull + void record_request_time(TaskRequest *p); + void record_rcv_cnt(void); + void record_snd_cnt(void); + int regist_resource(ClientResourceSlot **res, unsigned int &id, uint32_t &seq); + void unregist_resource(unsigned int id, uint32_t seq); + void clean_resource(unsigned int id); + +private: + DTCTableDefinition **tableDef; + RequestOutput output; + ClientResourcePool clientResourcePool; + + StatSample statRequestTime[8]; +}; + +#endif diff --git a/src/search_local/index_storage/common/compiler.h b/src/search_local/index_storage/common/compiler.h new file mode 100644 index 0000000..9b789c5 --- /dev/null +++ b/src/search_local/index_storage/common/compiler.h @@ -0,0 +1,50 @@ +/* + * ===================================================================================== + * + * Filename: compiler.h + * + * Description: macro definition. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __HIDDEN + +#if __GNUC__ >= 4 + +#define __HIDDEN __attribute__((__visibility__("hidden"))) +#define __EXPORT __attribute__((__visibility__("default"))) +#define __INLINE inline __attribute__((__always_inline__)) +#define HAS_TLS 1 + +#if __PIC__ || __pic__ +#define __TLS_MODEL __attribute__((tls_model("initial-exec"))) +#else +#define __TLS_MODEL __attribute__((tls_model("local-exec"))) +#endif + +#else // __GNUC__ +#define __HIDDEN /* */ +#define __EXPORT /* */ +#define __INLINE inline +#endif // __GNUC__ + +#ifndef likely +#if __GCC_MAJOR >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif +#endif + +#endif // __HIDDEN diff --git a/src/search_local/index_storage/common/compress.cc b/src/search_local/index_storage/common/compress.cc new file mode 100644 index 0000000..5fe6629 --- /dev/null +++ b/src/search_local/index_storage/common/compress.cc @@ -0,0 +1,97 @@ +/* + * ===================================================================================== + * + * Filename: compress.cc + * + * Description: add field compression. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "compress.h" +#include "zlib.h" +#include "mem_check.h" +#include +#include +#include + +DTCCompress::DTCCompress() +{ + _level = 1; + _buflen = 0; + _len = 0; + _buf = NULL; +} + +DTCCompress::~DTCCompress() +{ + if (_buf) + FREE(_buf); +} + +void DTCCompress::set_compress_level(int level) +{ + _level = level < 1 ? 1 : (level > 9 ? 9 : level); +} +int DTCCompress::set_buffer_len(unsigned long len) +{ + if (_buf == NULL) + { + _buflen = len; + _buf = (unsigned char *)MALLOC(len); + } + else if (_buflen < len) + { + _buflen = len; + FREE(_buf); + _buf = (unsigned char *)MALLOC(len); + } + if (_buf == NULL) + return -ENOMEM; + return 0; +} + +//source 被压缩的缓冲区 sourcelen 被压缩缓冲区的原始长度 +//dest 压缩后的缓冲区 destlen 被压缩后的缓冲区长度 +//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度 +int DTCCompress::compress(const char *source, unsigned long sourceLen) +{ + if (_buf == NULL || source == NULL) + { + return -111111; + } + _len = _buflen; + return compress2(_buf, &_len, (Bytef *)source, sourceLen, _level); +} + +//source 待解压的缓冲区 sourcelen 待解压缓冲区的原始长度 +//dest 解压后的缓冲区 destlen 解缩后的缓冲区长度 +//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度 +int DTCCompress::UnCompress(char **buf, int *lenp, const char *source, unsigned long sourceLen) +{ + if (_buf == NULL || source == NULL) + { + snprintf(_errmsg, sizeof(_errmsg), "input buffer or uncompress buffer is null"); + return -111111; + } + _len = _buflen; + int iret = uncompress(_buf, &_len, (Bytef *)source, sourceLen); + if (iret) + { + snprintf(_errmsg, sizeof(_errmsg), "uncompress error,error code is:%d.check it in /usr/include/zlib.h", iret); + return -111111; + } + *buf = (char *)MALLOC(_len); + if (*buf == NULL) + return -ENOMEM; + memcpy(*buf, _buf, _len); + *lenp = _len; + return 0; +} diff --git a/src/search_local/index_storage/common/compress.h b/src/search_local/index_storage/common/compress.h new file mode 100644 index 0000000..fe6d0fe --- /dev/null +++ b/src/search_local/index_storage/common/compress.h @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: compress.h + * + * Description: add field compression. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +class DTCCompress +{ +public: + DTCCompress(); + virtual ~DTCCompress(); + + void set_compress_level(int level); + int set_buffer_len(unsigned long len); + const char *error_message(void) const { return _errmsg; } + + //source 被压缩的缓冲区 sourcelen 被压缩缓冲区的原始长度 + //dest 压缩后的缓冲区 destlen 被压缩后的缓冲区长度 + //注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度 + int compress(const char *source, unsigned long sourceLen); + + //source 待解压的缓冲区 sourcelen 待解压缓冲区的原始长度 + //dest 解压后的缓冲区 destlen 解缩后的缓冲区长度 + //注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度 + int UnCompress(char **dest, int *destlen, const char *source, unsigned long sourceLen); + + unsigned long get_len(void) { return _len; } + char *get_buf(void) { return (char *)_buf; } + char _errmsg[512]; + +private: + unsigned long _buflen; + unsigned char *_buf; + unsigned long _len; + int _level; +}; diff --git a/src/search_local/index_storage/common/config.cc b/src/search_local/index_storage/common/config.cc new file mode 100644 index 0000000..191fff5 --- /dev/null +++ b/src/search_local/index_storage/common/config.cc @@ -0,0 +1,553 @@ +/* + * ===================================================================================== + * + * Filename: config.cc + * + * Description: configuration acquiry and store. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mem_check.h" +#include "config.h" +#include "log.h" + +int str2int(const char *strval, int def) +{ + int ret_code = def; + if (isdigit(strval[0]) || (strval[0] == '-' && isdigit(strval[1]))) + return atoi(strval); + + if (!strcasecmp(strval, "On")) + ret_code = 1; + else if (!strcasecmp(strval, "Off")) + ret_code = 0; + else if (!strcasecmp(strval, "Yes")) + ret_code = 1; + else if (!strcasecmp(strval, "No")) + ret_code = 0; + else if (!strcasecmp(strval, "True")) + ret_code = 1; + else if (!strcasecmp(strval, "False")) + ret_code = 0; + else if (!strcasecmp(strval, "Enable")) + ret_code = 1; + else if (!strcasecmp(strval, "Disable")) + ret_code = 0; + else if (!strcasecmp(strval, "Enabled")) + ret_code = 1; + else if (!strcasecmp(strval, "Disabled")) + ret_code = 0; + + return ret_code; +} + +void DTCConfig::Dump(FILE *fp, bool dec) +{ + fprintf(fp, "##### ORIGINAL FILE %s #####\n", filename.c_str()); + for (secmap_t::iterator i = smap.begin(); i != smap.end(); i++) + { + const char *sec = i->first.c_str(); + keymap_t &kmap = i->second; + if (kmap.size() <= 0) + continue; + fprintf(fp, "\n[%s]\n", sec); + for (keymap_t::iterator j = kmap.begin(); j != kmap.end(); j++) + { + const char *key = j->first.c_str(); + const char *val = j->second.c_str(); + char flag = val[0]; + if (flag == '\0') + { + if (dec == true) + fprintf(fp, "# %s NOT SET\n", key); + } + else + { + val++; + if (flag == ' ' || dec == false) + fprintf(fp, "%s = %s\n", key, val); + else + fprintf(fp, "%c %s = %s\n", flag, key, val); + } + } + } +} + +int DTCConfig::Dump(const char *fn, bool dec) +{ + FILE *fp = fopen(fn, "w"); + if (fp == NULL) + return -1; + Dump(fp, dec); + fclose(fp); + return 0; +} + +static char *skip_blank(char *p) +{ + while (isblank(*p)) + p++; + return p; +} + +int DTCConfig::parse_buffered_config(char *buf, const char *fn, const char *defsec, bool bakconfig) +{ + int len = strlen(buf), ln = 0, ret_code = 0; + char *start = buf, *end = buf; + + if (defsec == NULL) + defsec = ""; + if (fn == NULL) + fn = "buffered config"; + keymap_t *kmap = &smap[defsec]; + + for (start = buf, ln = 0; end && buf + len > start; start = end + 1) + { + end = strchr(start, '\n'); + if (end) + { + if (*end) + *end = '\0'; + if (end > start && end[-1] == '\r') + end[-1] = '\0'; + } + + std::string val("?"); + keymap_t *m = kmap; + + ln++; + + char *v = NULL; + char *key = skip_blank(start); + // blank or comment line + if (key[0] == '\0' || key[0] == '#' || key[0] == '?' || key[0] == ';') + continue; + // key must printable + if (!isprint(key[0])) + { + log_warning("%s(%d): invalid line: %s", fn, ln, key); + ret_code = -1; + continue; + } + + // find the equation + start = key + strcspn(key, "= \t"); + if (*start != '\0') + { + char *p = start[0] == '=' ? start : skip_blank(start + 1); + if (*p == '=') + { + v = skip_blank(p + 1); + } + else if (key[0] == '[') + /*OK*/; + else + { + log_warning("%s(%d): invalid line: %s", fn, ln, key); + ret_code = -1; + continue; + } + *start = '\0'; + } + else if (key[0] != '[') + { + log_warning("%s(%d): invalid line: %s", fn, ln, key); + ret_code = -1; + continue; + } + + if (key[0] == '[') + { + char *r = strchr(key, ']'); + if (!r) + { + log_warning("%s(%d): invalid section: %s", fn, ln, key); + ret_code = -1; + continue; + } + else if (!((r[1] == '\0' && v == NULL) || (r[1] == '.' && v != NULL))) + { + log_warning("%s(%d): invalid section: %s", fn, ln, key); + ret_code = -1; + continue; + } + else + { + *r = '\0'; + m = &smap[key + 1]; + if (r[1] == '\0') + { + kmap = m; + continue; + } + key = r + 2; + } + } + + if (v == NULL) + continue; + + switch (v[0]) + { + case '(': + start = strchr(v, ')'); + if (start == NULL) + goto error; + break; + case '[': + start = strchr(v, ']'); + if (start == NULL) + goto error; + break; + case '{': + start = strchr(v, '}'); + if (start == NULL) + goto error; + break; + + case '"': + start = strrchr(v + 1, '"'); + if (start == NULL) + goto error; + break; + case '\'': + start = strrchr(v + 1, '\''); + if (start == NULL) + goto error; + break; + default: + start = end ? end - 1 : v + strlen(v) - 1; + if (*start == '\0') + start--; + while (start > v && isblank(*start)) + start--; + break; + error: + log_warning("%s(%d): unended quote or bracket", fn, ln); + ret_code = -1; + continue; + } + start[1] = '\0'; + + if (v[0]) + val.append(v); + + (*m)[key] = val; + } + + if (bakconfig) + { + char bak_config[1024]; + int err; + snprintf(bak_config, sizeof(bak_config), "cp %s ../stat/", fn); + err = system(bak_config); + } + return ret_code; +} + +int DTCConfig::parse_config(const char *fn, const char *defsec, bool bakconfig) +{ + char *buf; + int len, fd, ret_code = -1; + + if (fn == NULL) + fn = filename.c_str(); + else if (filename.size() <= 0) + { + filename = fn; + } + if (filename[0] == '\0') + return -1; + + fd = open(fn, O_RDONLY); + if (fd < 0) + { + log_warning("open %s error, cwd=%s, %m\n", fn, get_current_dir_name()); + return -1; + } + len = lseek(fd, 0L, SEEK_END); + lseek(fd, 0L, SEEK_SET); + buf = (char *)MALLOC(len + 1); + int unused; + unused = read(fd, buf, len); + buf[len] = '\0'; + ret_code = len + 1; + close(fd); + ret_code = parse_buffered_config(buf, fn, defsec, bakconfig); + FREE(buf); + return ret_code; +} + +bool DTCConfig::has_section(const char *sec) +{ + secmap_t::iterator n = smap.find(sec); + return n != smap.end() && n->second.size() > 0 ? true : false; +} + +bool DTCConfig::has_key(const char *sec, const char *key) +{ + secmap_t::iterator n = smap.find(sec); + if (n == smap.end()) + return false; + keymap_t &kmap = n->second; + keymap_t::iterator m = kmap.find(key); + return m != kmap.end() && m->second.size() > 0 ? true : false; +} + +const char *DTCConfig::get_str_val(const char *sec, const char *key) +{ + keymap_t &kmap = smap[sec]; + std::string &v = kmap[key]; + if (v.size() == 0) + return NULL; + v[0] = ' '; + return v.c_str() + 1; +} + +int DTCConfig::get_int_val(const char *sec, const char *key, int def) +{ + const char *val = get_str_val(sec, key); + if (val == NULL) + return def; + + return str2int(val, def); +} + +unsigned long long DTCConfig::get_size_val( + const char *sec, + const char *key, + unsigned long long def, + char unit) +{ + const char *val = get_str_val(sec, key); + if (val == NULL || !isdigit(val[0])) + return def; + + const char *p; + double a = strtod(val, (char **)&p); + if (*p) + unit = *p; + switch (unit) + { + case 'b': + break; + case 'B': + break; + case 'k': + a *= 1000; + break; + case 'K': + a *= 1 << 10; + break; + case 'm': + a *= 1000000; + break; + case 'M': + a *= 1 << 20; + break; + case 'g': + a *= 1000000000; + break; + case 'G': + a *= 1 << 30; + break; + case 't': + a *= 1000000000000LL; + break; + case 'T': + a *= 1ULL << 40; + break; + } + + return (unsigned long long)a; +} + +int DTCConfig::get_idx_val( + const char *sec, const char *key, + const char *const *array, int nDefault) +{ + const char *val = get_str_val(sec, key); + if (val == NULL) + return nDefault; + + if (isdigit(val[0])) + { + char *p; + int n = strtol(val, &p, 0); + if (*p == '\0') + { + for (int i = 0; array[i]; i++) + { + if (n == i) + return i; + } + } + } + + for (int i = 0; array[i]; i++) + { + if (!strcasecmp(val, array[i])) + return i; + } + return -1; +} + +class AutoConfigSection : public AutoConfig +{ +private: + static pthread_mutex_t glock; + void GlobalLock(void) { pthread_mutex_lock(&glock); } + void GlobalUnlock(void) { pthread_mutex_unlock(&glock); } + +private: + DTCConfig *parent; + char *section; + // buf must have enough room place composed key name + char buf[256]; + char *last; + +public: + AutoConfigSection(DTCConfig *p, const char *sec); + ~AutoConfigSection(); + + virtual int get_int_val(const char *key, const char *inst, int def = 0); + virtual unsigned long long get_size_val(const char *key, const char *inst, unsigned long long def = 0, char unit = 0); + virtual int get_idx_val(const char *, const char *, const char *const *, int = 0); + virtual const char *get_str_val(const char *key, const char *inst); + +private: + // return composed key, or vanilla key, always non-null + const char *findkey(const char *key, const char *inst); + // strip suffix digits + int stripnumber(void); + // strip suffix alphaphetic + int stripalpha(void); + // strip suffix punct + int strippunct(void); +}; + +pthread_mutex_t AutoConfigSection::glock = PTHREAD_MUTEX_INITIALIZER; + +AutoConfigSection::AutoConfigSection(DTCConfig *p, const char *sec) +{ + this->parent = p; + this->section = STRDUP(sec); +} + +AutoConfigSection::~AutoConfigSection() +{ + FREE_CLEAR(section); +} + +int AutoConfigSection::stripnumber(void) +{ + int n = 0; + while (last >= buf && isdigit(*last)) + { + last--; + n++; + } + last[1] = 0; + strippunct(); + return n; +} + +int AutoConfigSection::stripalpha(void) +{ + int n = 0; + while (last >= buf && isalpha(*last)) + { + last--; + n++; + } + last[1] = 0; + strippunct(); + return n; +} + +int AutoConfigSection::strippunct(void) +{ + int n = 0; + while (last >= buf && *last != '@' && !isalnum(*last)) + { + last--; + n++; + } + last[1] = 0; + return n; +} + +const char *AutoConfigSection::findkey(const char *key, const char *inst) +{ + snprintf(buf, sizeof(buf), "%s@%s", key, inst); + last = buf + strlen(buf) - 1; + strippunct(); + + do + { + if (parent->get_str_val(section, buf) != NULL) + { + return buf; + } + } while (isdigit(*last) ? stripnumber() : stripalpha()); + + return key; +} + +int AutoConfigSection::get_int_val(const char *key, const char *inst, int def) +{ + int ret; + GlobalLock(); + ret = parent->get_int_val(section, findkey(key, inst), def); + GlobalUnlock(); + return ret; +} +unsigned long long AutoConfigSection::get_size_val(const char *key, const char *inst, unsigned long long def, char unit) +{ + unsigned long long ret; + GlobalLock(); + ret = parent->get_size_val(section, findkey(key, inst), def, unit); + GlobalUnlock(); + return ret; +} +int AutoConfigSection::get_idx_val(const char *key, const char *inst, const char *const *idxval, int def) +{ + int ret; + GlobalLock(); + ret = parent->get_idx_val(section, findkey(key, inst), idxval, def); + GlobalUnlock(); + return ret; +} +const char *AutoConfigSection::get_str_val(const char *key, const char *inst) +{ + const char *ret; + GlobalLock(); + ret = parent->get_str_val(section, findkey(key, inst)); + GlobalUnlock(); + return ret; +} + +AutoConfig *DTCConfig::get_auto_config_instance(const char *section) +{ + AutoConfigSection *inst; + NEW(AutoConfigSection(this, section), inst); + return inst; +} diff --git a/src/search_local/index_storage/common/config.h b/src/search_local/index_storage/common/config.h new file mode 100644 index 0000000..0fa527a --- /dev/null +++ b/src/search_local/index_storage/common/config.h @@ -0,0 +1,72 @@ +/* + * ===================================================================================== + * + * Filename: config.h + * + * Description: configuration acquiry and store. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CONFIG_H__ +#define __CONFIG_H__ + +#include +#include +#include +#include + +class AutoConfig +{ +public: + AutoConfig(){}; + virtual ~AutoConfig(){}; + virtual int get_int_val(const char *key, const char *inst, int def = 0) = 0; + virtual unsigned long long get_size_val(const char *key, const char *inst, unsigned long long def = 0, char unit = 0) = 0; + virtual int get_idx_val(const char *, const char *, const char *const *, int = 0) = 0; + virtual const char *get_str_val(const char *key, const char *inst) = 0; +}; + +class DTCConfig +{ +public: + DTCConfig(){}; + ~DTCConfig(){}; + + AutoConfig *get_auto_config_instance(const char *); + int get_int_val(const char *sec, const char *key, int def = 0); + unsigned long long get_size_val(const char *sec, const char *key, unsigned long long def = 0, char unit = 0); + int get_idx_val(const char *, const char *, const char *const *, int = 0); + const char *get_str_val(const char *sec, const char *key); + bool has_section(const char *sec); + bool has_key(const char *sec, const char *key); + void Dump(FILE *fp, bool dec = false); + int Dump(const char *fn, bool dec = false); + int parse_config(const char *f = 0, const char *s = 0, bool bakconfig = false); + int parse_buffered_config(char *buf, const char *fn = 0, const char *s = 0, bool bakconfig = false); + +private: + struct nocase + { + bool operator()(const std::string &a, const std::string &b) const + { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } + }; + + typedef std::map keymap_t; + typedef std::map secmap_t; + +private: + std::string filename; + secmap_t smap; +}; + +#endif diff --git a/src/search_local/index_storage/common/consistent_hash_selector.cc b/src/search_local/index_storage/common/consistent_hash_selector.cc new file mode 100644 index 0000000..5621470 --- /dev/null +++ b/src/search_local/index_storage/common/consistent_hash_selector.cc @@ -0,0 +1,63 @@ +/* + * ===================================================================================== + * + * Filename: consistent_hash_selector.cc + * + * Description: consistent hash selector. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include + +#include + +#include "consistent_hash_selector.h" +#include "log.h" + +const std::string &ConsistentHashSelector::Select(uint32_t hash) +{ + static std::string empty; + if (m_nodes.empty()) + return empty; + + std::map::iterator iter = m_nodes.upper_bound(hash); + if (iter != m_nodes.end()) + return m_nodeNames[iter->second]; + return m_nodeNames[m_nodes.begin()->second]; +} + +void ConsistentHashSelector::add_node(const char *name) +{ + if (find(m_nodeNames.begin(), m_nodeNames.end(), name) != m_nodeNames.end()) + { + log_crit("duplicate node name: %s in ClusterConfig", name); + abort(); + } + m_nodeNames.push_back(name); + int index = m_nodeNames.size() - 1; + char *buf = (char *)alloca(strlen(name) + 16); + for (int i = 0; i < VIRTUAL_NODE_COUNT; ++i) + { + snprintf(buf, strlen(name) + 16, "%s#%d", name, i); + uint32_t value = Hash(buf, strlen(buf)); + std::map::iterator iter = m_nodes.find(value); + if (iter != m_nodes.end()) + { + //hash值冲突，选取字符串中较小者 + if (m_nodeNames[iter->second] < name) + continue; + } + m_nodes[value] = index; + } +} diff --git a/src/search_local/index_storage/common/consistent_hash_selector.h b/src/search_local/index_storage/common/consistent_hash_selector.h new file mode 100644 index 0000000..3e9e02e --- /dev/null +++ b/src/search_local/index_storage/common/consistent_hash_selector.h @@ -0,0 +1,48 @@ +/* + * ===================================================================================== + * + * Filename: consistent_hash_selector.h + * + * Description: consistent hash selector. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef CONSISTENT_HASH_SELECTOR_H__ +#define CONSISTENT_HASH_SELECTOR_H__ + +#include +#include +#include +#include + +#include "chash.h" + +class ConsistentHashSelector +{ +public: + uint32_t Hash(const char *key, int len) { return chash(key, len); } + const std::string &Select(uint32_t hash); + const std::string &Select(const char *key, int len) { return Select(Hash(key, len)); } + + static const int VIRTUAL_NODE_COUNT = 100; + void add_node(const char *name); + void Clear() + { + m_nodes.clear(); + m_nodeNames.clear(); + } + +private: + std::map m_nodes; + std::vector m_nodeNames; +}; + +#endif diff --git a/src/search_local/index_storage/common/container.h b/src/search_local/index_storage/common/container.h new file mode 100644 index 0000000..0c4b8e9 --- /dev/null +++ b/src/search_local/index_storage/common/container.h @@ -0,0 +1,64 @@ +/* + * ===================================================================================== + * + * Filename: container.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_CONTAINER_HEADER__ +#define __DTC_CONTAINER_HEADER__ + +class DTCTableDefinition; +class TaskRequest; +class NCRequest; +class NCResultInternal; +union DTCValue; + +#if GCC_MAJOR >= 4 +#pragma GCC visibility push(default) +#endif +class IInternalService +{ +public: + virtual ~IInternalService(void) {} + + virtual const char *query_version_string(void) = 0; + virtual const char *query_service_type(void) = 0; + virtual const char *query_instance_name(void) = 0; +}; + +class IDTCTaskExecutor +{ +public: + virtual ~IDTCTaskExecutor(void) {} + + virtual NCResultInternal *task_execute(NCRequest &, const DTCValue *) = 0; +}; + +class IDTCService : public IInternalService +{ +public: + virtual ~IDTCService(void) {} + + virtual DTCTableDefinition *query_table_definition(void) = 0; + virtual DTCTableDefinition *query_admin_table_definition(void) = 0; + virtual IDTCTaskExecutor *query_task_executor(void) = 0; + virtual int match_listening_ports(const char *, const char * = 0) = 0; +}; +#if GCC_MAJOR >= 4 +#pragma GCC visibility pop +#endif + +IInternalService *query_internal_service(const char *name, const char *instance); + +#endif diff --git a/src/search_local/index_storage/common/daemon.cc b/src/search_local/index_storage/common/daemon.cc new file mode 100644 index 0000000..5237b2a --- /dev/null +++ b/src/search_local/index_storage/common/daemon.cc @@ -0,0 +1,257 @@ +/* + * ===================================================================================== + * + * Filename: daemon.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "table_def.h" +#include "admin_tdef.h" +#include "daemon.h" +#include "log.h" +#include +#include "dbconfig.h" +#include "version.h" +#include "table_def_manager.h" + +DTCConfig *gConfig = NULL; +DTCTableDefinition *gTableDef[2] = {NULL, NULL}; +DbConfig *dbConfig = NULL; + +volatile int stop = 0; +volatile int crash_signo = 0; +int background = 1; +extern const char progname[]; +extern const char usage_argv[]; + +pthread_t mainthreadid; + +bool flatMode; + +void ShowVersionDetail() +{ + printf("%s version: %s\n", progname, version_detail); +} + +void ShowCompDate() +{ + printf("%s compile date: %s %s\n", progname, compdatestr, comptimestr); +} + +void show_usage() +{ + ShowVersionDetail(); + ShowCompDate(); + printf("Usage:\n %s [-d] [-h] [-v] [-V]%s\n", progname, usage_argv); +} + +int dtc_daemon_init(int argc, char **argv) +{ + int c; + + while ((c = getopt(argc, argv, "df:t:hvV")) != -1) + { + switch (c) + { + case 'd': + background = 0; + break; + case 'f': + strncpy(cacheFile, optarg, sizeof(cacheFile) - 1); + break; + case 't': + strncpy(tableFile, optarg, sizeof(tableFile) - 1); + break; + case 'h': + show_usage(); + exit(0); + case 'v': + ShowVersionDetail(); + ShowCompDate(); + exit(0); + case 'V': + ShowVersionDetail(); + ShowCompDate(); + exit(0); + case '?': + show_usage(); + exit(-1); + } + } + + _init_log_("dtcd"); + log_info("%s v%s: starting....", progname, version); + + gConfig = new DTCConfig; + //load config file and copy it to ../stat + if (gConfig->parse_config(tableFile, "DB_DEFINE", true) == -1) + return -1; + + if (gConfig->parse_config(cacheFile, "cache", true)) + return -1; + + _set_log_level_(gConfig->get_idx_val("cache", "LogLevel", + ((const char *const[]){ + "emerg", + "alert", + "crit", + "error", + "warning", + "notice", + "info", + "debug", + NULL}), + 6)); + _set_log_thread_name_(progname); + _init_log_alerter_(); + + dbConfig = DbConfig::Load(gConfig); + if (dbConfig == NULL) + return -1; + + gTableDef[0] = dbConfig->build_table_definition(); + if (gTableDef[0] == NULL) + return -1; + + DTCTableDefinition *t; + if ((t = TableDefinitionManager::Instance()->load_table(tableFile)) == NULL) + return -1; + TableDefinitionManager::Instance()->set_cur_table_def(t, 0); + + gTableDef[1] = build_hot_backup_table(); + if (gTableDef[1] == NULL) + return -1; + + if (!TableDefinitionManager::Instance()->build_hot_backup_table_def()) + return -1; + + return 0; +} + +static void sigterm_handler(int signo) +{ + stop = 1; +} + +int daemon_start() +{ + struct sigaction sa; + sigset_t sset; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sigterm_handler; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + + signal(SIGPIPE, SIG_IGN); + signal(SIGCHLD, SIG_IGN); + + sigemptyset(&sset); + sigaddset(&sset, SIGTERM); + sigaddset(&sset, SIGSEGV); + sigaddset(&sset, SIGBUS); + sigaddset(&sset, SIGABRT); + sigaddset(&sset, SIGILL); + sigaddset(&sset, SIGCHLD); + sigaddset(&sset, SIGFPE); + sigprocmask(SIG_UNBLOCK, &sset, &sset); + + int ret = background ? daemon(1, 1) : 0; + mainthreadid = pthread_self(); + return ret; +} + +void DaemonCrashed(int signo) +{ + stop = 1; + crash_signo = signo; + if (mainthreadid) + pthread_kill(mainthreadid, SIGTERM); +} + +void daemon_cleanup(void) +{ +#if MEMCHECK + DELETE(gBConfig); + DELETE(gConfig); + DELETE(gTableDef[0]); + DELETE(gTableDef[1]); + if (dbConfig) + dbConfig->Destroy(); +#endif + // CrashProtect: special retval 85 means protected-crash + if (crash_signo) + exit(85); +} + +int daemon_get_fd_limit(void) +{ + struct rlimit rlim; + if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) + { + log_notice("Query FdLimit failed, errmsg[%s]", strerror(errno)); + return -1; + } + + return rlim.rlim_cur; +} + +int daemon_set_fd_limit(int maxfd) +{ + struct rlimit rlim; + if (maxfd) + { + /* raise open files */ + rlim.rlim_cur = maxfd; + rlim.rlim_max = maxfd; + if (setrlimit(RLIMIT_NOFILE, &rlim) == -1) + { + log_notice("Increase FdLimit failed, set val[%d] errmsg[%s]", maxfd, strerror(errno)); + return -1; + } + } + return 0; +} + +int daemon_enable_core_dump(void) +{ + struct rlimit rlim; + + /* allow core dump 100M */ + rlim.rlim_cur = 100UL << 20; + rlim.rlim_max = 100UL << 20; + + if (setrlimit(RLIMIT_CORE, &rlim) == -1) + { + if (getrlimit(RLIMIT_CORE, &rlim) == 0) + { + rlim.rlim_cur = rlim.rlim_max; + setrlimit(RLIMIT_CORE, &rlim); + } + + log_notice("EnableCoreDump size[%ld]", (long)rlim.rlim_max); + } + + return 0; +} diff --git a/src/search_local/index_storage/common/daemon.h b/src/search_local/index_storage/common/daemon.h new file mode 100644 index 0000000..14a2353 --- /dev/null +++ b/src/search_local/index_storage/common/daemon.h @@ -0,0 +1,50 @@ +/* + * ===================================================================================== + * + * Filename: daemon.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_DAEMON__H__ +#define __DTC_DAEMON__H__ + +#define MAXLISTENERS 10 + +class DTCConfig; +struct DbConfig; +class DTCTableDefinition; + +extern DTCConfig *gConfig; +extern DbConfig *dbConfig; +extern DTCTableDefinition *gTableDef[]; + +extern volatile int stop; +extern int background; + +extern const char progname[]; +extern const char version[]; +extern const char usage_argv[]; +extern char cacheFile[256]; +extern char tableFile[256]; +extern void show_version(void); +extern void show_usage(void); +extern int dtc_daemon_init(int argc, char **argv); +extern int bmp_daemon_init(int argc, char **argv); +extern int daemon_start(void); +extern void daemon_wait(void); +extern void daemon_cleanup(void); +extern int daemon_get_fd_limit(void); +extern int daemon_set_fd_limit(int maxfd); +extern int daemon_enable_core_dump(void); +extern unsigned int scan_process_openning_fd(void); +#endif diff --git a/src/search_local/index_storage/common/daemon_wait.cc b/src/search_local/index_storage/common/daemon_wait.cc new file mode 100644 index 0000000..d90bed8 --- /dev/null +++ b/src/search_local/index_storage/common/daemon_wait.cc @@ -0,0 +1,91 @@ +/* + * ===================================================================================== + * + * Filename: daemon_wait.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "log.h" +#include "daemon.h" +#include "mem_check.h" +#include +#include "thread_cpu_stat.h" + +void daemon_wait(void) +{ + StatItemU32 statfd = statmgr.get_item_u32(SERVER_OPENNING_FD); + statfd = 0; + + unsigned fdthreshold, fdlimit = daemon_get_fd_limit(); + if (fdlimit < 0) + fdlimit = 1024; + fdthreshold = (fdlimit * 80) / 100; + + sleep(10); + thread_cpu_stat cpu_stat; + cpu_stat.init(); + while (!stop) + { + sleep(10); + if (stop) + break; + + cpu_stat.do_stat(); + + /* 扫描进程打开的fd句柄数，如果超过配置阈值，向二级网管告警 */ + statfd = scan_process_openning_fd(); + if ((unsigned)statfd > fdthreshold) + { + log_alert("openning fd overload[current=%d threshold=%d max=%d]", + (unsigned)statfd, fdthreshold, fdlimit); + } +#if MEMCHECK + log_debug("memory allocated %lu virtual %lu", count_alloc_size(), count_virtual_size()); +#endif + } +} + +/* 扫描进程已打开的文件句柄数 */ +unsigned int scan_process_openning_fd(void) +{ + unsigned int count = 0; + char fd_dir[1024] = {0}; + DIR *dir; + struct dirent *ptr; + + snprintf(fd_dir, sizeof(fd_dir), "/proc/%d/fd", getpid()); + + if ((dir = opendir(fd_dir)) == NULL) + { + log_warning("open dir :%s failed, msg:%s", fd_dir, strerror(errno)); + return count; + } + + while ((ptr = readdir(dir)) != NULL) + { + if (strcasecmp(ptr->d_name, "..") != 0 && strcasecmp(ptr->d_name, ".") != 0) + count++; + } + + closedir(dir); + return count; +} diff --git a/src/search_local/index_storage/common/dbconfig.cc b/src/search_local/index_storage/common/dbconfig.cc new file mode 100644 index 0000000..28690c0 --- /dev/null +++ b/src/search_local/index_storage/common/dbconfig.cc @@ -0,0 +1,1223 @@ +/* + * ===================================================================================== + * + * Filename: dbconfig.cc + * + * Description: database config operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "value.h" +#include "protocol.h" +#include "log.h" +#include "dbconfig.h" +#include "table_def.h" +#include "config.h" +#include + +#define HELPERPATHFORMAT "@dtcd[%d]helper%d%c" +#define DTC_EXPIRE_TIME_FIELD "_dtc_sys_expiretime" + +static int +ParseDbLine(const char *buf, uint16_t *dbIdx) +{ + char *p = (char *)buf; // discard const + if (*p++ != '[') + return -1; + + int n = 0; + while (*p) + { + if (!isdigit(p[0])) + break; + int begin, end; + begin = strtol(p, &p, 0); + if (*p != '-') + end = begin; + else + { + p++; + if (!isdigit(p[0])) + break; + end = strtol(p, &p, 0); + } + while (begin <= end) + dbIdx[n++] = begin++; + + if (p[0] != ',') + break; + else + p++; + } + if (p[0] != ']') + return -1; + return n; +} + +/* 前置空格已经过滤了 */ +static char *skip_blank(char *p) +{ + char *iter = p; + while (!isspace(*iter) && *iter != '\0') + ++iter; + + *iter = '\0'; + return p; +} + +bool MachineConfig::is_same(MachineConfig *mach) +{ + // no need check NULL pointer as NULL is '\0' + if (strcmp(this->role[0].addr, mach->role[0].addr)) + return false; + if (strcmp(this->role[0].user, mach->role[0].user)) + return false; + if (strcmp(this->role[0].pass, mach->role[0].pass)) + return false; + if (mode != mach->mode) + return false; + if (mode > 0) + { + if (strcmp(this->role[1].addr, mach->role[1].addr)) + return false; + if (strcmp(this->role[1].user, mach->role[1].user)) + return false; + if (strcmp(this->role[1].pass, mach->role[1].pass)) + return false; + } + return true; +} + +bool FieldConfig::is_same(FieldConfig *field) +{ + if (type != field->type && + size != field->size && + flags != field->flags && + strcmp(name, field->name)) + return false; + switch (type) + { + case DField::Signed: + case DField::Unsigned: + case DField::Float: + if (dval.u64 != field->dval.u64) + return false; + break; + case DField::String: + case DField::Binary: + if (dval.bin.len != field->dval.bin.len) + return false; + if (dval.bin.len == 0) + return true; + if (memcmp(dval.bin.ptr, field->dval.bin.ptr, dval.bin.len) != 0) + return false; + break; + default: + return false; + } + return true; +} + +int DbConfig::load_key_hash(DTCConfig *raw) +{ + log_debug("tryto load key-hash plugin"); + + /* init. */ + keyHashConfig.keyHashEnable = 0; + keyHashConfig.keyHashFunction = 0; + keyHashConfig.keyHashLeftBegin = 0; + keyHashConfig.keyHashRightBegin = 0; + + /* not enable key-hash */ + if (raw->get_int_val("DB_DEFINE", "EnableKeyHash", 0) == 0) + { + log_debug("key-hash plugin disable"); + return 0; + } + + /* read KeyHashSo */ + const char *so = raw->get_str_val("DB_DEFINE", "KeyHashSo"); + if (NULL == so || 0 == so[0]) + { + log_notice("not set key-hash plugin name, use default value"); + so = (char *)DEFAULT_KEY_HASH_SO_NAME; + } + + /* read KeyHashFunction */ + const char *var = raw->get_str_val("DB_DEFINE", "KeyHashFunction"); + if (NULL == var || 0 == var[0]) + { + log_error("not set key-hash plugin function name"); + return -1; + } + + char *fun = 0; + int isfunalloc = 0; + const char *iter = strchr(var, '('); + if (NULL == iter) + { + /* + * 按照整个buffer来处理 + */ + keyHashConfig.keyHashLeftBegin = 1; + keyHashConfig.keyHashRightBegin = -1; + fun = (char *)var; + } + else + { + + fun = strndup(var, (size_t)(iter - var)); + isfunalloc = 1; + if (sscanf(iter, "(%d, %d)", + &keyHashConfig.keyHashLeftBegin, + &keyHashConfig.keyHashRightBegin) != 2) + { + free(fun); + log_error("key-hash plugin function format error, %s:%s", so, var); + return -1; + } + + if (keyHashConfig.keyHashLeftBegin == 0) + keyHashConfig.keyHashLeftBegin = 1; + if (keyHashConfig.keyHashRightBegin == 0) + keyHashConfig.keyHashRightBegin = -1; + } + + /* 过滤fun中的空格*/ + fun = skip_blank(fun); + + void *dll = dlopen(so, RTLD_NOW | RTLD_GLOBAL); + if (dll == (void *)NULL) + { + if (isfunalloc) + free(fun); + log_error("dlopen(%s) error: %s", so, dlerror()); + return -1; + } + + /* find key-hash function in key-hash.so */ + keyHashConfig.keyHashFunction = (key_hash_interface)dlsym(dll, fun); + if (keyHashConfig.keyHashFunction == NULL) + { + log_error("key-hash plugin function[%s] not found in [%s]", fun, so); + if (isfunalloc) + free(fun); + return -1; + } + + /* check passed， enable key-hash */ + keyHashConfig.keyHashEnable = 1; + + log_notice("key-hash plugin %s->%s(%d, %d) %s", basename((char *)so), fun, + keyHashConfig.keyHashLeftBegin, + keyHashConfig.keyHashRightBegin, + keyHashConfig.keyHashEnable ? "enable" : "disable"); + + if (isfunalloc) + free(fun); + + return 0; +} + +bool DbConfig::compare_mach(DbConfig *config) +{ + bool found = true; + if (this->machineCnt != config->machineCnt) + return false; + for (int i = 0; found && i < this->machineCnt; ++i) + { + found = false; + for (int j = 0; j < config->machineCnt; ++j) + { + if (this->mach[i].is_same(config->mach + j)) + { + found = true; + break; + } + } + if (!found) + return false; + } + return true; +} + +bool DbConfig::Compare(DbConfig *config, bool compareMach) +{ + if (compareMach && !compare_mach(config)) + return false; + + if (depoly != config->depoly || + keyFieldCnt != config->keyFieldCnt || + idxFieldCnt != config->idxFieldCnt || + fieldCnt != config->fieldCnt || + dbMax != config->dbMax || + dbDiv != config->dbDiv || + dbMod != config->dbMod || + tblMod != config->tblMod || + tblDiv != config->tblDiv) + { + log_error("before origin after param, depoly:[%d,%d],keyFieldCnt:[%d,%d],idxFieldCnt:[%d,%d],fieldCnt:[%d,%d],dbMax:[%d,%d],dbDiv:[%d,%d],dbMod:[%d,%d],tblMod:[%d,%d],tblDiv:[%d,%d]", + depoly, config->depoly, + keyFieldCnt, config->keyFieldCnt, + idxFieldCnt, config->idxFieldCnt, + fieldCnt, config->fieldCnt, + dbMax, config->dbMax, + dbDiv, config->dbDiv, + dbMod, config->dbMod, + tblMod, config->tblMod, + tblDiv, config->tblDiv); + return false; + } + + for (int i = 0; i < fieldCnt; ++i) + { + if (!(field + i)->is_same(config->field + i)) + { + log_error("field%d does not match", i + 1); + return false; + } + } + + return true; +} + +int DbConfig::find_new_mach(DbConfig *config, std::vector &newMach, std::map &machMap) +{ + // be careful, same machine numbers, but change db distribution with new machine cause bug + if (this->machineCnt == config->machineCnt) + return 0; + if (newMach.size() > 0) + newMach.clear(); + if (machMap.size() > 0) + machMap.clear(); + std::map old2New; + for (int i = 0; i < this->machineCnt; ++i) + { + bool found = false; + for (int j = 0; j < config->machineCnt; ++j) + { + if (config->mach[j].is_same(this->mach + i)) + { + if (old2New.find(j) != old2New.end()) + { + log_error("multi new machines to one old machine"); + return 0; + } + found = true; + machMap[i] = j; + old2New[j] = i; + break; + } + } + if (!found) + { + newMach.push_back(i); + log_debug("found new db machine No. %d", i + 1); + } + } + return newMach.size(); +} + +int DbConfig::parse_db_config(DTCConfig *raw) +{ + char buf[1024]; + char sectionStr[256]; + const char *cp; + char *p; + + //DB section + machineCnt = raw->get_int_val("DB_DEFINE", "MachineNum", 1); + if (machineCnt <= 0) + { + log_error("%s", "invalid MachineNum"); + return -1; + } + depoly = raw->get_int_val("DB_DEFINE", "Deploy", 0); + + cp = raw->get_str_val("DB_DEFINE", "DbName"); + if (cp == NULL || cp[0] == '\0') + { + log_error("[DB_DEFINE].DbName not defined"); + return -1; + } + + dbName = STRDUP(cp); + + dstype = raw->get_int_val("DB_DEFINE", "DataSourceType", 0); + + checkTable = raw->get_int_val("DB_DEFINE", "CheckTableConfig", 1); + + // key-hash dll + if (load_key_hash(raw) != 0) + return -1; + + if ((depoly & 1) == 0) + { + if (strchr(dbName, '%') != NULL) + { + log_error("Invalid [DB_DEFINE].DbName, cannot contain symbol '%%'"); + return -1; + } + + dbDiv = 1; + dbMod = 1; + dbFormat = dbName; + } + else + { + cp = raw->get_str_val("DB_DEFINE", "DbNum"); + if (sscanf(cp ?: "", "(%u,%u)", &dbDiv, &dbMod) != 2 || dbDiv == 0 || dbMod == 0) + { + log_error("invalid [DB_DEFINE].DbNum = %s", cp); + return -1; + } + + if (dbMod > 1000000) + { + log_error("invalid [DB_DEFINE].DbMod = %s, mod value too large", cp); + return -1; + } + + p = strchr(dbName, '%'); + if (p) + { + dbFormat = STRDUP(dbName); + *p = '\0'; + } + else + { + dbFormat = (char *)MALLOC(strlen(dbName) + 3); + snprintf(dbFormat, strlen(dbName) + 3, "%s%%d", dbName); + } + } + + // super group + int enableSuperMach = raw->get_int_val("SUPER_MACHINE", "Enable", 0); + if (enableSuperMach) + { + log_error("SUPER_MACHINE don't support anymore, please change to HelperType=DTC"); + return -1; + } + + //Table section + cp = raw->get_str_val("TABLE_DEFINE", "TableName"); + if (cp == NULL || cp[0] == '\0') + { + log_error("[TABLE_DEFINE].TableName not defined"); + return -1; + } + tblName = STRDUP(cp); + + if ((depoly & 2) == 0) + { + if (strchr(tblName, '%') != NULL) + { + log_error("Invalid TableName, cannot contain symbol '%%'"); + return -1; + } + + tblDiv = 1; + tblMod = 1; + tblFormat = tblName; + } + else + { + cp = raw->get_str_val("TABLE_DEFINE", "TableNum"); + if (sscanf(cp ?: "", "(%u,%u)", &tblDiv, &tblMod) != 2 || tblDiv == 0 || tblMod == 0) + { + log_error("invalid [TABLE_DEFINE].TableNum = %s", cp); + return -1; + } + + p = strchr(tblName, '%'); + if (p) + { + tblFormat = STRDUP(tblName); + *p = '\0'; + } + else + { + tblFormat = (char *)MALLOC(strlen(tblName) + 3); + snprintf(tblFormat, strlen(tblName) + 3, "%s%%d", tblName); + } + } + + dbMax = raw->get_int_val("DB_DEFINE", "dbMax", 1); + if (dbMax < 0 || dbMax > 10000) + { + log_error("%s", "invalid [DB_DEFINE].DbMax"); + return -1; + } + if (dbMax < (int)dbMod) + { + log_warning("invalid [TABLE_DEFINE].DbMax too small, increase to %d", dbMod); + dbMax = dbMod; + } + + fieldCnt = raw->get_int_val("TABLE_DEFINE", "FieldCount", 0); + if (fieldCnt <= 0 || fieldCnt > 240) + { + log_error("invalid [TABLE_DEFINE].FieldCount"); + return -1; + } + + keyFieldCnt = raw->get_int_val("TABLE_DEFINE", "KeyFieldCount", 1); + if (keyFieldCnt <= 0 || keyFieldCnt > 32 || keyFieldCnt > fieldCnt) + { + log_error("invalid [TABLE_DEFINE].KeyFieldCount"); + return -1; + } + + idxFieldCnt = raw->get_int_val("TABLE_DEFINE", "IndexFieldCount", 0); + if (keyFieldCnt < 0 || keyFieldCnt + idxFieldCnt > fieldCnt) + { + log_error("invalid [TABLE_DEFINE].IndexFieldCount"); + return -1; + } + + cp = raw->get_str_val("TABLE_DEFINE", "ServerOrderBySQL"); + if (cp && cp[0] != '\0') + { + int n = strlen(cp) - 1; + if (cp[0] != '"' || cp[n] != '"') + { + log_error("dbConfig error, [TABLE_DEFINE].ServerOrderBySQL must quoted"); + return -1; + } + ordSql = (char *)MALLOC(n); + memcpy(ordSql, cp + 1, n - 1); + ordSql[n - 1] = '\0'; + } + ordIns = raw->get_idx_val("TABLE_DEFINE", "ServerOrderInsert", + ((const char *const[]){"last", "first", "purge", NULL}), 0); + if (ordIns < 0) + { + log_error("bad [TABLE_DEFINE].ServerOrderInsert"); + return -1; + } + + //Machine setction + mach = (struct MachineConfig *)calloc(machineCnt, sizeof(struct MachineConfig)); + if (!mach) + { + log_error("malloc failed, %m"); + return -1; + } + for (int i = 0; i < machineCnt; i++) + { + struct MachineConfig *m = &mach[i]; + + snprintf(sectionStr, sizeof(sectionStr), "MACHINE%d", i + 1); + m->helperType = (HELPERTYPE)raw->get_idx_val(sectionStr, "HelperType", + ((const char *const[]){"DUMMY", "DTC", "MYSQL", "TDB", "CUSTOM", NULL}), 2); + + if (m->helperType == DUMMY_HELPER) + { + m->role[0].addr = "DUMMY"; + } + else + { + //master + m->role[0].addr = raw->get_str_val(sectionStr, "DbAddr"); + m->role[0].user = raw->get_str_val(sectionStr, "DbUser"); + m->role[0].pass = raw->get_str_val(sectionStr, "DbPass"); + m->role[0].optfile = raw->get_str_val(sectionStr, "MyCnf"); + + //slave + m->role[1].addr = raw->get_str_val(sectionStr, "DbAddr1"); + m->role[1].user = raw->get_str_val(sectionStr, "DbUser1"); + m->role[1].pass = raw->get_str_val(sectionStr, "DbPass1"); + m->role[1].optfile = raw->get_str_val(sectionStr, "MyCnf"); + + /* master DB settings */ + if (m->role[0].addr == NULL) + { + log_error("missing [%s].DbAddr", sectionStr); + return -1; + } + if (m->role[0].user == NULL) + m->role[0].user = ""; + if (m->role[0].pass == NULL) + m->role[0].pass = ""; + if (m->role[0].optfile == NULL) + m->role[0].optfile = ""; + + m->mode = raw->get_idx_val(sectionStr, "Workload", + ((const char *const[]){"master", "slave", "database", "table", "key", /*"fifo",*/ NULL}), 0); + + if (m->mode < 0) + { + log_error("bad [%s].Workload", sectionStr); + return -1; + } + if (m->mode > 0) + { + if (m->role[1].addr == NULL) + { + log_error("missing [%s].DbAddr1", sectionStr); + return -1; + } + if (m->role[1].user == NULL) + m->role[1].user = m->role[0].user; + if (m->role[1].pass == '\0') + m->role[1].pass = m->role[0].pass; + if (m->role[1].optfile == NULL) + m->role[1].optfile = m->role[0].optfile; + } + } + + cp = raw->get_str_val(sectionStr, "DbIdx"); + if (!cp || !cp[0]) + cp = "[0-0]"; + + m->dbCnt = ParseDbLine(cp, m->dbIdx); + + for (int j = 0; j < m->dbCnt; j++) + { + if (m->dbIdx[j] >= dbMax) + { + log_error("dbConfig error, dbMax=%d, machine[%d].dbIdx=%d", + dbMax, j + 1, m->dbIdx[j]); + return -1; + } + } + + /* Helper number alter */ + m->gprocs[0] = raw->get_int_val(sectionStr, "Procs", 0); + if (m->gprocs[0] < 1) + m->gprocs[0] = 0; + m->gprocs[1] = raw->get_int_val(sectionStr, "WriteProcs", 0); + if (m->gprocs[1] < 1) + m->gprocs[1] = 0; + m->gprocs[2] = raw->get_int_val(sectionStr, "CommitProcs", 0); + if (m->gprocs[2] < 1) + m->gprocs[2] = 0; + + /* Helper Queue Size */ + m->gqueues[0] = raw->get_int_val(sectionStr, "QueueSize", 0); + m->gqueues[1] = raw->get_int_val(sectionStr, "WriteQueueSize", 0); + m->gqueues[2] = raw->get_int_val(sectionStr, "CommitQueueSize", 0); + if (m->gqueues[0] <= 0) + m->gqueues[0] = 1000; + if (m->gqueues[1] <= 0) + m->gqueues[1] = 1000; + if (m->gqueues[2] <= 0) + m->gqueues[2] = 10000; + if (m->gqueues[0] <= 2) + m->gqueues[0] = 2; + if (m->gqueues[1] <= 2) + m->gqueues[1] = 2; + if (m->gqueues[2] <= 1000) + m->gqueues[2] = 1000; + + if (m->dbCnt == 0) // DbIdx is NULL, no helper needed + { + m->gprocs[0] = 0; + m->gprocs[1] = 0; + m->gprocs[2] = 0; + m->mode = 0; + } + + switch (m->mode) + { + case BY_SLAVE: + case BY_DB: + case BY_TABLE: + case BY_KEY: + m->gprocs[3] = m->gprocs[0]; + m->gqueues[3] = m->gqueues[0]; + if (slaveGuard == 0) + { + slaveGuard = raw->get_int_val("DB_DEFINE", "SlaveGuardTime", 15); + if (slaveGuard < 5) + slaveGuard = 5; + } + break; + default: + m->gprocs[3] = 0; + m->gqueues[3] = 0; + } + + m->procs = m->gprocs[0] + m->gprocs[1] + m->gprocs[2]; + procs += m->procs; + } + + //Field section + field = (struct FieldConfig *)calloc(fieldCnt, sizeof(struct FieldConfig)); + if (!field) + { + log_error("malloc failed, %m"); + return -1; + } + autoinc = -1; + lastmod = -1; + lastcmod = -1; + lastacc = -1; + compressflag = -1; + expireTime = -1; + for (int i = 0; i < fieldCnt; i++) + { + struct FieldConfig *f = &field[i]; + + snprintf(sectionStr, sizeof(sectionStr), "FIELD%d", i + 1); + f->name = raw->get_str_val(sectionStr, "FieldName"); + if (f->name == NULL) + { + log_error("field name missing for %s", sectionStr); + return -1; + } + f->type = raw->get_int_val(sectionStr, "FieldType", -1); + if (f->type < 0) + { + f->type = raw->get_idx_val(sectionStr, "FieldType", + ((const char *const[]){ + "int", + "signed", + "unsigned", + "float", + "string", + "binary", + NULL}), + -1); + if (f->type == 0) + f->type = 1; + } + if (f->type <= 0 || f->type > 5) + { + log_error("Invalid value [%s].FieldType", sectionStr); + return -1; + } + f->size = raw->get_int_val(sectionStr, "FieldSize", -1); + if (f->size == -1) + { + log_error("field size missing for %s", sectionStr); + return -1; + } + + if (i >= keyFieldCnt && i < keyFieldCnt + idxFieldCnt) + { // index field + if (field[i].size > 255) + { + log_error("index field[%s] size must less than 256", f->name); + return -1; + } + const char *idx_order = raw->get_str_val(sectionStr, "Order"); + if (idx_order && strcasecmp(idx_order, "DESC") == 0) + f->flags |= DB_FIELD_FLAGS_DESC_ORDER; + else + log_debug("index field[%s] order: ASC", f->name); + } + if (field[i].size >= (64 << 20)) + { + log_error("field[%s] size must less than 64M", f->name); + return -1; + } + + if (i >= keyFieldCnt && raw->get_int_val(sectionStr, "ReadOnly", 0) > 0) + f->flags |= DB_FIELD_FLAGS_READONLY; + if (raw->get_int_val(sectionStr, "UniqField", 0) > 0) + f->flags |= DB_FIELD_FLAGS_UNIQ; + if (raw->get_int_val(sectionStr, "Volatile", 0) > 0) + { + if (i < keyFieldCnt) + { + log_error("field%d: key field can't be volatile", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: uniq field can't be volatile", i + 1); + return -1; + } + f->flags |= DB_FIELD_FLAGS_VOLATILE; + } + if (raw->get_int_val(sectionStr, "Discard", 0) > 0) + { + if (i < keyFieldCnt) + { + log_error("field%d: key field can't be discard", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: uniq field can't be discard", i + 1); + return -1; + } + f->flags |= DB_FIELD_FLAGS_DISCARD | DB_FIELD_FLAGS_VOLATILE; + } + + if (!strcmp(f->name, DTC_EXPIRE_TIME_FIELD)) + { + if (f->type != DField::Unsigned && f->type != DField::Signed) + { + log_error("field%d: expire time field byte must be unsigned", i + 1); + return -1; + } + if (expireTime >= 0) + { + log_error("field%d already defined as expire time", expireTime + 1); + return -1; + } + expireTime = i; + } + + /* ATTN: must be last one */ + cp = raw->get_str_val(sectionStr, "DefaultValue"); + if (cp && !strcmp(cp, "auto_increment")) + { + if (f->type != DField::Unsigned && f->type != DField::Signed) + { + log_error("field%d: auto_increment field byte must be unsigned", i + 1); + return -1; + } + if (autoinc >= 0) + { + log_error("field%d already defined as auto_increment", autoinc + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_DISCARD)) + { + log_error("field%d: auto_increment can't be Discard", autoinc + 1); + return -1; + } + autoinc = i; + f->flags |= DB_FIELD_FLAGS_READONLY; + } + else if (cp && !strcmp(cp, "lastmod")) + { + if (i < keyFieldCnt) + { + log_error("key field%d can't be lastmod", i + 1); + return -1; + } + if ((f->type != DField::Unsigned && f->type != DField::Signed) || f->size < 4) + { + log_error("field%d: lastmod field byte must be unsigned & size>=4", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: lastmod field byte can't be UniqField", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_DISCARD)) + { + log_error("field%d: lastmod can't be Discard", autoinc + 1); + return -1; + } + if (lastmod >= 0) + { + log_error("field%d already defined as lastmod", lastmod + 1); + return -1; + } + lastmod = i; + } + else if (cp && !strcmp(cp, "lastcmod")) + { + if (i < keyFieldCnt) + { + log_error("key field%d can't be lastcmod", i + 1); + return -1; + } + if ((f->type != DField::Unsigned && f->type != DField::Signed) || f->size < 4) + { + log_error("field%d: lastcmod field byte must be unsigned & size>=4", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: lastcmod field byte can't be UniqField", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_DISCARD)) + { + log_error("field%d: lastcmod can't be Discard", autoinc + 1); + return -1; + } + if (lastcmod >= 0) + { + log_error("field%d already defined as lastcmod", lastcmod + 1); + return -1; + } + lastcmod = i; + } + else if (cp && !strcmp(cp, "lastacc")) + { + if (i < keyFieldCnt) + { + log_error("key field%d can't be lastacc", i + 1); + return -1; + } + if ((f->type != DField::Unsigned && f->type != DField::Signed) || f->size != 4) + { + log_error("field%d: lastacc field byte must be unsigned & size==4", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: lastacc field byte can't be UniqField", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_DISCARD)) + { + log_error("field%d: lastacc can't be Discard", autoinc + 1); + return -1; + } + if (lastacc >= 0) + { + log_error("field%d already defined as lastacc", lastacc + 1); + return -1; + } + lastacc = i; + } + else if (cp && !strcmp(cp, "compressflag")) + { + if (i < keyFieldCnt) + { + log_error("key field%d can't be compressflag", i + 1); + return -1; + } + if ((f->type != DField::Unsigned && f->type != DField::Signed) || f->size != 8) + { + log_error("field%d: compressflag field byte must be unsigned & size==8", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("field%d: compressflag field byte can't be UniqField", i + 1); + return -1; + } + if ((f->flags & DB_FIELD_FLAGS_DISCARD)) + { + log_error("field%d: compressflag can't be Discard", i + 1); + return -1; + } + if (compressflag >= 0) + { + log_error("field%d already defined as compressflag", i + 1); + return -1; + } + compressflag = i; + } + else + { + if (i == 0 && cp) + { + log_error("specify DefaultValue for Field1 is invalid"); + return -1; + } + switch (f->type) + { + case DField::Unsigned: + f->dval.Set(!cp || !cp[0] ? 0 : (uint64_t)strtoull(cp, NULL, 0)); + break; + case DField::Signed: + f->dval.Set(!cp || !cp[0] ? 0 : (int64_t)strtoll(cp, NULL, 0)); + break; + case DField::Float: + f->dval.Set(!cp || !cp[0] ? 0.0 : strtod(cp, NULL)); + break; + case DField::String: + case DField::Binary: + int len; + if (!cp || !cp[0]) + { + f->dval.Set(NULL, 0); + break; + } + else if (cp[0] == '"') + { + /* Decode quoted string */ + cp++; + len = strlen(cp); + if (cp[len - 1] != '"') + { + log_error("field%d: unmatched quoted default value", i + 1); + return -1; + } + len--; + } + else if (cp[0] == '0' && (cp[1] | 0x20) == 'x') + { + /* Decode hex value */ + int j = 2; + len = 0; + while (cp[j]) + { + char v; + if (cp[j] == '-') + continue; + if (cp[j] >= '0' && cp[j] <= '9') + v = cp[j] - '0'; + else if (cp[j] >= 'a' && cp[j] <= 'f') + v = cp[j] - 'a' + 10; + else if (cp[j] >= 'A' && cp[j] <= 'F') + v = cp[j] - 'A' + 10; + else + { + log_error("field%d: invalid hex default value", i + 1); + return -1; + } + j++; + if (cp[j] >= '0' && cp[j] <= '9') + v = (v << 4) + cp[j] - '0'; + else if (cp[j] >= 'a' && cp[j] <= 'f') + v = (v << 4) + cp[j] - 'a' + 10; + else if (cp[j] >= 'A' && cp[j] <= 'F') + v = (v << 4) + cp[j] - 'A' + 10; + else + { + log_error("field%d: invalid hex default value", i + 1); + return -1; + } + j++; + buf[len++] = v; + } + } + else + { + log_error("field%d: string default value must quoted or hex value", i + 1); + return -1; + } + + if (len > f->size) + { + log_error("field%d: default value size %d truncated to %d", + i + 1, len, f->size); + return -1; + } + if (len == 0) + f->dval.Set(NULL, 0); + else + { + char *p = (char *)MALLOC(len + 1); + memcpy(p, cp, len); + p[len] = '\0'; + f->dval.Set(p, len); + } + break; + } + } + } + + if (field[0].type == DField::Float) + { + log_error("%s", "FloatPoint key not supported"); + return -1; + } + + if (((field[0].type == DField::String || field[0].type == DField::Binary) && keyHashConfig.keyHashEnable == 0) || + autoinc == 0) + { + if (machineCnt != 1) + { + log_error("%s", "String/Binary/AutoInc key require MachineNum==1"); + return -1; + } + if (dbMax != 1) + { + log_error("%s", "String/Binary/AutoInc key require dbMax==1"); + return -1; + } + if (depoly != 0) + { + log_error("%s", "String/Binary/AutoInc key require Depoly==0"); + return -1; + } + } + + // expire time only support uniq key + if (expireTime != -1 && !(field[0].flags & DB_FIELD_FLAGS_UNIQ)) + { + log_error("%s", "expire time only support uniq key"); + return -1; + } + + if (keyFieldCnt > 1) + { + for (int j = 0; j < keyFieldCnt; j++) + { + struct FieldConfig *f1 = &field[j]; + if (f1->type != DField::Signed && f1->type != DField::Unsigned) + { + log_error("%s", "Only Multi-Integer-Key supported"); + return -1; + } + } + } + + return 0; +} + +struct DbConfig *DbConfig::load_buffered(const char *buf) +{ + DTCConfig *raw = new DTCConfig(); + if (raw->parse_buffered_config(buf) < 0) + { + delete raw; + return NULL; + } + + DbConfig *dbConfig = (struct DbConfig *)calloc(1, sizeof(struct DbConfig)); + dbConfig->cfgObj = raw; + + if (dbConfig->parse_db_config(raw) == -1) + { + dbConfig->Destroy(); + dbConfig = NULL; + } + + return dbConfig; +} + +struct DbConfig *DbConfig::Load(const char *file) +{ + DTCConfig *raw = new DTCConfig(); + if (raw->parse_config(file) < 0) + { + delete raw; + return NULL; + } + + DbConfig *dbConfig = (struct DbConfig *)calloc(1, sizeof(struct DbConfig)); + dbConfig->cfgObj = raw; + + if (dbConfig->parse_db_config(raw) == -1) + { + dbConfig->Destroy(); + dbConfig = NULL; + } + + return dbConfig; +} + +struct DbConfig *DbConfig::Load(DTCConfig *raw) +{ + DbConfig *dbConfig = (struct DbConfig *)calloc(1, sizeof(struct DbConfig)); + dbConfig->cfgObj = raw; + + if (dbConfig->parse_db_config(raw) == -1) + { + dbConfig->Destroy(); + dbConfig = NULL; + } + + return dbConfig; +} + +bool DbConfig::build_path(char *path, int n, int pid, int group, int role, int type) +{ + if (type == DUMMY_HELPER || type == DTC_HELPER) + return false; + memset(path, 0, n); + snprintf(path, n - 1, HELPERPATHFORMAT, pid, group, MACHINEROLESTRING[role]); + return true; +} + +void DbConfig::set_helper_path(int serverId) +{ + //multi DTC datasource + for (int i = 0; i < machineCnt; i++) + { + MachineConfig *m = &mach[i]; + for (int j = 0; j < ROLES_PER_MACHINE; j++) + { + //check dtc datasource + switch (m->helperType) + { + case DUMMY_HELPER: + break; + case DTC_HELPER: + snprintf(m->role[j].path, sizeof(m->role[j].path) - 1, "%s", m->role[j].addr); + log_info("build helper path, serverId:%d, addr:%s, path:%s, dstype:%d", serverId, m->role[j].addr, m->role[j].path, dstype); + break; + default: + snprintf(m->role[j].path, sizeof(m->role[j].path) - 1, HELPERPATHFORMAT, serverId, i, MACHINEROLESTRING[j]); + log_info("build helper path, serverId:%d, addr:%s, path:%s, dstype:%d", serverId, m->role[j].addr, m->role[j].path, dstype); + } + + // rocksdb only start one process + if (dstype == 2) + break; + } + } +} + +void DbConfig::Destroy(void) +{ + if (this == NULL) + return; + + // machine hasn't dynamic objects + FREE_IF(mach); + + if (field) + { + for (int i = 0; i < fieldCnt; i++) + { + switch (field[i].type) + { + case DField::String: + case DField::Binary: + if (field[i].dval.str.ptr) + FREE(field[i].dval.str.ptr); + } + } + FREE(field); + } + + if (dbFormat != dbName) + FREE_IF(dbFormat); + FREE_IF(dbName); + if (tblFormat != tblName) + FREE(tblFormat); + FREE_IF(tblName); + FREE_IF(ordSql); + DELETE(cfgObj); + FREE((void *)this); +} + +void dump_db_config(const struct DbConfig *cf) +{ + int i, j; + + printf("DbName: %s\n", cf->dbName); + printf("DbNum: (%d,%d)\n", cf->dbDiv, cf->dbMod); + printf("MachineNum: %d\n", cf->machineCnt); + for (i = 0; i < cf->machineCnt; i++) + { + struct MachineConfig *mach = &cf->mach[i]; + printf("\n"); + printf("Machine[%d].addr: %s\n", i, mach->role[0].addr); + printf("Machine[%d].user: %s\n", i, mach->role[0].user); + printf("Machine[%d].pass: %s\n", i, mach->role[0].pass); + printf("Machine[%d].procs: %d\n", i, mach->procs); + printf("Machine[%d].dbCnt: %d\n", i, mach->dbCnt); + printf("Machine[%d].dbIdx: ", i); + for (j = 0; j < mach->dbCnt; j++) + printf("%d ", mach->dbIdx[j]); + printf("\n"); + } + printf("\nTableName: %s\n", cf->tblName); + printf("TableNum: (%d,%d)\n", cf->tblDiv, cf->tblMod); + printf("FieldCount: %d key %d\n", cf->fieldCnt, cf->keyFieldCnt); + for (i = 0; i < cf->fieldCnt; i++) + printf("Field[%d].name: %s, size: %d, type: %d\n", i, + cf->field[i].name, cf->field[i].size, cf->field[i].type); +} diff --git a/src/search_local/index_storage/common/dbconfig.h b/src/search_local/index_storage/common/dbconfig.h new file mode 100644 index 0000000..5b0315a --- /dev/null +++ b/src/search_local/index_storage/common/dbconfig.h @@ -0,0 +1,164 @@ +/* + * ===================================================================================== + * + * Filename: dbconfig.h + * + * Description: database config operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _H_DTC_DB_CONFIG_H_ +#define _H_DTC_DB_CONFIG_H_ + +#include "table_def.h" +#include "config.h" +#include +#include +#include + +#define MAXDB_ON_MACHINE 1000 +#define GROUPS_PER_MACHINE 4 +#define GROUPS_PER_ROLE 3 +#define ROLES_PER_MACHINE 2 +#define MACHINEROLESTRING "ms?????" + +#define DB_FIELD_FLAGS_READONLY 1 +#define DB_FIELD_FLAGS_UNIQ 2 +#define DB_FIELD_FLAGS_DESC_ORDER 4 +#define DB_FIELD_FLAGS_VOLATILE 8 +#define DB_FIELD_FLAGS_DISCARD 0x10 + +/* 默认key-hash so文件名及路径 */ +#define DEFAULT_KEY_HASH_SO_NAME "../lib/key-hash.so" +/* key-hash接口函数 */ +typedef uint64_t (*key_hash_interface)(const char *key, int len, int left, int right); + +enum +{ + INSERT_ORDER_LAST = 0, + INSERT_ORDER_FIRST = 1, + INSERT_ORDER_PURGE = 2 +}; + +enum +{ + BY_MASTER, + BY_SLAVE, + BY_DB, + BY_TABLE, + BY_KEY, + BY_FIFO +}; + +typedef enum +{ + DUMMY_HELPER = 0, + DTC_HELPER, + MYSQL_HELPER, + TDB_HELPER, + CUSTOM_HELPER, +} HELPERTYPE; + +struct MachineConfig +{ + struct + { + const char *addr; + const char *user; + const char *pass; + const char *optfile; + //DataMerge Addr + const char *dm; + char path[32]; + } role[GROUPS_PER_MACHINE]; // GROUPS_PER_ROLE should be ROLES_PER_MACHINE + + HELPERTYPE helperType; + int mode; + uint16_t dbCnt; + uint16_t procs; + uint16_t dbIdx[MAXDB_ON_MACHINE]; + uint16_t gprocs[GROUPS_PER_MACHINE]; + uint32_t gqueues[GROUPS_PER_MACHINE]; + bool is_same(MachineConfig *mach); +}; + +struct FieldConfig +{ + const char *name; + char type; + int size; + DTCValue dval; + int flags; + bool is_same(FieldConfig *field); +}; + +struct KeyHash +{ + int keyHashEnable; + int keyHashLeftBegin; /* buff 的左起始位置 */ + int keyHashRightBegin; /* buff 的右起始位置 */ + key_hash_interface keyHashFunction; +}; + +struct DbConfig +{ + DTCConfig *cfgObj; + char *dbName; + char *dbFormat; + char *tblName; + char *tblFormat; + + int dstype; /* data-source type: default is mysql 0: mysql 1: gaussdb 2: rocksdb */ + int checkTable; + unsigned int dbDiv; + unsigned int dbMod; + unsigned int tblMod; + unsigned int tblDiv; + int fieldCnt; + int keyFieldCnt; + int idxFieldCnt; + int machineCnt; + int procs; //all machine procs total + int dbMax; //max db index + char depoly; //0:none 1:multiple db 2:multiple table 3:both + + struct KeyHash keyHashConfig; + + int slaveGuard; + int autoinc; + int lastacc; + int lastmod; + int lastcmod; + int expireTime; + int compressflag; + int ordIns; + char *ordSql; + struct FieldConfig *field; + struct MachineConfig *mach; + + static struct DbConfig *load_buffered(const char *buf); + static struct DbConfig *Load(const char *file); + static struct DbConfig *Load(DTCConfig *); + static bool build_path(char *path, int n, int pid, int group, int role, int type); + void Destroy(void); + void dump_db_config(FILE *); + void set_helper_path(int); + DTCTableDefinition *build_table_definition(void); + int load_key_hash(DTCConfig *); + bool compare_mach(DbConfig *config); + bool Compare(DbConfig *config, bool compareMach = false); + int find_new_mach(DbConfig *config, std::vector &newMach, std::map &machMap); + +private: + int parse_db_config(DTCConfig *); +}; + +#endif diff --git a/src/search_local/index_storage/common/dbconfig_tdef.cc b/src/search_local/index_storage/common/dbconfig_tdef.cc new file mode 100644 index 0000000..7b23034 --- /dev/null +++ b/src/search_local/index_storage/common/dbconfig_tdef.cc @@ -0,0 +1,76 @@ +/* + * ===================================================================================== + * + * Filename: dbconfig_tdef.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "log.h" +#include "dbconfig.h" +#include "table_def.h" + +DTCTableDefinition *DbConfig::build_table_definition(void) +{ + DTCTableDefinition *tdef = new DTCTableDefinition(fieldCnt); + tdef->set_table_name(tblName); + for (int i = 0; i < fieldCnt; i++) + { + if (tdef->add_field(i, field[i].name, field[i].type, field[i].size) != 0) + { + log_error("add_field failed, name=%s, size=%d, type=%d", + field[i].name, field[i].size, field[i].type); + delete tdef; + return NULL; + } + tdef->set_default_value(i, &field[i].dval); + if ((field[i].flags & DB_FIELD_FLAGS_READONLY)) + tdef->mark_as_read_only(i); + if ((field[i].flags & DB_FIELD_FLAGS_VOLATILE)) + tdef->mark_as_volatile(i); + if ((field[i].flags & DB_FIELD_FLAGS_DISCARD)) + tdef->mark_as_discard(i); + if ((field[i].flags & DB_FIELD_FLAGS_UNIQ)) + tdef->mark_uniq_field(i); + if ((field[i].flags & DB_FIELD_FLAGS_DESC_ORDER)) + { + tdef->mark_order_desc(i); + if (tdef->is_desc_order(i)) + log_debug("success set field[%d] desc order", i); + else + log_debug("set field[%d] desc roder fail", i); + } + } + if (autoinc >= 0) + tdef->set_auto_increment(autoinc); + if (lastacc >= 0) + tdef->set_lastacc(lastacc); + if (lastmod >= 0) + tdef->set_lastmod(lastmod); + if (lastcmod >= 0) + tdef->set_lastcmod(lastcmod); + if (compressflag >= 0) + tdef->set_compress_flag(compressflag); + if (expireTime >= 0) + tdef->set_expire_time(expireTime); + + if (tdef->set_key_fields(keyFieldCnt) < 0) + { + log_error("Table key size %d too large, must <= 255", + tdef->key_format() > 0 ? tdef->key_format() : -tdef->key_format()); + delete tdef; + return NULL; + } + tdef->set_index_fields(idxFieldCnt); + tdef->build_info_cache(); + return tdef; +} diff --git a/src/search_local/index_storage/common/decode.cc b/src/search_local/index_storage/common/decode.cc new file mode 100644 index 0000000..54f233c --- /dev/null +++ b/src/search_local/index_storage/common/decode.cc @@ -0,0 +1,330 @@ +/* + * ===================================================================================== + * + * Filename: decode.cc + * + * Description: decode/encode data. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "decode.h" +#include "buffer_error.h" +#include "log.h" +#include "table_def.h" +#define CONVERT_NULL_TO_EMPTY_STRING 1 + +// BYTE0 BYTE1 BYTE2 BYTE3 BYTE4 +// 0-1110111 <224 +// 11110000-1100 8 <3328 +// 11111101 8 8 <64K +// 11111110 8 8 8 <16M +// 11111111 8 8 8 8 <4G +int decode_length(DTCBinary &bin, uint32_t &len) +{ + if (!bin) + return -EC_BAD_SECTION_LENGTH; + len = *bin++; + if (len < 240) + { + } + else if (len <= 252) + { + if (!bin) + return -EC_BAD_SECTION_LENGTH; + len = ((len & 0xF) << 8) + *bin++; + } + else if (len == 253) + { + if (bin < 2) + return -EC_BAD_SECTION_LENGTH; + len = (bin[0] << 8) + bin[1]; + bin += 2; + } + else if (len == 254) + { + if (bin < 3) + return -EC_BAD_SECTION_LENGTH; + len = (bin[0] << 16) + (bin[1] << 8) + bin[2]; + bin += 3; + } + else + { + if (bin < 4) + return -EC_BAD_SECTION_LENGTH; + len = (bin[0] << 24) + (bin[1] << 16) + (bin[2] << 8) + bin[3]; + bin += 4; + if (len > MAXPACKETSIZE) + return -EC_BAD_VALUE_LENGTH; + } + return 0; +} + +char *encode_length(char *p, uint32_t len) +{ + if (len < 240) + { + p[0] = len; + return p + 1; + } + else if (len < (13 << 8)) + { + p[0] = 0xF0 + (len >> 8); + p[1] = len & 0xFF; + return p + 2; + } + else if (len < (1 << 16)) + { + p[0] = 253; + p[1] = len >> 8; + p[2] = len & 0xFF; + return p + 3; + } + else if (len < (1 << 24)) + { + p[0] = 254; + p[1] = len >> 16; + p[2] = len >> 8; + p[3] = len & 0xFF; + return p + 4; + } + else + { + p[0] = 255; + p[1] = len >> 24; + p[2] = len >> 16; + p[3] = len >> 8; + p[4] = len & 0xFF; + return p + 5; + } + return 0; +} + +int encoded_bytes_length(uint32_t n) +{ + if (n < 240) + return 1; + if (n < (13 << 8)) + return 2; + if (n < (1 << 16)) + return 3; + if (n < (1 << 24)) + return 4; + return 5; +} + +/* + * decode value by type + * datavalue format: + * + */ +int decode_data_value(DTCBinary &bin, DTCValue &val, int type) +{ + uint8_t *p; + uint32_t len; + int rv; + + rv = decode_length(bin, len); + if (rv) + { + return rv; + } + + if (bin.len < (int64_t)len) + return -EC_BAD_SECTION_LENGTH; + + switch (type) + { + case DField::None: + break; + + case DField::Signed: + case DField::Unsigned: + /* integer always encoded as signed value */ + if (len == 0 || len > 8) + return -EC_BAD_VALUE_LENGTH; + p = (uint8_t *)bin.ptr + 1; + int64_t s64; + s64 = *(int8_t *)bin.ptr; + switch (len) + { + case 8: + s64 = (s64 << 8) | *p++; + case 7: + s64 = (s64 << 8) | *p++; + case 6: + s64 = (s64 << 8) | *p++; + case 5: + s64 = (s64 << 8) | *p++; + case 4: + s64 = (s64 << 8) | *p++; + case 3: + s64 = (s64 << 8) | *p++; + case 2: + s64 = (s64 << 8) | *p++; + } + val.Set(s64); + break; + + case DField::Float: + /* float value encoded as %A string */ + if (len < 3) + return -EC_BAD_VALUE_LENGTH; + if (bin[len - 1] != '\0') + return -EC_BAD_FLOAT_VALUE; + if (!strcmp(bin.ptr, "NAN")) + val.flt = NAN; + else if (!strcmp(bin.ptr, "INF")) + val.flt = INFINITY; + else if (!strcmp(bin.ptr, "-INF")) + val.flt = -INFINITY; + else + { + long double ldf; + if (sscanf(bin.ptr, __FLTFMT__, &ldf) != 1) + return -EC_BAD_FLOAT_VALUE; + val.flt = ldf; + } + break; + + case DField::String: + /* NULL encoded as zero length, others padded '\0' */ + if (len == 0) + { +#if CONVERT_NULL_TO_EMPTY_STRING + val.Set(bin.ptr - 1, 0); +#else + val.Set(NULL, 0); +#endif + } + else + { + if (bin[len - 1] != '\0') + return -EC_BAD_STRING_VALUE; + val.Set(bin.ptr, len - 1); + } + break; + + case DField::Binary: + /* NULL encoded as zero length, others padded '\0' */ + if (len == 0) + { +#if CONVERT_NULL_TO_EMPTY_STRING + val.Set(bin.ptr - 1, 0); +#else + val.Set(NULL, 0); +#endif + } + else + { + if (bin[len - 1] != '\0') + return -EC_BAD_STRING_VALUE; + val.Set(bin.ptr, len - 1); + } + break; + } + bin += len; + return 0; +} + +/* + * two form of field Id + * ID>0, by ID + * 0 byname, LEN is NAME length, no '\0' + */ +int decode_field_id(DTCBinary &bin, uint8_t &id, const DTCTableDefinition *tdef, int &needDefinition) +{ + if (!bin) + return -EC_BAD_SECTION_LENGTH; + uint8_t n = *bin++; + if (n) + { + id = n; + } + else + { + if (!bin) + return -EC_BAD_SECTION_LENGTH; + n = *bin++; + if (n == 0) + { + id = n; + } + else + { + if (bin < n) + return -EC_BAD_SECTION_LENGTH; + int fid; + // if(n <= 0 || (fid = tdef->field_id(bin.ptr, n)) <= 0){ + if (n <= 0 || (fid = tdef->field_id(bin.ptr, n)) < 0) + { // allow select key-field + log_debug("bad field name: %s", bin.ptr); + return -EC_BAD_FIELD_NAME; + } + id = fid; + bin += n; + needDefinition = 1; + } + } + return 0; +} + +/* + * Tag format: + * + * ID: 1 bytes + * LEN: Length encoding + * VALUE: DataValue encoding, predefined type + * Simpel Section format: + * ... + */ +int decode_simple_section(DTCBinary &bin, SimpleSection &ss, uint8_t kt) +{ + uint8_t mask[32]; + FIELD_ZERO(mask); + while (!!bin) + { + int id = *bin++; + + if (FIELD_ISSET(id, mask)) + return -EC_DUPLICATE_TAG; + int type = id == 0 ? kt : ss.tag_type(id); + /* avoid one more copy of tag DTCValue */ + /* int(len, value): buf -> local; buf -> local -> tag */ + /* str(len, value): buf -> local -> tag; buf -> tag */ + int rv = decode_data_value(bin, *ss.get_this_tag(id), type); + if (rv) + { + log_debug("decode tag[id:%d] error: %d", id, rv); + return rv; + } + if (type != DField::None) + { + //ss.set_tag(id, val); + /* no need of check if none type section and check if duplicate tag */ + ss.SetTagMask(id); + FIELD_SET(id, mask); + } + } + return 0; +} + +int decode_simple_section(char *p, int l, SimpleSection &ss, uint8_t kt) +{ + DTCBinary bin = {l, p}; + return decode_simple_section(bin, ss, kt); +} diff --git a/src/search_local/index_storage/common/decode.h b/src/search_local/index_storage/common/decode.h new file mode 100644 index 0000000..461a03a --- /dev/null +++ b/src/search_local/index_storage/common/decode.h @@ -0,0 +1,67 @@ +/* + * ===================================================================================== + * + * Filename: decode.h + * + * Description: decode/encode data. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_DECODE_H__ +#define __CH_DECODE_H__ + +/* + * Base Decode/Encode routines: + * p = Encode...(p, ...); // encode object and advance pointer + * EncodedBytes...(...); // calculate encoded object size + * Decode...(...); // Decode objects + */ +#include "value.h" +#include "table_def.h" +#include "section.h" +#include "field.h" +#include "field_api.h" + +#define __FLTFMT__ "%LA" +static inline char *encode_data_type(char *p, uint8_t type) +{ + *p++ = type == DField::Unsigned ? DField::Signed : type; + return p; +} + +extern int decode_length(DTCBinary &bin, uint32_t &len); +extern char *encode_length(char *p, uint32_t len); +extern int encoded_bytes_length(uint32_t n); + +extern int decode_data_value(DTCBinary &bin, DTCValue &val, int type); +extern char *encode_data_value(char *p, const DTCValue *v, int type); +extern int encoded_bytes_data_value(const DTCValue *v, int type); + +extern int decode_field_id(DTCBinary &, uint8_t &id, const DTCTableDefinition *tdef, int &needDefinition); +extern int decode_simple_section(DTCBinary &, SimpleSection &, uint8_t); +extern int decode_simple_section(char *, int, SimpleSection &, uint8_t); + +extern int encoded_bytes_simple_section(const SimpleSection &, uint8_t); +extern char *encode_simple_section(char *p, const SimpleSection &, uint8_t); +extern int encoded_bytes_field_set(const DTCFieldSet &); +extern char *encode_field_set(char *p, const DTCFieldSet &); +extern int encoded_bytes_field_value(const DTCFieldValue &); +extern char *encode_field_value(char *, const DTCFieldValue &); + +extern int encoded_bytes_multi_key(const DTCValue *v, const DTCTableDefinition *tdef); +extern char *encode_multi_key(char *, const DTCValue *v, const DTCTableDefinition *tdef); +class FieldSetByName; +extern int encoded_bytes_field_set(const FieldSetByName &); +extern char *encode_field_set(char *p, const FieldSetByName &); +extern int encoded_bytes_field_value(const FieldValueByName &); +extern char *encode_field_value(char *, const FieldValueByName &); + +#endif diff --git a/src/search_local/index_storage/common/decoder_base.cc b/src/search_local/index_storage/common/decoder_base.cc new file mode 100644 index 0000000..2eb07fe --- /dev/null +++ b/src/search_local/index_storage/common/decoder_base.cc @@ -0,0 +1,30 @@ +/* + * ===================================================================================== + * + * Filename: decoder_base.cc + * + * Description: decode unit. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "decoder_base.h" +#include "poll_thread.h" + +DecoderUnit::DecoderUnit(PollThread *owner, int idletimeout) +{ + this->owner = owner; + this->idleTime = idletimeout; + this->idleList = owner->get_timer_list(idletimeout); +} + +DecoderUnit::~DecoderUnit() +{ +} diff --git a/src/search_local/index_storage/common/decoder_base.h b/src/search_local/index_storage/common/decoder_base.h new file mode 100644 index 0000000..7716bf8 --- /dev/null +++ b/src/search_local/index_storage/common/decoder_base.h @@ -0,0 +1,43 @@ +/* + * ===================================================================================== + * + * Filename: decoder_base.h + * + * Description: decode unit. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_DECODER_UNIT_H__ +#define __H_DTC_DECODER_UNIT_H__ + +class PollThread; +class TimerList; + +class DecoderUnit +{ +public: + DecoderUnit(PollThread *, int idletimeout); + virtual ~DecoderUnit(); + + TimerList *idle_list(void) const { return idleList; } + int idle_time(void) const { return idleTime; } + PollThread *owner_thread(void) const { return owner; } + + virtual int process_stream(int fd, int req, void *, int) = 0; + virtual int process_dgram(int fd) = 0; + +protected: + PollThread *owner; + int idleTime; + TimerList *idleList; +}; + +#endif diff --git a/src/search_local/index_storage/common/dtc_global.cc b/src/search_local/index_storage/common/dtc_global.cc new file mode 100644 index 0000000..2db2721 --- /dev/null +++ b/src/search_local/index_storage/common/dtc_global.cc @@ -0,0 +1,22 @@ +/* + * ===================================================================================== + * + * Filename: dtc_global.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "dtc_global.h" + +int DTCGlobal::_pre_alloc_NG_num = 1024; +int DTCGlobal::_min_chunk_size = 0; +int DTCGlobal::_pre_purge_nodes = 0; diff --git a/src/search_local/index_storage/common/dtc_global.h b/src/search_local/index_storage/common/dtc_global.h new file mode 100644 index 0000000..6a0d34d --- /dev/null +++ b/src/search_local/index_storage/common/dtc_global.h @@ -0,0 +1,36 @@ +/* + * ===================================================================================== + * + * Filename: dtc_global.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _DTC_GLOBAL_H_ +#define _DTC_GLOBAL_H_ +#include "non_copyable.h" + +#define TABLE_CONF_NAME "../conf/table.conf" +#define CACHE_CONF_NAME "../conf/cache.conf" +#define ALARM_CONF_FILE "../conf/dtcalarm.conf" +class DTCGlobal : private noncopyable +{ +public: + DTCGlobal(void); + ~DTCGlobal(void); + +public: + static int _pre_alloc_NG_num; + static int _min_chunk_size; + static int _pre_purge_nodes; +}; +#endif diff --git a/src/search_local/index_storage/common/dtcutils.h b/src/search_local/index_storage/common/dtcutils.h new file mode 100644 index 0000000..30b5987 --- /dev/null +++ b/src/search_local/index_storage/common/dtcutils.h @@ -0,0 +1,113 @@ +/* + * ===================================================================================== + * + * Filename: dtcutils.h + * + * Description: general function. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_UTILS__ +#define __DTC_UTILS__ +#include +#include + +#include +#include +#include +#include +#include + +/*此文件放置dtc的工具函数*/ +namespace dtc +{ + namespace utils + { + + /************************************************* + 获取本机的ip tomchen + **************************************************/ + inline std::string get_local_ip() + { + int iClientSockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (iClientSockfd < 0) + return "0.0.0.0"; + + struct sockaddr_in stINETAddr; + stINETAddr.sin_addr.s_addr = inet_addr("192.168.0.1"); + stINETAddr.sin_family = AF_INET; + stINETAddr.sin_port = htons(8888); + + int iCurrentFlag = fcntl(iClientSockfd, F_GETFL, 0); + fcntl(iClientSockfd, F_SETFL, iCurrentFlag | FNDELAY); + + if (connect(iClientSockfd, (struct sockaddr *)&stINETAddr, sizeof(sockaddr)) != 0) + { + close(iClientSockfd); + return "0.0.0.0"; + } + + struct sockaddr_in stINETAddrLocal; + socklen_t iAddrLenLocal = sizeof(stINETAddrLocal); + getsockname(iClientSockfd, (struct sockaddr *)&stINETAddrLocal, &iAddrLenLocal); + + close(iClientSockfd); + + return inet_ntoa(stINETAddrLocal.sin_addr); + } + /************************************************* + 切割字符串strOri, 以_Ch为分隔符，结果为theVec + **************************************************/ + inline void split_str(std::string strOri, char _Ch, std::vector &theVec) + { + std::string::size_type nLastPos = 0; + std::string strSub; + std::string::size_type iIndex = strOri.find(_Ch); + + if (std::string::npos == iIndex) + { + theVec.push_back(strOri); + return; + } + + while (std::string::npos != iIndex) + { + strSub = strOri.substr(nLastPos, iIndex - nLastPos); + nLastPos = iIndex + 1; + iIndex = strOri.find(_Ch, nLastPos); + theVec.push_back(strSub); + } + + if (nLastPos != 0) + { + strSub = strOri.substr(nLastPos, strOri.length() - nLastPos); + theVec.push_back(strSub); + } + } + + inline int get_bid() + { + char buf[1024]; + getcwd(buf, sizeof(buf)); + std::vector pathVec; + dtc::utils::split_str(std::string(buf), '/', pathVec); + if (pathVec.size() < 5) + { + return 0; + } + std::string strAccessKey = pathVec[4]; + std::string strModuleId = strAccessKey.substr(0, 8); + return atoi(strModuleId.c_str()); + } + } // namespace utils +} // namespace dtc + +#endif \ No newline at end of file diff --git a/src/search_local/index_storage/common/dynamic_helper_collection.cc b/src/search_local/index_storage/common/dynamic_helper_collection.cc new file mode 100644 index 0000000..0dd4549 --- /dev/null +++ b/src/search_local/index_storage/common/dynamic_helper_collection.cc @@ -0,0 +1,117 @@ +/* + * ===================================================================================== + * + * Filename: dynamic_helper_collection.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "helper_group.h" +#include "stat_dtc_def.h" + +#include "dynamic_helper_collection.h" + +DynamicHelperCollection::DynamicHelperCollection(PollThread *owner, int clientPerGroup) : TaskDispatcher(owner), + m_recvTimerList(0), m_connTimerList(0), m_retryTimerList(0), m_clientPerGroup(clientPerGroup) +{ +} + +DynamicHelperCollection::~DynamicHelperCollection() +{ + for (HelperMapType::iterator iter = m_groups.begin(); + iter != m_groups.end(); ++iter) + { + delete iter->second.group; + } + + m_groups.clear(); +} + +void DynamicHelperCollection::task_notify(TaskRequest *t) +{ + log_debug("DynamicHelperCollection::task_notify start, t->remote_addr: %s", t->remote_addr()); + HelperMapType::iterator iter = m_groups.find(t->remote_addr()); + if (iter == m_groups.end()) + { + HelperGroup *g = new HelperGroup( + t->remote_addr(), /* sock path */ + t->remote_addr(), /* name */ + m_clientPerGroup, /* helper client count */ + m_clientPerGroup * 32 /* queue size */, + DTC_FORWARD_USEC_ALL); + g->set_timer_handler(m_recvTimerList, m_connTimerList, m_retryTimerList); + g->Attach(owner, &m_taskQueueAllocator); + helper_group hg = {g, 0}; + m_groups[t->remote_addr()] = hg; + iter = m_groups.find(t->remote_addr()); + } + t->push_reply_dispatcher(this); + iter->second.group->task_notify(t); + iter->second.used = 1; + log_debug("DynamicHelperCollection::task_notify end"); +} + +void DynamicHelperCollection::reply_notify(TaskRequest *t) +{ + if (t->result_code() == 0) + { + log_debug("reply from remote dtc success,append result start "); + + if (t->result) + { + t->prepare_result(); + int iRet = t->pass_all_result(t->result); + if (iRet < 0) + { + log_notice("task append_result error: %d", iRet); + t->set_error(iRet, "DynamicHelperCollection", "append_result() error"); + t->reply_notify(); + return; + } + } + t->reply_notify(); + return; + } + else + { + log_debug("reply from remote dtc error:%d", t->result_code()); + t->reply_notify(); + return; + } +} + +void DynamicHelperCollection::set_timer_handler(TimerList *recv, + TimerList *conn, TimerList *retry) +{ + m_recvTimerList = recv; + m_connTimerList = conn; + m_retryTimerList = retry; + + attach_timer(m_retryTimerList); +} + +void DynamicHelperCollection::timer_notify() +{ + for (HelperMapType::iterator i = m_groups.begin(); i != m_groups.end();) + { + if (i->second.used == 0) + { + delete i->second.group; + m_groups.erase(i++); + } + else + { + i->second.used = 0; + ++i; + } + } +} diff --git a/src/search_local/index_storage/common/dynamic_helper_collection.h b/src/search_local/index_storage/common/dynamic_helper_collection.h new file mode 100644 index 0000000..5e999f9 --- /dev/null +++ b/src/search_local/index_storage/common/dynamic_helper_collection.h @@ -0,0 +1,58 @@ +/* + * ===================================================================================== + * + * Filename: dynamic_helper_collection.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef DYNAMIC_HELPER_COLLECTION_H__ +#define DYNAMIC_HELPER_COLLECTION_H__ + +#include +#include +#include "task_request.h" + +class HelperGroup; +class PollThread; +class TimerList; + +class DynamicHelperCollection : public TaskDispatcher, + public ReplyDispatcher, + public TimerObject +{ +public: + DynamicHelperCollection(PollThread *owner, int clientPerGroup); + ~DynamicHelperCollection(); + + void set_timer_handler(TimerList *recv, TimerList *conn, + TimerList *retry); + +private: + virtual void task_notify(TaskRequest *t); + virtual void reply_notify(TaskRequest *t); + virtual void timer_notify(); + + struct helper_group + { + HelperGroup *group; + int used; + }; + + typedef std::map HelperMapType; + HelperMapType m_groups; + LinkQueue::allocator m_taskQueueAllocator; + TimerList *m_recvTimerList, *m_connTimerList, *m_retryTimerList; + int m_clientPerGroup; +}; + +#endif diff --git a/src/search_local/index_storage/common/encode.cc b/src/search_local/index_storage/common/encode.cc new file mode 100644 index 0000000..43e3ed1 --- /dev/null +++ b/src/search_local/index_storage/common/encode.cc @@ -0,0 +1,425 @@ +/* + * ===================================================================================== + * + * Filename: encode.cc + * + * Description: field set and field value query by name. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "packet.h" +#include "table_def.h" +#include "decode.h" +#include "task_base.h" + +/* encoding DataValue by type */ +char *encode_data_value(char *p, const DTCValue *v, int type) +{ + char *t; + int n; + + switch (type) + { + case DField::None: + *p++ = 0; + break; + + case DField::Signed: + case DField::Unsigned: + if (v->s64 >= 0) + { + if (v->s64 < 0x80LL) + n = 1; + else if (v->s64 < 0x8000LL) + n = 2; + else if (v->s64 < 0x800000LL) + n = 3; + else if (v->s64 < 0x80000000LL) + n = 4; + else if (v->s64 < 0x8000000000LL) + n = 5; + else if (v->s64 < 0x800000000000LL) + n = 6; + else if (v->s64 < 0x80000000000000LL) + n = 7; + else + n = 8; + } + else + { + if (v->s64 >= -0x80LL) + n = 1; + else if (v->s64 >= -0x8000LL) + n = 2; + else if (v->s64 >= -0x800000LL) + n = 3; + else if (v->s64 >= -0x80000000LL) + n = 4; + else if (v->s64 >= -0x8000000000LL) + n = 5; + else if (v->s64 >= -0x800000000000LL) + n = 6; + else if (v->s64 >= -0x80000000000000LL) + n = 7; + else + n = 8; + } + t = (char *)&v->s64; + *p++ = n; +#if __BYTE_ORDER == __BIG_ENDIAN + for (int i = 0; i < n; i++) + p[i] = t[7 - n + i]; +#else + for (int i = 0; i < n; i++) + p[i] = t[n - 1 - i]; +#endif + p += n; + break; + + case DField::Float: + char buf[sizeof(long double) * 2 + 8]; + if (snprintf(buf, sizeof(buf), __FLTFMT__, (long double)v->flt) >= (int)sizeof(buf)) + memcpy(buf, "NAN", 4); + n = strlen(buf) + 1; + *p++ = n; + memcpy(p, buf, n); + p += n; + break; + + case DField::String: + case DField::Binary: + if (v->str.ptr == NULL) + { + *p++ = 0; + } + else + { + p = encode_length(p, v->str.len + 1); + if (v->bin.len) + memcpy(p, v->bin.ptr, v->str.len); + p += v->str.len; + *p++ = '\0'; + } + break; + } + return p; +} + +int encoded_bytes_data_value(const DTCValue *v, int type) +{ + switch (type) + { + case DField::None: + return 1; + case DField::Signed: + case DField::Unsigned: + if (v->s64 >= 0) + { + if (v->s64 < 0x80LL) + return 2; + if (v->s64 < 0x8000LL) + return 3; + if (v->s64 < 0x800000LL) + return 4; + if (v->s64 < 0x80000000LL) + return 5; + if (v->s64 < 0x8000000000LL) + return 6; + if (v->s64 < 0x800000000000LL) + return 7; + if (v->s64 < 0x80000000000000LL) + return 8; + } + else + { + if (v->s64 >= -0x80LL) + return 2; + if (v->s64 >= -0x8000LL) + return 3; + if (v->s64 >= -0x800000LL) + return 4; + if (v->s64 >= -0x80000000LL) + return 5; + if (v->s64 >= -0x8000000000LL) + return 6; + if (v->s64 >= -0x800000000000LL) + return 7; + if (v->s64 >= -0x80000000000000LL) + return 8; + } + return 9; + case DField::Float: + char b[sizeof(long double) * 2 + 8]; + if (snprintf(b, sizeof(b), __FLTFMT__, (long double)v->flt) >= (int)sizeof(b)) + return 5; + return 2 + strlen(b); + case DField::String: + case DField::Binary: + if (v->str.ptr == NULL) + return 1; + return v->str.len + 1 + encoded_bytes_length(v->str.len + 1); + } + return 0; +} + +/* + * Encoding simple section + * ... + */ +int encoded_bytes_simple_section(const SimpleSection &sct, uint8_t kt) +{ + int len = 0; + for (int i = 0; i <= sct.max_tags(); i++) + { + if (sct.tag_present(i) == 0) + continue; + const int t = i == 0 ? kt : sct.tag_type(i); + len += 1 + encoded_bytes_data_value(sct.get_tag(i), t); + } + return len; +} + +char *encode_simple_section(char *p, const SimpleSection &sct, uint8_t kt) +{ + for (int i = 0; i <= sct.max_tags(); i++) + { + if (sct.tag_present(i) == 0) + continue; + const int t = i == 0 ? kt : sct.tag_type(i); + *p++ = i; + p = encode_data_value(p, sct.get_tag(i), t); + } + return p; +} + +/* + * FieldSet format: + * ... + * NUM: 1 byte, total fields + * ID: 1 byte per fieldid + */ +int encoded_bytes_field_set(const DTCFieldSet &fs) +{ + if (fs.num_fields() == 0) + return 0; + if (fs.field_present(0)) + { + return fs.num_fields() + 2; + } + return fs.num_fields() + 1; +} + +char *encode_field_set(char *p, const DTCFieldSet &fs) +{ + if (fs.num_fields() == 0) + return p; + *p++ = fs.num_fields(); + for (int i = 0; i < fs.num_fields(); i++) + { + *p++ = fs.field_id(i); + if (fs.field_id(i) == 0) + *p++ = 0; + } + return p; +} + +/* + * field_value format: + * ... + * NUM: 1 byte, total fields + * ID: 1 byte + */ +int encoded_bytes_field_value(const DTCFieldValue &fv) +{ + if (fv.num_fields() == 0) + return 0; + int len = 1; + for (int i = 0; i < fv.num_fields(); i++) + { + //for migrate + if (fv.field_id(i) == 0) + len++; + len += 2 + encoded_bytes_data_value(fv.field_value(i), fv.field_type(i)); + } + return len; +} + +char *encode_field_value(char *p, const DTCFieldValue &fv) +{ + if (fv.num_fields() == 0) + return p; + *p++ = fv.num_fields(); + for (int i = 0; i < fv.num_fields(); i++) + { + const int n = fv.field_id(i); + const int t = fv.field_type(i); + *p++ = (fv.field_operation(i) << 4) + t; + *p++ = n; + //for migrate + if (n == 0) + *p++ = 0; + + p = encode_data_value(p, fv.field_value(i), t); + } + return p; +} + +int encoded_bytes_multi_key(const DTCValue *v, const DTCTableDefinition *tdef) +{ + if (tdef->key_fields() <= 1) + return 0; + int len = 1; + for (int i = 1; i < tdef->key_fields(); i++) + len += 2 + encoded_bytes_data_value(v + i, tdef->field_type(i)); + return len; +} + +char *encode_multi_key(char *p, const DTCValue *v, const DTCTableDefinition *tdef) +{ + if (tdef->key_fields() <= 1) + return p; + *p++ = tdef->key_fields() - 1; + for (int i = 1; i < tdef->key_fields(); i++) + { + const int t = tdef->field_type(i); + *p++ = ((DField::Set) << 4) + t; + *p++ = i; + p = encode_data_value(p, v + i, t); + } + return p; +} + +/* + * FieldSet format: + * ... + * NUM: 1 byte, total fields + * FIELD: encoded field ID/name + */ +int encoded_bytes_field_set(const FieldSetByName &fs) +{ + if (fs.num_fields() == 0) + return 0; + int len = 1; + for (int i = 0; i < fs.num_fields(); i++) + { + switch (fs.field_id(i)) + { + case 255: + len += 2 + fs.field_name_length(i); + break; + case 0: + len += 2; + break; + default: + len += 1; + } + } + return len; +} + +char *encode_field_set(char *p, const FieldSetByName &fs) +{ + if (fs.num_fields() == 0) + return p; + *p++ = fs.num_fields(); + for (int i = 0; i < fs.num_fields(); i++) + { + switch (fs.field_id(i)) + { + case 0: + *p++ = 0; + *p++ = 0; + break; + + default: + *p++ = fs.field_id(i); + break; + + case 255: + *p++ = 0; + const int n = fs.field_name_length(i); + *p++ = n; + memcpy(p, fs.field_name(i), n); + p += n; + break; + } + } + return p; +} + +/* + * field_value format: + * + * NUM: 1 byte, total fields + * FIELD: encoded field ID/name + */ +int encoded_bytes_field_value(const FieldValueByName &fv) +{ + if (fv.num_fields() == 0) + return 0; + int len = 1; + for (int i = 0; i < fv.num_fields(); i++) + { + switch (fv.field_id(i)) + { + case 255: + len += 3 + fv.field_name_length(i); + break; + case 0: + len += 3; + break; + default: + len += 2; + } + len += encoded_bytes_data_value(fv.field_value(i), fv.field_type(i)); + } + return len; +} + +char *encode_field_value(char *p, const FieldValueByName &fv) +{ + if (fv.num_fields() == 0) + return p; + *p++ = fv.num_fields(); + for (int i = 0; i < fv.num_fields(); i++) + { + int n = fv.field_name_length(i); + int t = fv.field_type(i); + if (t == DField::Unsigned) + t = DField::Signed; + *p++ = (fv.field_operation(i) << 4) + t; + switch (fv.field_id(i)) + { + case 0: + *p++ = 0; + *p++ = 0; + break; + case 255: + *p++ = 0; + *p++ = n; + memcpy(p, fv.field_name(i), n); + p += n; + break; + default: + *p++ = fv.field_id(i); + } + + p = encode_data_value(p, fv.field_value(i), t); + } + return p; +} diff --git a/src/search_local/index_storage/common/field.cc b/src/search_local/index_storage/common/field.cc new file mode 100644 index 0000000..5db6088 --- /dev/null +++ b/src/search_local/index_storage/common/field.cc @@ -0,0 +1,437 @@ +/* + * ===================================================================================== + * + * Filename: field.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include "field.h" +#include "buffer_error.h" +#include "bitsop.h" +#include "log.h" + +#define TP(x, y, z) (((x) << 16) + ((y) << 8) + (z)) +#define TC(x, y, z) ((DField::x << 16) + (DField::y << 8) + DField::z) + +static void TruncateIntValue(DTCValue &v1, int type, int size) +{ + switch (size * 2 + type) + { + default: + break; + case 1 * 2 + DField::Signed: + if (v1.s64 < -0x80) + v1.s64 = -0x80; + else if (v1.s64 > 0x7F) + v1.s64 = 0x7F; + break; + case 1 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFF) + v1.s64 = 0xFF; + break; + case 2 * 2 + DField::Signed: + if (v1.s64 < -0x8000) + v1.s64 = -0x8000; + else if (v1.s64 > 0x7FFF) + v1.s64 = 0x7FFF; + break; + case 2 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFF) + v1.s64 = 0xFFFF; + break; + case 3 * 2 + DField::Signed: + if (v1.s64 < -0x800000) + v1.s64 = -0x800000; + else if (v1.s64 > 0x7FFFFF) + v1.s64 = 0x7FFFFF; + break; + case 3 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFFFF) + v1.s64 = 0xFFFFFF; + break; + case 4 * 2 + DField::Signed: + if (v1.s64 < -0x80000000LL) + v1.s64 = -0x80000000LL; + else if (v1.s64 > 0x7FFFFFFF) + v1.s64 = 0x7FFFFFFF; + break; + case 4 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFFFFFFLL) + v1.s64 = 0xFFFFFFFFLL; + break; + case 5 * 2 + DField::Signed: + if (v1.s64 < -0x8000000000LL) + v1.s64 = -0x8000000000LL; + else if (v1.s64 > 0x7FFFFFFFFFLL) + v1.s64 = 0x7FFFFFFFFFLL; + break; + case 5 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFFFFFFFFLL) + v1.s64 = 0xFFFFFFFFFFLL; + break; + case 6 * 2 + DField::Signed: + if (v1.s64 < -0x800000000000LL) + v1.s64 = -0x800000000000LL; + else if (v1.s64 > 0x7FFFFFFFFFFFLL) + v1.s64 = 0x7FFFFFFFFFFFLL; + break; + case 6 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFFFFFFFFFFLL) + v1.s64 = 0xFFFFFFFFFFFFLL; + break; + case 7 * 2 + DField::Signed: + if (v1.s64 < -0x80000000000000LL) + v1.s64 = -0x80000000000000LL; + else if (v1.s64 > 0x7FFFFFFFFFFFFFLL) + v1.s64 = 0x7FFFFFFFFFFFFFLL; + break; + case 7 * 2 + DField::Unsigned: + if (v1.s64 < 0) + v1.s64 = 0; + else if (v1.s64 > 0xFFFFFFFFFFFFFFLL) + v1.s64 = 0xFFFFFFFFFFFFFFLL; + break; + } +} +/* + * Update RowValue by Operation:this: + * left side: RowValue + * right side: this + * valid right side type: Signed, Float, String. + * Unsigned encoded as Signed, Binary encoded as String + */ +int DTCFieldValue::Update(RowValue &r) +{ + for (int i = 0; i < num_fields(); i++) + { + const int id = fieldValue[i].id; + DTCValue &v1 = r[id]; + const DTCValue &v2 = fieldValue[i].val; + + switch (TP(r.field_type(id), fieldValue[i].type, fieldValue[i].oper)) + { + default: + log_notice("id:%d TC(%d,%d,%d) not surport", id, r.field_type(id), fieldValue[i].type, fieldValue[i].oper); + break; + case TC(Signed, Signed, Set): + case TC(Unsigned, Signed, Set): + case TC(Unsigned, Unsigned, Set): + v1.s64 = v2.s64; + TruncateIntValue(v1, r.field_type(id), r.field_size(id)); + break; + + case TC(Signed, Signed, Add): + case TC(Unsigned, Signed, Add): + if (v1.s64 > 0) + { + v1.s64 += v2.s64; + if (v2.s64 > 0 && v1.s64 < 0) + { + /* overflow */ + v1.s64 = 0x7FFFFFFFFFFFFFFFLL; + } + } + else + { + v1.s64 += v2.s64; + if (v2.s64 < 0 && v1.s64 > 0) + { + /* overflow */ + v1.s64 = 0x8000000000000000LL; + } + } + TruncateIntValue(v1, r.field_type(id), r.field_size(id)); + break; + + case TC(Signed, Signed, OR): + case TC(Unsigned, Signed, OR): + v1.s64 |= v2.s64; + TruncateIntValue(v1, r.field_type(id), r.field_size(id)); + break; + + case TC(Float, Signed, Set): + v1.flt = v2.s64; + break; + case TC(Float, Unsigned, Set): + v1.flt = v2.u64; + break; + case TC(Float, Float, Set): + v1.flt = v2.flt; + break; + + case TC(Float, Signed, Add): + v1.flt += v2.s64; + break; + case TC(Float, Unsigned, Add): + v1.flt += v2.u64; + break; + case TC(Float, Float, Add): + v1.flt += v2.flt; + break; + + case TC(String, String, Set): + v1.str = v2.str; + if (v1.str.len > r.field_size(id)) + v1.str.len = r.field_size(id); + break; + case TC(Binary, String, Set): + v1.bin = v2.bin; + if (v1.str.len > r.field_size(id)) + v1.str.len = r.field_size(id); + break; + case TC(Binary, Binary, Set): + v1.bin = v2.bin; + if (v1.str.len > r.field_size(id)) + v1.str.len = r.field_size(id); + break; + + //setbits operation + case TC(Signed, Signed, SetBits): + case TC(Signed, Unsigned, SetBits): + case TC(Unsigned, Signed, SetBits): + case TC(Unsigned, Unsigned, SetBits): + { + const int len = 8; + unsigned int off = v2.u64 >> 32; + unsigned int size = off >> 24; + off &= 0xFFFFFF; + unsigned int value = v2.u64 & 0xFFFFFFFF; + + if (off >= 8 * len || size == 0) + break; + if (size > 32) + size = 32; + if (size > 8 * len - off) + size = 8 * len - off; + + log_debug("SetMultBits, off:%d, size:%d, value:%d", off, size, value); + + uint64_t mask = ((1ULL << size) - 1) << off; + v1.u64 &= ~mask; + v1.u64 |= ((uint64_t)value << off) & mask; + } + break; + case TC(Binary, Signed, SetBits): + case TC(Binary, Unsigned, SetBits): + case TC(String, Signed, SetBits): + case TC(String, Unsigned, SetBits): + { + const int len = v1.bin.len; + int off = v2.u64 >> 32; + int size = off >> 24; + off &= 0xFFFFFF; + unsigned int value = v2.u64 & 0xFFFFFFFF; + if (off >= 8 * len || size == 0) + break; + if (size > 32) + size = 32; + if (size > 8 * len - off) + size = 8 * len - off; + + for (size += off; off < size; off++, value >>= 1) + { + if ((value & 1) == 0) + CLR_B(off, v1.bin.ptr); + else + SET_B(off, v1.bin.ptr); + } + } + break; + } + } + return 0; +} + +int DTCFieldValue::Compare(const RowValue &r, int iCmpFirstNRows) +{ + for (int i = 0; i < num_fields(); i++) + { + const int id = fieldValue[i].id; + if (id < r.table_definition()->key_fields() || id > iCmpFirstNRows - 1) + continue; + const DTCValue &v1 = r[id]; + const DTCValue &v2 = fieldValue[i].val; + + switch (TP(r.field_type(id), fieldValue[i].type, fieldValue[i].oper)) + { + default: + return 0; + case TC(Signed, Signed, EQ): + case TC(Unsigned, Signed, EQ): + if (v1.s64 == v2.s64) + break; + else + return 0; + + case TC(Signed, Signed, NE): + case TC(Unsigned, Signed, NE): + if (v1.s64 != v2.s64) + break; + else + return 0; + + case TC(Signed, Signed, LT): + case TC(Unsigned, Signed, LT): + if (v1.s64 < v2.s64) + break; + else + return 0; + + case TC(Signed, Signed, LE): + case TC(Unsigned, Signed, LE): + if (v1.s64 <= v2.s64) + break; + else + return 0; + + case TC(Signed, Signed, GT): + case TC(Unsigned, Signed, GT): + if (v1.s64 > v2.s64) + break; + else + return 0; + + case TC(Signed, Signed, GE): + case TC(Unsigned, Signed, GE): + if (v1.s64 >= v2.s64) + break; + else + return 0; + + /* case insensitive for string comparison */ + case TC(String, String, EQ): + if (string_equal(v1, v2)) + break; + else + return 0; + case TC(String, String, NE): + if (!string_equal(v1, v2)) + break; + else + return 0; + + /* case sensitive for binary comparison */ + case TC(Binary, Binary, EQ): + case TC(String, Binary, EQ): + case TC(Binary, String, EQ): + if (binary_equal(v1, v2)) + break; + else + return 0; + case TC(Binary, Binary, NE): + case TC(String, Binary, NE): + case TC(Binary, String, NE): + if (!binary_equal(v1, v2)) + break; + else + return 0; + } + } + return 1; +} + +int RowValue::Compare(const RowValue &rv, uint8_t *fieldIDList, uint8_t num) +{ + for (int i = 0; i < num; ++i) + { + switch (field_type(fieldIDList[i])) + { + case DField::Signed: + case DField::Unsigned: + if (value[fieldIDList[i]].u64 != rv.field_value(fieldIDList[i])->u64) + return -1; + break; + case DField::Float: //浮点数 + //浮点数不允许比较 + return -2; + break; + case DField::String: //字符串 + if (!string_equal(value[fieldIDList[i]], *(rv.field_value(fieldIDList[i])))) + return -1; + break; + case DField::Binary: //二进制数据 + if (!binary_equal(value[fieldIDList[i]], *(rv.field_value(fieldIDList[i])))) + return -1; + break; + default: + return -3; + break; + } + } + return 0; +}; + +int RowValue::Copy(const RowValue *r) +{ + // left value: target, right value: source + int id = -1; + for (int i = 0; i < num_fields(); ++i) + { + if ((id = r->field_id(field_name(i + 1))) == -1) + continue; + memcpy(value + i + 1, r->value + id, sizeof(DTCValue)); + } + return 0; +} + +void RowValue::update_timestamp(uint32_t now, int updateall) +{ + const DTCTableDefinition *tdef = table_definition(); + int id; + + id = tdef->lastacc_field_id(); + if (id > 0) + { + value[id].u64 = now; + } + + id = tdef->lastmod_field_id(); + if (id > 0) + { + value[id].u64 = now; + } + + id = tdef->lastcmod_field_id(); + if (id > 0 && updateall) + { + value[id].u64 = now; + } +} + +void RowValue::update_expire_time() +{ + const DTCTableDefinition *tdef = table_definition(); + int id = tdef->expire_time_field_id(); + if (id > 0) + { + if (value[id].s64 > 0) + { + value[id].s64 = value[id].s64 - time(NULL); + } + } +} diff --git a/src/search_local/index_storage/common/field.h b/src/search_local/index_storage/common/field.h new file mode 100644 index 0000000..7e6a89d --- /dev/null +++ b/src/search_local/index_storage/common/field.h @@ -0,0 +1,385 @@ +/* + * ===================================================================================== + * + * Filename: field.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_FIELD_H__ +#define __CH_FIELD_H__ + +#include +#include + +#include "value.h" +#include "table_def.h" +#include "protocol.h" +#include "mem_check.h" + +class DTCFieldValue; +class FieldSetByName; +class FieldValueByName; + +class DTCFieldSet +{ +private: + uint8_t *fieldId; + uint8_t fieldMask[32]; + +public: + DTCFieldSet(const DTCFieldSet &fs) + { + int n = fs.fieldId[-2]; + fieldId = (uint8_t *)MALLOC(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + memcpy(fieldId, fs.fieldId - 2, n + 2); + memcpy(fieldMask, fs.fieldMask, sizeof(fieldMask)); + fieldId += 2; + } + DTCFieldSet(int n) + { + if (n > 255) + n = 255; + fieldId = (uint8_t *)MALLOC(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + fieldId += 2; + fieldId[-2] = n; + memset(fieldId - 1, 0, n + 1); + memset(fieldMask, 0, sizeof(fieldMask)); + } + DTCFieldSet(const uint8_t *idtab, int n) + { + if (n > 255) + n = 255; + fieldId = (uint8_t *)MALLOC(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + *fieldId++ = n; + *fieldId++ = n; + memcpy(fieldId, idtab, n); + build_field_mask(fieldMask); + } + + /* allocate max field cnt, real field num in first and second byte */ + DTCFieldSet(const uint8_t *idtab, int n, int total) + { + if (n > 255) + n = 255; + if (total > 255) + total = 255; + fieldId = (uint8_t *)MALLOC(total + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + *fieldId++ = total; + *fieldId++ = n; + memcpy(fieldId, idtab, n); + memset(fieldMask, 0, sizeof(fieldMask)); + build_field_mask(fieldMask); + } + + inline void Clean() + { + fieldId[-1] = 0; + memset(fieldMask, 0, sizeof(fieldMask)); + } + + /* clean before set */ + inline int Set(const uint8_t *idtab, int n) + { + if (n > 255) + n = 255; + if (fieldId == NULL) + return -1; + fieldId[-1] = n; + memcpy(fieldId, idtab, n); + build_field_mask(fieldMask); + return 0; + } + + inline int max_fields() { return fieldId[-2]; } + inline void Realloc(int total) + { + fieldId -= 2; + fieldId = (uint8_t *)REALLOC(fieldId, total + 2); + *fieldId = total; + fieldId += 2; + } + + int Copy(const FieldSetByName &); + ~DTCFieldSet() + { + if (fieldId) + FREE(fieldId - 2); + } + + int num_fields(void) const { return fieldId[-1]; } + int field_id(int n) const + { + return n >= 0 && n < num_fields() ? fieldId[n] : 0; + } + void add_field(int id) + { + const int n = fieldId[-1]; + if (n >= fieldId[-2]) + return; + fieldId[n] = id; + fieldId[-1]++; + FIELD_SET(id, fieldMask); + } + int field_present(int id) const + { + if (id >= 0 && id < 255) + return FIELD_ISSET(id, fieldMask); + return 0; + } + void build_field_mask(uint8_t *mask) const + { + for (int i = 0; i < num_fields(); i++) + FIELD_SET(fieldId[i], mask); + } +}; + +class RowValue : public TableReference +{ +private: + DTCValue *value; + +public: + RowValue(DTCTableDefinition *t) : TableReference(t) + { + value = (DTCValue *)calloc(num_fields() + 1, sizeof(DTCValue)); + if (value == NULL) + throw std::bad_alloc(); + }; + + RowValue(const RowValue &r) : TableReference(r.table_definition()) + { + value = (DTCValue *)calloc(num_fields() + 1, sizeof(DTCValue)); + if (value == NULL) + throw std::bad_alloc(); + memcpy(value, r.value, sizeof(DTCValue) * (num_fields() + 1)); + } + + virtual ~RowValue() + { + FREE_IF(value); + }; + + inline void Clean() + { + memset(value, 0, sizeof(DTCValue) * (num_fields() + 1)); + } + +#if 0 + int key_fields(void) const { return tableDef->key_fields(); } + int num_fields(void) const { return tableDef->num_fields(); } + int field_type(int n) const { return tableDef->field_type(n); } + int field_size(int n) const { return tableDef->field_size(n); } + int field_b_size(int n) const {return tableDef->field_b_size(n);} + int field_b_offset(int n) const {return tableDef->field_b_offset(n);} + int is_same_table(const RowValue &rv) const { return tableDef->is_same_table(rv.tableDef); } + int is_same_table(const RowValue *rv) const { return rv ? is_same_table(*rv) : 0; } + const DTCTableDefinition *table_definition(void) const { return tableDef; } +#endif +#if !CLIENTAPI + // this macro test is scope-test only, didn't affected the class implementation + void default_value(void) { get_default_row(value); } +#endif + int is_same_table(const RowValue &rv) const + { + return is_same_table(rv.table_definition()); + } + int is_same_table(const RowValue *rv) const { return rv ? is_same_table(rv->table_definition()) : 0; } + + /*Compare tow RowValue by FieldIDList*/ + int Compare(const RowValue &rv, uint8_t *fieldIDList, uint8_t num); + DTCValue &operator[](int n) { return value[n]; } + const DTCValue &operator[](int n) const { return value[n]; } + DTCValue *field_value(int id) + { + return id >= 0 && id <= num_fields() ? &value[id] : NULL; + } + const DTCValue *field_value(int id) const + { + return id >= 0 && id <= num_fields() ? &value[id] : NULL; + } + DTCValue *field_value(const char *name) + { + return field_value(field_id(name)); + } + const DTCValue *field_value(const char *name) const + { + return field_value(field_id(name)); + } + void copy_value(const DTCValue *v, int id, int n) + { + memcpy(&value[id], v, n * sizeof(DTCValue)); + } + int Copy(const RowValue *r); + void update_timestamp(uint32_t now, int updateall /*update all timestamp, include lastcmod*/); + void update_expire_time(); +}; + +class DTCFieldValue +{ +private: + struct SFieldValue + { + uint8_t id; + uint8_t oper; + uint8_t type; + DTCValue val; + } * fieldValue; + + //total + int maxFields; + //real + int numFields; + FieldDefinition::fieldflag_t typeMask[2]; + +public: + DTCFieldValue(int total) + { + fieldValue = NULL; + maxFields = numFields = 0; + if (total <= 0) + return; +#if ROCKSDB_COMPILER + fieldValue = (decltype(fieldValue))MALLOC(total * sizeof(*fieldValue)); +#else + fieldValue = (typeof(fieldValue))MALLOC(total * sizeof(*fieldValue)); +#endif + if (fieldValue == NULL) + throw(-ENOMEM); + maxFields = total; + typeMask[0] = 0; + typeMask[1] = 0; + } + DTCFieldValue(const DTCFieldValue &fv, int sparse = 0) + { + numFields = fv.numFields; + typeMask[0] = fv.typeMask[0]; + typeMask[1] = fv.typeMask[1]; + if (sparse < 0) + sparse = 0; + sparse += fv.numFields; + maxFields = sparse; + if (fv.fieldValue != NULL) + { +#if ROCKSDB_COMPILER + fieldValue = (decltype(fieldValue))MALLOC(sparse * sizeof(*fieldValue)); +#else + fieldValue = (typeof(fieldValue))MALLOC(sparse * sizeof(*fieldValue)); +#endif + if (fieldValue == NULL) + throw(-ENOMEM); + memcpy(fieldValue, fv.fieldValue, fv.numFields * sizeof(*fieldValue)); + } + else + { + fieldValue = NULL; + } + } + int Copy(const FieldValueByName &, int mode, const DTCTableDefinition *); + ~DTCFieldValue() + { + FREE_IF(fieldValue); + } + + /* should be inited as just constructed */ + inline void Clean() + { + numFields = 0; + memset(typeMask, 0, 2 * sizeof(FieldDefinition::fieldflag_t)); + } + + inline void Realloc(int total) + { + maxFields = total; + fieldValue = (struct SFieldValue *)REALLOC(fieldValue, sizeof(struct SFieldValue) * total); + } + + int max_fields(void) const { return maxFields; } + int num_fields(void) const { return numFields; } + + int field_id(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].id : 0; + } + + int field_type(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].type : DField::None; + } + + int field_operation(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].oper : 0; + } + + DTCValue *field_value(int n) const + { + return n >= 0 && n < numFields ? &fieldValue[n].val : NULL; + } + + void add_value(uint8_t id, uint8_t op, uint8_t t, const DTCValue &val) + { + if (numFields == maxFields) + return; + fieldValue[numFields].id = id; + fieldValue[numFields].oper = op; + fieldValue[numFields].type = t; + fieldValue[numFields].val = val; + numFields++; + } + DTCValue *next_field_value() + { + if (numFields == maxFields) + return NULL; + return &fieldValue[numFields].val; + } + void add_value_no_val(uint8_t id, uint8_t op, uint8_t t) + { + if (numFields == maxFields) + return; + fieldValue[numFields].id = id; + fieldValue[numFields].oper = op; + fieldValue[numFields].type = t; + numFields++; + } + void update_type_mask(unsigned int flag) + { + typeMask[0] |= flag; + typeMask[1] |= ~flag; + } + int has_type_ro(void) const { return typeMask[0] & FieldDefinition::FF_READONLY; } + int has_type_rw(void) const { return typeMask[1] & FieldDefinition::FF_READONLY; } + int has_type_sync(void) const { return 1; } + int has_type_async(void) const { return 0; } + int has_type_volatile(void) const { return typeMask[0] & FieldDefinition::FF_VOLATILE; } + int has_type_commit(void) const { return typeMask[1] & FieldDefinition::FF_VOLATILE; } + int has_type_timestamp(void) const { return typeMask[0] & FieldDefinition::FF_TIMESTAMP; } + + void build_field_mask(uint8_t *mask) const + { + for (int i = 0; i < num_fields(); i++) + FIELD_SET(fieldValue[i].id, mask); + } + + int Update(RowValue &); + int Compare(const RowValue &, int iCmpFirstNRows = 256); +}; + +#endif diff --git a/src/search_local/index_storage/common/field_api.cc b/src/search_local/index_storage/common/field_api.cc new file mode 100644 index 0000000..7221dc7 --- /dev/null +++ b/src/search_local/index_storage/common/field_api.cc @@ -0,0 +1,334 @@ +/* + * ===================================================================================== + * + * Filename: filed_api.cc + * + * Description: field set and field value query by name. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include "field.h" +#include "field_api.h" +#include "buffer_error.h" +#include "bitsop.h" +#include "log.h" + +int check_int_value(const DTCValue &v1, int type, int size) +{ + int res = 0; + switch (size * 2 + type) + { + default: + break; + case 1 * 2 + DField::Signed: + if (v1.s64 < -0x80 || v1.s64 > 0x7F) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 1 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 2 * 2 + DField::Signed: + if (v1.s64 < -0x8000 || v1.s64 > 0x7FFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 2 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 3 * 2 + DField::Signed: + if (v1.s64 < -0x800000 || v1.s64 > 0x7FFFFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 3 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFFFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 4 * 2 + DField::Signed: + if (v1.s64 < -0x80000000LL || v1.s64 > 0x7FFFFFFF) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 4 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 5 * 2 + DField::Signed: + if (v1.s64 < -0x8000000000LL || v1.s64 > 0x7FFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 5 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 6 * 2 + DField::Signed: + if (v1.s64 < -0x800000000000LL || v1.s64 > 0x7FFFFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 6 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFFFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 7 * 2 + DField::Signed: + if (v1.s64 < -0x80000000000000LL || v1.s64 > 0x7FFFFFFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + case 7 * 2 + DField::Unsigned: + if (v1.s64 < 0 || v1.s64 > 0xFFFFFFFFFFFFFFLL) + res = -EC_BAD_FIELD_SIZE_ON_RESOLVE; + break; + } + return res; +} + +/* + * resolve fieldname at client side + * solved: all field has resolved + */ +int FieldValueByName::Resolve(const DTCTableDefinition *tdef, int force) +{ + if (force) + solved = 0; + else if (solved) + return 0; + else if (numFields == 0) + return 0; + + if (tdef == NULL) + return -EINVAL; + + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].name == NULL) + continue; + const int fid = tdef->field_id(fieldValue[i].name, fieldValue[i].nlen); + if (fid < 0) + return -EC_BAD_FIELD_NAME; + + if (tdef->field_type(fid) == 1 || tdef->field_type(fid) == 2) + { + int res = check_int_value(fieldValue[i].val, tdef->field_type(fid), tdef->field_size(fid)); + if (res < 0) + { + return res; + } + } + else if (tdef->field_type(fid) == 4 || tdef->field_type(fid) == 5) + { + if (tdef->field_size(fid) < fieldValue[i].val.str.len) + { + return -EC_BAD_FIELD_SIZE_ON_RESOLVE; + } + } + else if (tdef->field_type(fid) == 3) + { + continue; + } + else + { + return -EC_EXCEPTION_ON_RESOLVE; + } + fieldValue[i].fid = fid; + } + solved = 1; + return 0; +} + +/* + * resolve fieldname at client side + * solved: all field has resolved + */ +void FieldValueByName::Unresolve(void) +{ + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].name != NULL) + { + fieldValue[i].fid = INVALID_FIELD_ID; + solved = 0; + } + } +} + +/* + * resolve fieldname at client side + * solved: all field has resolved + */ +int FieldSetByName::Resolve(const DTCTableDefinition *tdef, int force) +{ + if (force) + solved = 0; + else if (solved) + return 0; + else if (numFields == 0) + return 0; + if (tdef == NULL) + return -EINVAL; + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].name == NULL) + continue; + const int fid = tdef->field_id(fieldValue[i].name, fieldValue[i].nlen); + // if(fid<=0) + if (fid < 0) + return -EC_BAD_FIELD_NAME; + fieldValue[i].fid = fid; + } + solved = 1; + return 0; +} + +/* + * resolve fieldname at client side + * solved: all field has resolved + */ +void FieldSetByName::Unresolve(void) +{ + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].name != NULL) + { + fieldValue[i].fid = INVALID_FIELD_ID; + solved = 0; + } + } +} + +const uint8_t *FieldSetByName::virtual_map(void) const +{ + if (maxvid == 0) + return NULL; + if (solved == 0) + return NULL; + uint8_t *m = (uint8_t *)calloc(1, maxvid); + if (m == NULL) + throw std::bad_alloc(); + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].vid) + m[fieldValue[i].vid - 1] = fieldValue[i].fid; + } + return m; +} + +int FieldSetByName::add_field(const char *name, int vid) +{ + int nlen = strlen(name); + if (nlen >= 1024) + { + return -EC_BAD_FIELD_NAME; + } + if (vid < 0 || vid >= 256) + { + vid = 0; + return -EINVAL; + } + else if (vid) + { + for (int i = 0; i < numFields; i++) + { + if (fieldValue[i].vid == vid) + return -EINVAL; + } + } + if (numFields == maxFields) + { + if (maxFields == 255) + return -E2BIG; + int n = maxFields + 8; + if (n > 255) + n = 255; + typeof(fieldValue) p; + if (fieldValue == NULL) + { + p = (typeof(fieldValue))MALLOC(n * sizeof(*fieldValue)); + } + else + { + p = (typeof(fieldValue))REALLOC(fieldValue, n * sizeof(*fieldValue)); + } + if (p == NULL) + throw std::bad_alloc(); + fieldValue = p; + maxFields = n; + } + + char *str = (char *)MALLOC(nlen + 1); + if (str == NULL) + throw std::bad_alloc(); + memcpy(str, name, nlen + 1); + fieldValue[numFields].name = str; + fieldValue[numFields].nlen = nlen; + fieldValue[numFields].fid = INVALID_FIELD_ID; // allow select key-field + fieldValue[numFields].vid = vid; + if (vid > maxvid) + maxvid = vid; + solved = 0; + numFields++; + return 0; +} + +int FieldValueByName::add_value(const char *name, uint8_t op, uint8_t type, const DTCValue &val) +{ + int nlen = strlen(name); + if (nlen >= 1024) + { + return -EC_BAD_FIELD_NAME; + } + if (numFields == maxFields) + { + if (maxFields == 255) + return -E2BIG; + int n = maxFields + 8; + if (n > 255) + n = 255; + typeof(fieldValue) p; + if (fieldValue == NULL) + { + p = (typeof(fieldValue))MALLOC(n * sizeof(*fieldValue)); + } + else + { + p = (typeof(fieldValue))REALLOC(fieldValue, n * sizeof(*fieldValue)); + } + if (p == NULL) + throw std::bad_alloc(); + fieldValue = p; + maxFields = n; + } + + char *str = (char *)MALLOC(nlen + 1); + if (str == NULL) + throw std::bad_alloc(); + memcpy(str, name, nlen + 1); + fieldValue[numFields].name = str; + fieldValue[numFields].nlen = nlen; + fieldValue[numFields].type = type; + fieldValue[numFields].fid = INVALID_FIELD_ID; + fieldValue[numFields].oper = op; + fieldValue[numFields].val = val; + if (type == DField::String || type == DField::Binary) + { + if (val.bin.ptr != NULL) + { + char *p = (char *)MALLOC(val.bin.len + 1); + if (p == NULL) + throw std::bad_alloc(); + memcpy(p, val.bin.ptr, val.bin.len); + p[val.bin.len] = '\0'; + fieldValue[numFields].val.bin.ptr = p; + } + } + solved = 0; + numFields++; + return 0; +} diff --git a/src/search_local/index_storage/common/field_api.h b/src/search_local/index_storage/common/field_api.h new file mode 100644 index 0000000..6d52732 --- /dev/null +++ b/src/search_local/index_storage/common/field_api.h @@ -0,0 +1,182 @@ +/* + * ===================================================================================== + * + * Filename: filed_api.h + * + * Description: field set and field value query by name. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_FIELD_API_H__ +#define __CH_FIELD_API_H__ + +#define INVALID_FIELD_ID 255 +class FieldSetByName +{ +private: + uint16_t maxFields; + uint8_t numFields; + uint8_t solved; + uint8_t maxvid; + struct + { + uint8_t fid; + uint8_t vid; + uint8_t nlen; + char *name; + } * fieldValue; + + FieldSetByName(const FieldSetByName &); // NOT IMPLEMENTED +public: + FieldSetByName(void) : maxFields(0), numFields(0), solved(1), maxvid(0), fieldValue(NULL) {} + ~FieldSetByName() + { + if (fieldValue) + { + for (int i = 0; i < numFields; i++) + FREE(fieldValue[i].name); + FREE(fieldValue); + } + } + + int Solved(void) const { return solved; } + int Resolve(const DTCTableDefinition *, int); + void Unresolve(void); + + int max_fields(void) const { return maxFields; } + int num_fields(void) const { return numFields; } + + int field_present(const char *name) const + { + for (int i = 0; i < numFields; i++) + if (strncasecmp(fieldValue[i].name, name, 256) == 0) + return 1; + return 0; + } + + const char *field_name(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].name : NULL; + } + + int field_name_length(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].nlen : 255; + } + + int field_id(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].fid : 255; + } + + int virtual_id(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].vid : 0; + } + + int max_virtual_id(void) const + { + return maxvid; + } + + int add_field(const char *name, int vid); + int field_v_id(const char *name) const + { + for (int i = 0; i < numFields; i++) + if (strncasecmp(fieldValue[i].name, name, 256) == 0) + return fieldValue[i].vid; + return -1; + } + + const uint8_t *virtual_map(void) const; +}; + +extern int check_int_value(const DTCValue &v1, int type, int size); + +class FieldValueByName +{ +private: + uint16_t maxFields; + uint8_t numFields; + uint8_t solved; + + struct + { + uint8_t type; + uint8_t oper; + uint8_t fid; + uint8_t nlen; + char *name; + DTCValue val; + } * fieldValue; + + FieldValueByName(const FieldValueByName &); // NOT IMPLEMENTED +public: + FieldValueByName(void) : maxFields(0), numFields(0), solved(1), fieldValue(NULL) + { + } + + ~FieldValueByName() + { + if (fieldValue) + { + for (int i = 0; i < numFields; i++) + { + FREE_IF(fieldValue[i].name); + if (fieldValue[i].type == DField::String || + fieldValue[i].type == DField::Binary) + FREE_IF(fieldValue[i].val.bin.ptr); + } + FREE(fieldValue); + } + } + + int Solved(void) const { return solved; } + int Resolve(const DTCTableDefinition *, int); + void Unresolve(void); + + int max_fields(void) const { return maxFields; } + int num_fields(void) const { return numFields; } + + const char *field_name(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].name : NULL; + } + + int field_name_length(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].nlen : 0; + } + + int field_id(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].fid : 255; + } + + int field_type(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].type : DField::None; + } + + int field_operation(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].oper : 0; + } + + const DTCValue *field_value(int n) const + { + return n >= 0 && n < numFields ? &fieldValue[n].val : NULL; + } + + int add_value(const char *n, uint8_t op, uint8_t t, const DTCValue &v); +}; + +#endif diff --git a/src/search_local/index_storage/common/field_export.h b/src/search_local/index_storage/common/field_export.h new file mode 100644 index 0000000..4c90a08 --- /dev/null +++ b/src/search_local/index_storage/common/field_export.h @@ -0,0 +1,266 @@ +/* + * ===================================================================================== + * + * Filename: field_export.h + * + * Description: field: column value of table + * FieldSet: set of field + * field_value: set of combo(field, oper, type, value) + * oper can't be operation or comparison + * ByName: used by client API, field can be ID or name + * RowValue: a row of value, same element with total subtable fields + * DTCResultSet: some RowValue, subtable result received from server/helper + * ResultPacket: encoded subtable result, prepare for sending + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_FIELD_H__ +#define __CH_FIELD_H__ + +/* + + */ +#include +#include +#include +#include + +#include "value.h" +#include "protocol.h" + +class DTCTableDefinition; +class DTCFieldValue; + +class DTCFieldSet +{ +private: + uint8_t *fieldId; + uint8_t fieldMask[32]; + +public: + DTCFieldSet(const DTCFieldSet &fs) + { + int n = fs.fieldId[-2]; + fieldId = (uint8_t *)malloc(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + memcpy(fieldId, fs.fieldId - 2, n + 2); + fieldId += 2; + } + DTCFieldSet(int n) + { + if (n > 255) + n = 255; + fieldId = (uint8_t *)malloc(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + fieldId += 2; + fieldId[-2] = n; + memset(fieldId - 1, 0, n + 1); + memset(fieldMask, 0, sizeof(fieldMask)); + } + DTCFieldSet(uint8_t *idtab, int n) + { + if (n > 255) + n = 255; + fieldId = (uint8_t *)malloc(n + 2); + if (fieldId == NULL) + throw std::bad_alloc(); + *fieldId++ = n; + *fieldId++ = n; + memcpy(fieldId, idtab, n); + build_field_mask(fieldMask); + } + ~DTCFieldSet() + { + if (fieldId) + free(fieldId - 2); + } + + int num_fields(void) const { return fieldId[-1]; } + int field_id(int n) const + { + return n >= 0 && n < num_fields() ? fieldId[n] : 0; + } + void add_field(int id) + { + const int n = fieldId[-1]; + if (n >= fieldId[-2]) + return; + fieldId[n] = id; + fieldId[-1]++; + FIELD_SET(id, fieldMask); + } + int field_present(int id) const + { + if (id >= 0 && id < 255) + return FIELD_ISSET(id, fieldMask); + return 0; + } + void build_field_mask(uint8_t *mask) const; +}; + +class TableReference +{ +public: + TableReference(void *t) : ref(t) {} + void *table_definition(void) const { return ref; } + void *ref; + int num_fields() const { return 255; } +}; + +class RowValue : public TableReference +{ +private: + DTCValue *value; + +public: + RowValue(DTCTableDefinition *t) : TableReference(t) + { + assert(table_definition() != NULL); + assert(num_fields() > 0); + value = (DTCValue *)calloc(num_fields() + 1, sizeof(DTCValue)); + if (value == NULL) + throw std::bad_alloc(); + }; + + RowValue(const RowValue &r) : TableReference(r.table_definition()) + { + value = (DTCValue *)calloc(num_fields() + 1, sizeof(DTCValue)); + if (value == NULL) + throw std::bad_alloc(); + memcpy(value, r.value, sizeof(DTCValue) * num_fields() + 1); + } + + ~RowValue() + { + if (value) + free(value); + }; + + /*Compare tow RowValue by FieldIDList*/ + int Compare(const RowValue &rv, uint8_t *fieldIDList, uint8_t num); + DTCValue &operator[](int n) { return value[n]; } + const DTCValue &operator[](int n) const { return value[n]; } + DTCValue *field_value(int id) + { + return id >= 0 && id <= num_fields() ? &value[id] : NULL; + } + const DTCValue *field_value(int id) const + { + return id >= 0 && id <= num_fields() ? &value[id] : NULL; + } + void copy_value(const DTCValue *v, int id, int n) + { + memcpy(&value[id], v, n * sizeof(DTCValue)); + } +}; + +class DTCFieldValue +{ +private: + struct + { + uint8_t id; + uint8_t oper; + uint8_t type; + DTCValue val; + } * fieldValue; + + int maxFields; + int numFields; + int typeMask; + +public: + DTCFieldValue(int total) + { + fieldValue = NULL; + maxFields = numFields = 0; + if (total <= 0) + return; + fieldValue = (typeof(fieldValue))malloc(total * sizeof(*fieldValue)); + if (fieldValue == NULL) + throw(-ENOMEM); + maxFields = total; + typeMask = 0; + } + DTCFieldValue(const DTCFieldValue &fv, int sparse = 0) + { + numFields = fv.numFields; + typeMask = fv.typeMask; + if (sparse < 0) + sparse = 0; + sparse += fv.numFields; + maxFields = sparse; + if (fv.fieldValue != NULL) + { + fieldValue = (typeof(fieldValue))malloc(sparse * sizeof(*fieldValue)); + if (fieldValue == NULL) + throw(-ENOMEM); + memcpy(fieldValue, fv.fieldValue, fv.numFields * sizeof(*fieldValue)); + } + else + { + fieldValue = NULL; + } + } + ~DTCFieldValue() + { + if (fieldValue) + free(fieldValue); + } + + int max_fields(void) const { return maxFields; } + int num_fields(void) const { return numFields; } + + int field_id(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].id : 0; + } + + int field_type(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].type : DField::None; + } + + int field_operation(int n) const + { + return n >= 0 && n < numFields ? fieldValue[n].oper : 0; + } + + DTCValue *field_value(int n) const + { + return n >= 0 && n < numFields ? &fieldValue[n].val : NULL; + } + + void add_value(uint8_t id, uint8_t op, uint8_t t, const DTCValue &val) + { + if (numFields == maxFields) + return; + fieldValue[numFields].id = id; + fieldValue[numFields].oper = op; + fieldValue[numFields].type = t; + fieldValue[numFields].val = val; + numFields++; + } + void update_type_mask_ro(int ro) { typeMask |= (ro ? 1 : 2); }; + void update_type_mask_sync(int sy) { typeMask |= (sy ? 4 : 8); }; + int has_type_ro(void) const { return typeMask & 1; } + int has_type_rw(void) const { return typeMask & 2; } + int has_type_sync(void) const { return typeMask & 4; } + int has_type_async(void) const { return typeMask & 8; } + + void build_field_mask(uint8_t *mask) const; + int Update(RowValue &); + int Compare(const RowValue &, int iCmpFirstNRows = 256); +}; + +#endif diff --git a/src/search_local/index_storage/common/file_backed_key_set.cc b/src/search_local/index_storage/common/file_backed_key_set.cc new file mode 100644 index 0000000..a3acbcb --- /dev/null +++ b/src/search_local/index_storage/common/file_backed_key_set.cc @@ -0,0 +1,322 @@ +/* + * ===================================================================================== + * + * Filename: file_backed_key_set.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __USE_FILE_OFFSET64 +#define __USE_FILE_OFFSET64 +#endif + +#ifndef __USE_LARGEFILE64 +#define __USE_LARGEFILE64 +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "new_hash.h" +#include "log.h" +#include "file_backed_key_set.h" + +FileBackedKeySet::hash_node_allocator::~hash_node_allocator() +{ + reset(); +} + +void FileBackedKeySet::hash_node_allocator::reset() +{ + for (std::vector::iterator iter = m_buffs.begin(); + iter != m_buffs.end(); ++iter) + { + ::free(*iter); + } + m_buffs.clear(); + m_freeNode = 0; +} + +FileBackedKeySet::hash_node *FileBackedKeySet::hash_node_allocator::alloc() +{ + if (!m_freeNode) + { + FileBackedKeySet::hash_node *nodes = (hash_node *)calloc(GROW_COUNT, sizeof(hash_node)); + + FileBackedKeySet::hash_node *n = nodes; + for (int i = 0; i < GROW_COUNT - 1; ++i) + { + n->next = n + 1; + ++n; + } + m_freeNode = nodes; + m_buffs.push_back((char *)nodes); + } + + hash_node *rtn = m_freeNode; + m_freeNode = (hash_node *)m_freeNode->next; + return rtn; +} + +void FileBackedKeySet::hash_node_allocator::free(hash_node *n) +{ + n->next = m_freeNode; + m_freeNode = n; +} + +FileBackedKeySet::FileBackedKeySet(const char *file, int keySize) : m_filePath(file), m_fd(-1), m_keySize(keySize), m_base((char *)MAP_FAILED), + m_buckets(0) +{ +} + +FileBackedKeySet::~FileBackedKeySet() +{ + if (m_fd >= 0) + { + if (m_base != MAP_FAILED) + munmap(m_base, get_meta_info()->size); + close(m_fd); + } + + if (m_buckets) + free(m_buckets); +} + +int FileBackedKeySet::Open() +{ + assert(m_fd < 0); + assert(m_base == MAP_FAILED); + + m_fd = open(m_filePath.c_str(), O_RDWR | O_CREAT | O_LARGEFILE, 0644); + if (m_fd < 0) + { + log_crit("open %s failed, %m", m_filePath.c_str()); + return -1; + } + + //discard all content if any + ftruncate(m_fd, 0); + ftruncate(m_fd, INIT_FILE_SIZE); + + m_base = (char *)mmap(NULL, INIT_FILE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd, 0); + if (m_base == MAP_FAILED) + { + log_crit("mmap failed, %m"); + close(m_fd); + m_fd = -1; + + return -1; + } + + meta_info *m = get_meta_info(); + m->size = INIT_FILE_SIZE; + m->writePos = sizeof(meta_info); + + m_buckets = (hash_node **)calloc(BUCKET_SIZE, sizeof(hash_node *)); + + return 0; +} + +int FileBackedKeySet::Load() +{ + assert(m_fd < 0); + assert(m_base == MAP_FAILED); + + m_fd = open(m_filePath.c_str(), O_RDWR | O_LARGEFILE); + if (m_fd < 0) + { + log_crit("open %s failed, %m", m_filePath.c_str()); + return -1; + } + + struct stat64 st; + int rtn = fstat64(m_fd, &st); + if (rtn < 0) + { + log_crit("fstat64 failed, %m"); + close(m_fd); + m_fd = -1; + return -1; + } + + m_base = (char *)mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd, 0); + if (m_base == (char *)MAP_FAILED) + { + log_crit("mmap failed, %m"); + close(m_fd); + m_fd = -1; + return -1; + } + + m_buckets = (hash_node **)calloc(BUCKET_SIZE, sizeof(hash_node *)); + + char *start = m_base + sizeof(meta_info); + char *end = m_base + get_meta_info()->writePos; + + //variable key size + if (m_keySize == 0) + { + while (start < end) + { + int keyLen = *(unsigned char *)start + 1; + insert_to_set(start + 1, keyLen + 1); + start += keyLen + 1 + 1; + } + } + else + { + while (start < end) + { + insert_to_set(start + 1, m_keySize); + start += m_keySize + 1; + } + } + + return 0; +} + +int FileBackedKeySet::Insert(const char *key) +{ + if (Contains(key, false)) + return 0; + + int keyLen = m_keySize == 0 ? (*(unsigned char *)key) + 1 : m_keySize; + char *k = insert_to_file(key, keyLen); + if (!k) //remap failed? + return -1; + insert_to_set(k, keyLen); + return 0; +} + +bool FileBackedKeySet::Contains(const char *key, bool checkStatus) +{ + int keyLen = m_keySize == 0 ? (*(unsigned char *)key) + 1 : m_keySize; + uint32_t hash = new_hash(key, keyLen) % BUCKET_SIZE; + hash_node *n = m_buckets[hash]; + while (n) + { + char *k = m_base + n->offset; + if (memcmp(key, k, keyLen) == 0) + { + if ((checkStatus == true && *(k - 1) == MIGRATE_SUCCESS) || + checkStatus == false) + return true; + break; + } + n = n->next; + } + return false; +} + +bool FileBackedKeySet::is_migrating(const char *key) +{ + int keyLen = m_keySize == 0 ? (*(unsigned char *)key) + 1 : m_keySize; + uint32_t hash = new_hash(key, keyLen) % BUCKET_SIZE; + hash_node *n = m_buckets[hash]; + while (n) + { + char *k = m_base + n->offset; + if (memcmp(key, k, keyLen) == 0) + { + if (*(k - 1) == MIGRATE_START) + return true; + break; + } + n = n->next; + } + // key not found + return false; +} + +int FileBackedKeySet::Migrated(const char *key) +{ + int keyLen = m_keySize == 0 ? (*(unsigned char *)key) + 1 : m_keySize; + uint32_t hash = new_hash(key, keyLen) % BUCKET_SIZE; + hash_node *n = m_buckets[hash]; + while (n) + { + char *k = m_base + n->offset; + if (memcmp(key, k, keyLen) == 0) + { + *(k - 1) = MIGRATE_SUCCESS; + return 0; + } + n = n->next; + } + // key not found + return -1; +} + +void FileBackedKeySet::insert_to_set(const char *key, int len) +{ + uint32_t hash = new_hash(key, len) % BUCKET_SIZE; + hash_node *n = m_allocator.alloc(); + n->next = m_buckets[hash]; + n->offset = key - m_base; + m_buckets[hash] = n; +} + +char *FileBackedKeySet::insert_to_file(const char *key, int len) +{ + meta_info *m = get_meta_info(); + if (m->writePos + len + 1 > m->size) + { + uintptr_t new_size = m->size + GROW_FILE_SIZE; + if (ftruncate(m_fd, new_size) < 0) + { + log_crit("grow file to %p failed, %m", (void *)new_size); + return NULL; + } + + char *base = (char *)mremap(m_base, m->size, new_size, MREMAP_MAYMOVE); + if (base == MAP_FAILED) + { + log_crit("mremap failed, %m"); + return NULL; + } + + m_base = base; + m = get_meta_info(); + m->size = new_size; + } + + char *writePos = m_base + m->writePos; + *writePos = MIGRATE_START; + ++writePos; + memcpy(writePos, key, len); + m->writePos += len + 1; + + return writePos; +} + +void FileBackedKeySet::Clear() +{ + if (m_fd >= 0) + { + free(m_buckets); + m_allocator.reset(); + munmap(m_base, get_meta_info()->size); + close(m_fd); + m_buckets = 0; + m_base = (char *)MAP_FAILED; + m_fd = -1; + + unlink(m_filePath.c_str()); + } +} diff --git a/src/search_local/index_storage/common/file_backed_key_set.h b/src/search_local/index_storage/common/file_backed_key_set.h new file mode 100644 index 0000000..ae0ec51 --- /dev/null +++ b/src/search_local/index_storage/common/file_backed_key_set.h @@ -0,0 +1,92 @@ +/* + * ===================================================================================== + * + * Filename: file_backed_key_set.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef FILE_BACKED_KEY_SET_H__ +#define FILE_BACKED_KEY_SET_H__ + +#include + +#include +#include +#include + +#define MIGRATE_START 1 +#define MIGRATE_SUCCESS 2 + +class FileBackedKeySet +{ +public: + FileBackedKeySet(const char *file, int keySize); + ~FileBackedKeySet(); + + bool Contains(const char *key, bool checkStatus = true); + bool is_migrating(const char *key); + int Open(); + int Load(); + int Insert(const char *key); + int Migrated(const char *key); + void Clear(); + +private: + struct hash_node + { + hash_node *next; + uintptr_t offset; + }; + + class hash_node_allocator + { + public: + hash_node_allocator() : m_freeNode(NULL) {} + ~hash_node_allocator(); + + hash_node *alloc(); + void free(hash_node *n); + void reset(); + + static const int GROW_COUNT = 4096 * 16; + + private: + hash_node *m_freeNode; + std::vector m_buffs; + }; + + hash_node_allocator m_allocator; + + struct meta_info + { + uintptr_t size; + uintptr_t writePos; + }; + + //4M + static const int INIT_FILE_SIZE = 4096 * 1024; + static const int GROW_FILE_SIZE = 1024 * 1024; + static const int BUCKET_SIZE = 1024 * 1024; + + meta_info *get_meta_info() { return (meta_info *)m_base; } + void insert_to_set(const char *key, int len); + char *insert_to_file(const char *key, int len); + + std::string m_filePath; + int m_fd; + int m_keySize; + char *m_base; + hash_node **m_buckets; +}; + +#endif diff --git a/src/search_local/index_storage/common/helper_client.cc b/src/search_local/index_storage/common/helper_client.cc new file mode 100644 index 0000000..6c6233d --- /dev/null +++ b/src/search_local/index_storage/common/helper_client.cc @@ -0,0 +1,712 @@ +/* + * ===================================================================================== + * + * Filename: helper_cient.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "helper_client.h" +#include "helper_group.h" +#include "unix_socket.h" +#include "table_def_manager.h" + +HelperClient::HelperClient(PollerUnit *o, HelperGroup *hg, int idx) : PollerObject(o) +{ + packet = NULL; + task = NULL; + + helperGroup = hg; + helperIdx = idx; + + supportBatchKey = 0; + connectErrorCnt = 0; + ready = 0; + Ready(); // 开始默认可用 +} + +HelperClient::~HelperClient() +{ + if ((0 != task)) + { + if (stage == HelperRecvVerifyState) + { + DELETE(task); + } + else if (stage != HelperRecvRepState) + { + queue_back_task(); + } + else + { + if (task->result_code() >= 0) + set_error(-EC_UPSTREAM_ERROR, __FUNCTION__, + "Server Shutdown"); + task->reply_notify(); + task = NULL; + } + } + + DELETE(packet); +} + +int HelperClient::Ready() +{ + if (ready == 0) + { + helperGroup->add_ready_helper(); + } + + ready = 1; + connectErrorCnt = 0; + + return 0; +} + +int HelperClient::connect_error() +{ + connectErrorCnt++; + if (connectErrorCnt > maxTryConnect && ready) + { + log_debug("helper-client[%d] try connect %lu times, switch invalid.", + helperIdx, + (unsigned long)connectErrorCnt); + helperGroup->dec_ready_helper(); + ready = 0; + } + + return 0; +} + +int HelperClient::attach_task(TaskRequest *p, Packet *s) +{ + log_debug("HelperClient::attach_task()"); + + task = p; + packet = s; + + int ret = packet->Send(netfd); + if (ret == SendResultDone) + { + DELETE(packet); + stopWatch.start(); + task->prepare_decode_reply(); + receiver.attach(netfd); + receiver.erase(); + + stage = HelperRecvRepState; + enable_input(); + } + else + { + stage = HelperSendReqState; + enable_output(); + } + + attach_timer(helperGroup->recvList); + return delay_apply_events(); +} + +void HelperClient::complete_task(void) +{ + DELETE(packet); + if (task != NULL) + { + task->reply_notify(); + task = NULL; + } +} + +void HelperClient::queue_back_task(void) +{ + DELETE(packet); + helperGroup->queue_back_task(task); + task = NULL; +} + +int HelperClient::Reset() +{ + if (stage == HelperSendVerifyState || stage == HelperRecvVerifyState) + { + DELETE(packet); + DELETE(task); + } + else + { + if (task != NULL && task->result_code() >= 0) + { + if (stage == HelperRecvRepState) + set_error(-EC_UPSTREAM_ERROR, + "HelperGroup::Reset", "helper recv error"); + else if (stage == HelperSendReqState) + set_error(-EC_SERVER_ERROR, + "HelperGroup::Reset", "helper send error"); + } + complete_task(); + } + + if (stage == HelperIdleState) + helperGroup->connection_reset(this); + + DisableInput(); + disable_output(); + PollerObject::detach_poller(); + if (netfd > 0) + close(netfd); + netfd = -1; + stage = HelperDisconnected; + attach_timer(helperGroup->retryList); + return 0; +} + +int HelperClient::connect_server(const char *path) +{ + if (path == NULL || path[0] == '\0') + return -1; + + if (is_unix_socket_path(path)) + { + if ((netfd = socket(AF_LOCAL, SOCK_STREAM, 0)) == -1) + { + log_error("%s", "socket error,%m"); + return -2; + } + + fcntl(netfd, F_SETFL, O_RDWR | O_NONBLOCK); + + struct sockaddr_un unaddr; + socklen_t addrlen; + addrlen = init_unix_socket_address(&unaddr, path); + return connect(netfd, (struct sockaddr *)&unaddr, addrlen); + } + else + { + const char *addr = NULL; + const char *port = NULL; + const char *begin = strchr(path, ':'); + if (begin) + { + char *p = (char *)alloca(begin - path + 1); + memcpy(p, path, begin - path); + p[begin - path] = '\0'; + addr = p; + } + else + { + log_error("address error,correct address is addr:port/protocol"); + return -5; + } + + const char *end = strchr(path, '/'); + if (begin && end) + { + char *p = (char *)alloca(end - begin); + memcpy(p, begin + 1, end - begin - 1); + p[end - begin - 1] = '\0'; + port = p; + } + else + { + log_error("protocol error,correct address is addr:port/protocol"); + return -6; + } + + struct sockaddr_in inaddr; + bzero(&inaddr, sizeof(struct sockaddr_in)); + inaddr.sin_family = AF_INET; + inaddr.sin_port = htons(atoi(port)); + + if (strcmp(addr, "*") != 0 && + inet_pton(AF_INET, addr, &inaddr.sin_addr) <= 0) + { + log_error("invalid address %s:%s", addr, port); + return -3; + } + + if (strcasestr(path, "tcp")) + netfd = socket(AF_INET, SOCK_STREAM, 0); + else + netfd = socket(AF_INET, SOCK_DGRAM, 0); + + if (netfd < 0) + { + log_error("%s", "socket error,%m"); + return -4; + } + + fcntl(netfd, F_SETFL, O_RDWR | O_NONBLOCK); + + return connect(netfd, (const struct sockaddr *)&inaddr, sizeof(inaddr)); + } + return 0; +} + +int HelperClient::Reconnect(void) +{ + // increase connect count + connect_error(); + + if (stage != HelperDisconnected) + Reset(); + + const char *sockpath = helperGroup->sock_path(); + if (connect_server(sockpath) == 0) + { + log_debug("Connected to helper[%d]: %s", helperIdx, sockpath); + + packet = new Packet; + packet->encode_detect(TableDefinitionManager::Instance()->get_cur_table_def()); + + if (attach_poller() != 0) + { + log_error("helper[%d] attach poller error", helperIdx); + return -1; + } + DisableInput(); + enable_output(); + stage = HelperSendVerifyState; + return send_verify(); + } + + if (errno != EINPROGRESS) + { + log_error("connect helper-%s error: %m", sockpath); + close(netfd); + netfd = -1; + attach_timer(helperGroup->retryList); + //check helpergroup task queue expire. + helperGroup->check_queue_expire(); + return 0; + } + + log_debug("Connectting to helper[%d]: %s", helperIdx, sockpath); + + DisableInput(); + enable_output(); + attach_timer(helperGroup->connList); + stage = HelperConnecting; + return attach_poller(); +} + +int HelperClient::send_verify() +{ + int ret = packet->Send(netfd); + if (ret == SendResultDone) + { + DELETE(packet); + + task = new TaskRequest(TableDefinitionManager::Instance()->get_cur_table_def()); + if (task == NULL) + { + log_error("%s: %m", "new task & packet error"); + return -1; + } + task->prepare_decode_reply(); + receiver.attach(netfd); + receiver.erase(); + + stage = HelperRecvVerifyState; + disable_output(); + enable_input(); + } + else + { + stage = HelperSendVerifyState; + enable_output(); + } + + attach_timer(helperGroup->recvList); + return delay_apply_events(); +} + +int HelperClient::recv_verify() +{ + static int logwarn; + int ret = task->Decode(receiver); + + supportBatchKey = 0; + switch (ret) + { + default: + case DecodeFatalError: + log_error("decode fatal error retcode[%d] msg[%m] from helper", ret); + goto ERROR_RETURN; + + case DecodeDataError: + log_error("decode data error from helper %d", task->result_code()); + goto ERROR_RETURN; + + case DecodeWaitData: + case DecodeIdle: + attach_timer(helperGroup->recvList); + return 0; + + case DecodeDone: + switch (task->result_code()) + { + case -EC_EXTRA_SECTION_DATA: + supportBatchKey = 1; + break; + case -EC_BAD_FIELD_NAME: // old version dtc + supportBatchKey = 0; + break; + default: + log_error("detect helper-%s error: %d, %s", + helperGroup->sock_path(), + task->result_code(), + task->resultInfo.error_message()); + goto ERROR_RETURN; + } + break; + } + + if (supportBatchKey) + { + log_debug("helper-%s support batch-key", helperGroup->sock_path()); + } + else + { + if (logwarn++ == 0) + log_warning("helper-%s unsupported batch-key", helperGroup->sock_path()); + else + log_debug("helper-%s unsupported batch-key", helperGroup->sock_path()); + } + + DELETE(task); + Ready(); + + enable_input(); + disable_output(); + stage = HelperIdleState; + helperGroup->request_completed(this); + disable_timer(); + return delay_apply_events(); + +ERROR_RETURN: + Reset(); + attach_timer(helperGroup->retryList); + //check helpergroup task queue expire. + helperGroup->check_queue_expire(); + return 0; +} + +//client peer +int HelperClient::recv_response() +{ + int ret = task->Decode(receiver); + + switch (ret) + { + default: + case DecodeFatalError: + log_notice("decode fatal error retcode[%d] msg[%m] from helper", ret); + task->set_error(-EC_UPSTREAM_ERROR, __FUNCTION__, "decode fatal error from helper"); + break; + + case DecodeDataError: + log_notice("decode data error from helper %d", task->result_code()); + task->set_error(-EC_UPSTREAM_ERROR, __FUNCTION__, "decode data error from helper"); + break; + + case DecodeWaitData: + case DecodeIdle: + attach_timer(helperGroup->recvList); + return 0; + + case DecodeDone: + break; + } + + stopWatch.stop(); + helperGroup->record_process_time(task->request_code(), stopWatch); + complete_task(); + helperGroup->request_completed(this); + + // ?? + enable_input(); + stage = HelperIdleState; + if (ret != DecodeDone) + return -1; + return 0; +} + +int HelperClient::send_request() +{ + int ret = packet->Send(netfd); + + log_debug("[HelperClient][task=%d]Send Request result=%d, fd=%d", task->Role(), ret, netfd); + + switch (ret) + { + case SendResultMoreData: + break; + + case SendResultDone: + DELETE(packet); + stopWatch.start(); + task->prepare_decode_reply(); + receiver.attach(netfd); + receiver.erase(); + + stage = HelperRecvRepState; + disable_output(); + enable_input(); + break; + + case SendResultError: + default: + log_notice("send result error, ret = %d msg = %m", ret); + task->set_error(-EC_SERVER_ERROR, "Data source send failed", NULL); + return -1; + } + + attach_timer(helperGroup->recvList); + return 0; +} + +void HelperClient::input_notify(void) +{ + disable_timer(); + + if (stage == HelperRecvVerifyState) + { + if (recv_verify() < 0) + Reconnect(); + return; + } + else if (stage == HelperRecvNotifyReloadConfigState) + { + if (recv_notify_helper_reload_config() < 0) + Reconnect(); + return; + } + else if (stage == HelperRecvRepState) + { + if (recv_response() < 0) + Reconnect(); + return; + } + else if (stage == HelperIdleState) + { + /* no data from peer allowed in idle state */ + Reset(); + return; + } + DisableInput(); +} + +void HelperClient::output_notify(void) +{ + disable_timer(); + if (stage == HelperSendVerifyState) + { + if (send_verify() < 0) + { + DELETE(packet); + Reconnect(); + } + return; + } + else if (stage == HelperSendNotifyReloadConfigState) + { + if (send_notify_helper_reload_config() < 0) + { + DELETE(packet); + Reconnect(); + } + return; + } + else if (stage == HelperSendReqState) + { + if (send_request() < 0) + { + queue_back_task(); + Reconnect(); + } + return; + } + else if (stage == HelperConnecting) + { + packet = new Packet; + packet->encode_detect(TableDefinitionManager::Instance()->get_cur_table_def()); + + DisableInput(); + enable_output(); + stage = HelperSendVerifyState; + send_verify(); + return; + } + disable_output(); +} + +void HelperClient::hangup_notify(void) +{ +#if 0 + if(stage!=HelperConnecting) + Reconnect(); + else +#endif + Reset(); +} + +void HelperClient::timer_notify(void) +{ + switch (stage) + { + case HelperRecvRepState: + stopWatch.stop(); + helperGroup->record_process_time(task->request_code(), stopWatch); + + log_error("helper index[%d] execute timeout.", helperIdx); + set_error(-EC_UPSTREAM_ERROR, "HelperGroup::Timeout", "helper execute timeout"); + Reconnect(); + break; + case HelperSendReqState: + + log_error("helper index[%d] send timeout.", helperIdx); + set_error(-EC_SERVER_ERROR, "HelperGroup::Timeout", "helper send timeout"); + Reconnect(); + break; + case HelperDisconnected: + Reconnect(); + break; + case HelperConnecting: + Reset(); + break; + case HelperSendVerifyState: + case HelperRecvVerifyState: + DELETE(packet); + DELETE(task); + Reconnect(); + break; + case HelperSendNotifyReloadConfigState: + case HelperRecvNotifyReloadConfigState: + DELETE(packet); + DELETE(task); + Reconnect(); + break; + default: + break; + } +} + +int HelperClient::client_notify_helper_reload_config() +{ + packet = new Packet; + packet->encode_reload_config(TableDefinitionManager::Instance()->get_cur_table_def()); + if (0 != attach_poller()) + { + log_error("notify reload config helper [%d] attach poller failed!", helperIdx); + DELETE(packet); + return -1; + } + DisableInput(); + enable_output(); + stage = HelperSendNotifyReloadConfigState; + return send_notify_helper_reload_config(); +} + +int HelperClient::send_notify_helper_reload_config() +{ + int ret = packet->Send(netfd); + if (SendResultDone == ret) + { + DELETE(packet); + task = new TaskRequest(TableDefinitionManager::Instance()->get_cur_table_def()); + if (NULL == task) + { + log_error("new task error, maybe not have enough memory!"); + return -1; + } + task->prepare_decode_reply(); + receiver.attach(netfd); + receiver.erase(); + + stage = HelperRecvNotifyReloadConfigState; + disable_output(); + enable_input(); + } + else + { + stage = HelperSendNotifyReloadConfigState; + enable_output(); + } + + attach_timer(helperGroup->recvList); + return delay_apply_events(); +} + +int HelperClient::recv_notify_helper_reload_config() +{ + int ret = task->Decode(receiver); + switch (ret) + { + default: + case DecodeFatalError: + { + log_error("decode fatal error retcode [%d] from helper", ret); + goto ERROR_RETURN; + } + case DecodeDataError: + { + log_error("decode data error retcode [%d] from helper", ret); + goto ERROR_RETURN; + } + case DecodeWaitData: + case DecodeIdle: + { + attach_timer(helperGroup->recvList); + return 0; + } + case DecodeDone: + { + switch (task->result_code()) + { + case 0: + break; + case -EC_RELOAD_CONFIG_FAILED: + { + log_error("reload config failed EC_RELOAD_CONFIG_FAILED resultcode [%d] from helper", task->result_code()); + goto ERROR_RETURN; + } + default: + { + log_error("reload config failed unknow resultcode [%d] from helper", task->result_code()); + goto ERROR_RETURN; + } + } + } + } + DELETE(task); + + enable_input(); + disable_output(); + stage = HelperIdleState; + helperGroup->request_completed(this); + disable_timer(); + return delay_apply_events(); + +ERROR_RETURN: + Reset(); + attach_timer(helperGroup->retryList); + //check helpergroup task queue expire. + helperGroup->check_queue_expire(); + return 0; +} diff --git a/src/search_local/index_storage/common/helper_client.h b/src/search_local/index_storage/common/helper_client.h new file mode 100644 index 0000000..2a07362 --- /dev/null +++ b/src/search_local/index_storage/common/helper_client.h @@ -0,0 +1,153 @@ +/* + * ===================================================================================== + * + * Filename: helper_cient.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __HELPER_CLIENT_H__ +#define __HELPER_CLIENT_H__ + +#include "poller.h" +#include "packet.h" +#include "timer_list.h" +#include "task_request.h" +#include "stop_watch.h" + +enum HelperState +{ + HelperDisconnected = 0, + HelperConnecting, + HelperIdleState, + HelperRecvRepState, //wait for recv response, client side + HelperSendReqState, //wait for send request, client side + HelperSendVerifyState, + HelperRecvVerifyState, + HelperSendNotifyReloadConfigState, + HelperRecvNotifyReloadConfigState, +}; + +class HelperGroup; + +/* + State Machine: HelperClient object is static, beyond reconnect + Disconnected + wait retryTimeout --> trying reconnect + Reconnect + If connected --> IdleState + If inprogress --> ConnectingState + ConnectingState + If hangup_notify --> Disconnected + If output_notify --> SendVerifyState + If timer_notify:connTimeout --> Disconnected + SendVerifyState + If hangup_notify --> Disconnected + If output_notify --> Trying Sending + If timer_notify:connTimeout --> Disconnected + RecvVerifyStat + If hangup_notify --> complete_task(error) -->Reconnect + If input_notify --> DecodeDone --> IdleState + IdleState + If hangup_notify --> Reconnect + If attach_task --> Trying Sending + Trying Sending + If Sent --> RecvRepState + If MoreData --> SendRepState + If SentError --> PushBackTask --> Reconnect + SendRepState + If hangup_notify --> PushBackTask -->Reconnect + If output_notify --> Trying Sending + RecvRepState + If hangup_notify --> complete_task(error) -->Reconnect + If input_notify --> Decode Reply + DecodeReply + If DecodeDone --> IdleState + If MoreData --> RecvRepState + If FatalError --> complete_task(error) --> Reconnect + If DataError --> complete_task(error) --> Reconnect + + */ +class HelperClient : public PollerObject, + private TimerObject +{ +public: + friend class HelperGroup; + + HelperClient(PollerUnit *, HelperGroup *hg, int id); + virtual ~HelperClient(); + + int attach_task(TaskRequest *, Packet *); + + int support_batch_key(void) const { return supportBatchKey; } + +private: + int Reset(); + int Reconnect(); + + int send_verify(); + int recv_verify(); + + int client_notify_helper_reload_config(); + int send_notify_helper_reload_config(); + int recv_notify_helper_reload_config(); + + int Ready(); + int connect_error(); + + void complete_task(void); + void queue_back_task(void); + void set_error(int err, const char *msg, const char *msg1) + { + task->set_error(err, msg, msg1); + } + void set_error_dup(int err, const char *msg, const char *msg1) + { + task->set_error_dup(err, msg, msg1); + } + +public: + const char *state_string(void) + { + return this == NULL ? "NULL" : ((const char *[]){"DISC", "CONN", "IDLE", "RECV", "SEND", "SND_VER", "RECV_VER", "BAD"})[stage]; + } + +private: + virtual void input_notify(void); + virtual void output_notify(void); + virtual void hangup_notify(void); + virtual void timer_notify(void); + +private: + int recv_response(); + int send_request(); + int connect_server(const char *path); + + SimpleReceiver receiver; + TaskRequest *task; + Packet *packet; + + TaskRequest *verify_task; + Packet *verify_packet; + + HelperGroup *helperGroup; + int helperIdx; + + HelperState stage; + + int supportBatchKey; + static const unsigned int maxTryConnect = 10; + uint64_t connectErrorCnt; + int ready; + stopwatch_usec_t stopWatch; +}; +#endif diff --git a/src/search_local/index_storage/common/helper_collect.cc b/src/search_local/index_storage/common/helper_collect.cc new file mode 100644 index 0000000..e40276b --- /dev/null +++ b/src/search_local/index_storage/common/helper_collect.cc @@ -0,0 +1,790 @@ +/* + * ===================================================================================== + * + * Filename: helper_collect.cc + * + * Description: database collection helper. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "list.h" +#include "dbconfig.h" +#include "helper_group.h" +#include "helper_collect.h" +#include "request_base.h" +#include "task_request.h" +#include "log.h" +#include "key_guard.h" +#include "stat_dtc.h" +#include "protocol.h" +#include "dtc_global.h" +#include "watchdog_listener.h" +#include "watchdog_helper.h" +#include "unix_socket.h" + +extern const char *HelperName[]; + +class GuardNotify : public ReplyDispatcher +{ +public: + GuardNotify(GroupCollect *o) : owner(o) {} + ~GuardNotify() {} + virtual void reply_notify(TaskRequest *); + +private: + GroupCollect *owner; +}; + +void GuardNotify::reply_notify(TaskRequest *task) +{ + if (task->result_code() >= 0) + owner->guard->add_key(task->barrier_key(), task->packed_key()); + task->reply_notify(); +} + +GroupCollect::GroupCollect() : TaskDispatcher(NULL), + dbConfig({NULL, NULL}), + hasDummyMachine(0), + groups({NULL, NULL}), + groupMap({NULL, NULL}), + guardReply(NULL), + tableNo(0), + guard(NULL) +{ + /*总队列的统计，暂时还有意义，暂时保留*/ + statQueueCurCount = statmgr.get_item_u32(CUR_QUEUE_COUNT); + statQueueMaxCount = statmgr.get_item_u32(MAX_QUEUE_COUNT); + + /*新增的四个组中最大的队列长度统计项，用来进行告警监控*/ + statReadQueueCurMaxCount = statmgr.get_item_u32(HELPER_READ_GROUR_CUR_QUEUE_MAX_SIZE); + statWriteQueueMaxCount = statmgr.get_item_u32(HELPER_WRITE_GROUR_CUR_QUEUE_MAX_SIZE); + statCommitQueueCurMaxCount = statmgr.get_item_u32(HELPER_COMMIT_GROUR_CUR_QUEUE_MAX_SIZE); + statSlaveReadQueueMaxCount = statmgr.get_item_u32(HELPER_SLAVE_READ_GROUR_CUR_QUEUE_MAX_SIZE); +} + +GroupCollect::~GroupCollect() +{ + if (groups[0]) + { + for (int i = 0; i < dbConfig[0]->machineCnt * GROUPS_PER_MACHINE; i++) + DELETE(groups[0][i]); + + FREE_CLEAR(groups[0]); + } + + FREE_CLEAR(groupMap[0]); + DELETE(guard); + DELETE(guardReply); +} + +HelperGroup *GroupCollect::select_group(TaskRequest *task) +{ + + const DTCValue *key = task->request_key(); + uint64_t uk; + /* key-hash disable */ + if (dbConfig[0]->keyHashConfig.keyHashEnable == 0 || key == NULL) + { + if (NULL == key) + uk = 0; + else if (key->s64 < 0) + uk = 0 - key->s64; + else + uk = key->s64; + } + else + { + switch (task->field_type(0)) + { + case DField::Signed: + case DField::Unsigned: + uk = dbConfig[0]->keyHashConfig.keyHashFunction((const char *)&(key->u64), + sizeof(key->u64), + dbConfig[0]->keyHashConfig.keyHashLeftBegin, + dbConfig[0]->keyHashConfig.keyHashRightBegin); + break; + case DField::String: + case DField::Binary: + uk = dbConfig[0]->keyHashConfig.keyHashFunction(key->bin.ptr, + key->bin.len, + dbConfig[0]->keyHashConfig.keyHashLeftBegin, + dbConfig[0]->keyHashConfig.keyHashRightBegin); + break; + default: + uk = 0; + } + } + + if (dbConfig[1]) + { + int idx = uk / dbConfig[1]->dbDiv % dbConfig[1]->dbMod; + int machineId = groupMap[1][idx]; + HelperGroup *ptr = groups[1][machineId * GROUPS_PER_MACHINE]; + if (ptr != NULL && task->request_code() != DRequest::Get) + return GROUP_READONLY; + } + + int idx = uk / dbConfig[0]->dbDiv % dbConfig[0]->dbMod; + + int machineId = groupMap[0][idx]; + if (machineId == GMAP_NONE) + return NULL; + if (machineId == GMAP_DUMMY) + return GROUP_DUMMY; + + HelperGroup **ptr = &groups[0][machineId * GROUPS_PER_MACHINE]; + + if (task->request_code() == DRequest::Get && ptr[GROUPS_PER_ROLE] && + false == guard->in_set(task->barrier_key(), task->packed_key())) + { + int role = 0; + switch (dbConfig[0]->mach[machineId].mode) + { + case BY_SLAVE: + role = 1; + break; + + case BY_DB: + role = (uk / dbConfig[0]->dbDiv) & 1; + + case BY_TABLE: + role = (uk / dbConfig[0]->tblDiv) & 1; + + case BY_KEY: + role = task->barrier_key() & 1; + } + + return ptr[role * GROUPS_PER_ROLE]; + } + + int g = task->request_type(); + + while (--g >= 0) + { + if (ptr[g] != NULL) + { + return ptr[g]; + } + } + return NULL; +} + +bool GroupCollect::is_commit_full(TaskRequest *task) +{ + if (task->request_code() != DRequest::Replace) + return false; + + HelperGroup *helperGroup = select_group(task); + if (helperGroup == NULL || helperGroup == GROUP_DUMMY || helperGroup == GROUP_READONLY) + return false; + + if (helperGroup->queue_full()) + { + log_warning("NO FREE COMMIT QUEUE SLOT"); + helperGroup->dump_state(); + } + return helperGroup->queue_full() ? true : false; +} + +int GroupCollect::Cleanup() +{ + newDb.clear(); + new2old.clear(); + return 0; +} + +int GroupCollect::Cleanup2() +{ + if (groups[1]) + { + for (int i = 0; i < dbConfig[1]->machineCnt; ++i) + { + std::vector::iterator it = find(newDb.begin(), newDb.end(), i); + if (it != newDb.end()) + { + for (int j = 0; j < GROUPS_PER_MACHINE; ++j) + { + DELETE(groups[1][j]); + } + } + } + FREE_CLEAR(groups[1]); + } + FREE_CLEAR(groupMap[1]); + if (dbConfig[1]) + { + dbConfig[1]->Destroy(); + dbConfig[1] = NULL; + } + return 0; +} + +int GroupCollect::build_helper_object(int idx) +{ + if (groups[idx] != NULL) + { + log_error("groups[%d] exists", idx); + return -1; + } + groups[idx] = (HelperGroup **)CALLOC(sizeof(HelperGroup *), dbConfig[idx]->machineCnt * GROUPS_PER_MACHINE); + if (!groups[idx]) + { + log_error("malloc failed, %m"); + return -1; + } + + /* build helper object */ + for (int i = 0; i < dbConfig[idx]->machineCnt; i++) + { + if (dbConfig[idx]->mach[i].helperType == DUMMY_HELPER) + continue; + if (idx == 1 && find(newDb.begin(), newDb.end(), i) == newDb.end()) + { + // if not new db mach, just continue, copy old mach when switch + continue; + } + for (int j = 0; j < GROUPS_PER_MACHINE; j++) + { + if (dbConfig[idx]->mach[i].gprocs[j] == 0) + continue; + + char name[24]; + snprintf(name, sizeof(name), "%d%c%d", i, MACHINEROLESTRING[j / GROUPS_PER_ROLE], j % GROUPS_PER_ROLE); + groups[idx][i * GROUPS_PER_MACHINE + j] = + new HelperGroup( + dbConfig[idx]->mach[i].role[j / GROUPS_PER_ROLE].path, + name, + dbConfig[idx]->mach[i].gprocs[j], + dbConfig[idx]->mach[i].gqueues[j], + DTC_SQL_USEC_ALL); + if (j >= GROUPS_PER_ROLE) + groups[idx][i * GROUPS_PER_MACHINE + j]->fallback = + groups[idx][i * GROUPS_PER_MACHINE]; + log_debug("start worker %s", name); + } + } + + return 0; +} + +int GroupCollect::build_master_group_mapping(int idx) +{ + if (groupMap[idx] != NULL) + { + log_error("groupMap[%d] exist", idx); + return -1; + } + groupMap[idx] = (short *)MALLOC(sizeof(short) * dbConfig[idx]->dbMax); + if (groupMap[idx] == NULL) + { + log_error("malloc error for groupMap[%d]", idx); + return -1; + } + for (int i = 0; i < dbConfig[idx]->dbMax; i++) + groupMap[idx][i] = GMAP_NONE; + + /* build master group mapping */ + for (int i = 0; i < dbConfig[idx]->machineCnt; i++) + { + int gm_id = i; + if (dbConfig[idx]->mach[i].helperType == DUMMY_HELPER) + { + gm_id = GMAP_DUMMY; + hasDummyMachine = 1; + } + else if (dbConfig[idx]->mach[i].procs == 0) + { + continue; + } + for (int j = 0; j < dbConfig[idx]->mach[i].dbCnt; j++) + { + const int db = dbConfig[idx]->mach[i].dbIdx[j]; + if (groupMap[idx][db] >= 0) + { + log_error("duplicate machine, db %d machine %d %d", + db, groupMap[idx][db] + 1, i + 1); + return -1; + } + groupMap[idx][db] = gm_id; + } + } + for (int i = 0; i < dbConfig[idx]->dbMax; ++i) + { + if (groupMap[idx][i] == GMAP_NONE) + { + log_error("db completeness check error, db %d not found", i); + return -1; + } + } + return 0; +} + +DbConfig *GroupCollect::get_db_config(TaskRequest *task) +{ + RowValue row(task->table_definition()); + DTCConfig *config = NULL; + DbConfig *newdb = NULL; + // parse db config + if (!task->request_operation()) + { + log_error("table.conf not found when migrate db"); + task->set_error(-EC_DATA_NEEDED, "group collect", "migrate db need table.conf"); + return NULL; + } + task->update_row(row); + log_debug("strlen: %ld, row[3].bin.ptr: %s", strlen(row[3].bin.ptr), row[3].bin.ptr); + char *buf = row[3].bin.ptr; + config = new DTCConfig(); + if (config->parse_buffered_config(buf, NULL, "DB_DEFINE", false) != 0) + { + log_error("table.conf illeagl when migrate db, parse error"); + task->set_error(-EC_ERR_MIGRATEDB_ILLEGAL, "group collect", "table.conf illegal, parse error"); + delete config; + return NULL; + } + if ((newdb = DbConfig::Load(config)) == NULL) + { + log_error("table.conf illeagl when migrate db, load error"); + task->set_error(-EC_ERR_MIGRATEDB_ILLEGAL, "group collect", "table.conf illegal, load error"); + return NULL; + } + return newdb; +} + +int GroupCollect::migrate_db(TaskRequest *task) +{ + int ret = 0; + DbConfig *newDbConfig = get_db_config(task); + if (newDbConfig == NULL) + return -2; + if (dbConfig[1]) + { + bool same = dbConfig[1]->Compare(newDbConfig, true); + newDbConfig->Destroy(); + if (!same) + { + log_error("new table.conf when migrating db"); + task->set_error(-EC_ERR_MIGRATEDB_MIGRATING, "group collect", "new table.conf when migrating db"); + return -2; + } + log_notice("duplicate table.conf when migrating db"); + task->set_error(-EC_ERR_MIGRATEDB_DUPLICATE, "group collect", "duplicate table.conf when migrating db"); + return 0; + } + // check are others fields same + if (!newDbConfig->Compare(dbConfig[0], false)) + { + newDbConfig->Destroy(); + log_error("new table.conf does not match old one"); + task->set_error(-EC_ERR_MIGRATEDB_DISTINCT, "group collect", "new table.conf does not match old one"); + return -2; + } + // set read only on new db + dbConfig[1] = newDbConfig; + // find new db + dbConfig[1]->find_new_mach(dbConfig[0], newDb, new2old); + log_debug("found %ld new db machine", newDb.size()); + if (newDb.size() == 0) + { + log_error("table.conf does not contain new db when migrate db"); + task->set_error(-EC_DATA_NEEDED, "group collect", "table.conf does not contain new db"); + return -1; + } + // check db completeness of new db config + if (build_master_group_mapping(1) != 0) + { + log_error("table.conf db mapping is not complete"); + task->set_error(-EC_DATA_NEEDED, "group collect", "table.conf db mapping is not complete"); + return -1; + } + + // save new table.conf as table%d.conf + char tableName[64]; + snprintf(tableName, 64, "../conf/table%d.conf", tableNo); + log_debug("table.conf: %s", tableName); + if (dbConfig[1]->cfgObj->Dump(tableName, true) != 0) + { + log_error("save table.conf as table2.conf error"); + task->set_error(-EC_SERVER_ERROR, "group collect", "save table.conf as table2.conf error"); + return -1; + } + + // start listener, connect db, check access, start worker + if ((ret = start_listener(task)) != 0) + return ret; + ++tableNo; + + // start worker and create class member variable + if (build_helper_object(1) != 0) + { + log_error("verify connect error: %m"); + task->set_error(-EC_ERR_MIGRATEDB_HELPER, "group collect", "start helper worker error"); + return -1; + } + + // disable commit as none async + disable_commit_group(1); + set_timer_handler(recvList, connList, retryList, 1); + + return 0; +} + +int GroupCollect::switch_db(TaskRequest *task) +{ + if (!dbConfig[1]) + { + log_notice("migrate db not start"); + task->set_error(-EC_ERR_MIGRATEDB_NOT_START, "group collect", "migrate db not start"); + return -2; + } + DbConfig *newDbConfig = get_db_config(task); + if (newDbConfig == NULL) + return -2; + // check is table same + bool same = newDbConfig->Compare(dbConfig[1], true); + newDbConfig->Destroy(); + if (!same) + { + log_error("switch db with different table.conf"); + task->set_error(-EC_ERR_MIGRATEDB_DISTINCT, "group collect", "switch db with different table.conf"); + return -2; + } + // start worker helper + Attach(NULL, 1); + // switch to new, unset read only + std::swap(dbConfig[0], dbConfig[1]); + std::swap(groups[0], groups[1]); + std::swap(groupMap[0], groupMap[1]); + // copy old client + for (int i = 0; i < dbConfig[0]->machineCnt; ++i) + { + if (dbConfig[0]->mach[i].helperType == DUMMY_HELPER) + continue; + if (find(newDb.begin(), newDb.end(), i) != newDb.end()) + continue; + memmove(groups[0] + i * GROUPS_PER_MACHINE, groups[1] + new2old[i] * GROUPS_PER_MACHINE, sizeof(HelperGroup *) * GROUPS_PER_MACHINE); + log_debug("copy old client ptr: %p", *(groups[0] + i * GROUPS_PER_MACHINE)); + } + // release old + FREE_CLEAR(groupMap[1]); + FREE_CLEAR(groups[1]); + dbConfig[1]->Destroy(); + dbConfig[1] = NULL; + // write conf file + dbConfig[0]->cfgObj->Dump("../conf/table.conf", false); + Cleanup(); + + return 0; +} + +int GroupCollect::notify_watch_dog(StartHelperPara *para) +{ + char buf[16]; + if (sizeof(*para) > 15) + return -1; + char *env = getenv(ENV_WATCHDOG_SOCKET_FD); + int fd = env == NULL ? -1 : atoi(env); + if (fd > 0) + { + memset(buf, 0, 16); + buf[0] = WATCHDOG_INPUT_HELPER; + log_debug("sizeof(*para): %d", sizeof(*para)); + memcpy(buf + 1, para, sizeof(*para)); + send(fd, buf, sizeof(buf), 0); + return 0; + } + else + { + return -2; + } +} + +int GroupCollect::start_listener(TaskRequest *task) +{ + int ret = 0; + log_debug("starting new db listener..."); + int nh = 0; + dbConfig[1]->set_helper_path(getppid()); + for (std::vector::iterator it = newDb.begin(); it != newDb.end(); ++it) + { + // start listener + HELPERTYPE t = dbConfig[1]->mach[*it].helperType; + log_debug("helper type = %d", t); + if (DTC_HELPER >= t) + continue; + for (int r = 0; r < ROLES_PER_MACHINE; ++r) + { + int i, n = 0; + for (i = 0; i < GROUPS_PER_ROLE && (r * GROUPS_PER_ROLE + i) < GROUPS_PER_MACHINE; ++i) + n += dbConfig[1]->mach[*it].gprocs[r * GROUPS_PER_ROLE + i]; + if (n <= 0) + continue; + StartHelperPara para; + para.type = t; + para.backlog = n + 1; + para.mach = *it; + para.role = r; + para.conf = DBHELPER_TABLE_NEW; + para.num = tableNo; + if ((ret = notify_watch_dog(¶)) < 0) + { + log_error("notify watchdog error for group %d role %d, ret: %d", *it, r, ret); + return -1; + } + ++nh; + } + } + log_info("%d helper listener started", nh); + return 0; +} + +void GroupCollect::task_notify(TaskRequest *task) +{ + if (DRequest::ReloadConfig == task->request_code() && TaskTypeHelperReloadConfig == task->request_type()) + { + collect_notify_helper_reload_config(task); + return; + } + + int ret = 0; + if (task->request_code() == DRequest::SvrAdmin) + { + switch (task->requestInfo.admin_code()) + { + case DRequest::ServerAdminCmd::MigrateDB: + log_debug("GroupCollect::task_notify DRequest::SvrAdmin::MigrateDB"); + ret = migrate_db(task); + if (ret == -1) + { + Cleanup2(); + Cleanup(); + } + task->reply_notify(); + return; + + case DRequest::ServerAdminCmd::MigrateDBSwitch: + log_debug("GroupCollect::task_notify DRequest::SvrAdmin::MigrateDBSwitch"); + ret = switch_db(task); + task->reply_notify(); + return; + default: + // this should not happen + log_error("unknown admin code: %d", task->requestInfo.admin_code()); + task->set_error(-EC_SERVER_ERROR, "helper collect", "unkown admin code"); + task->reply_notify(); + return; + } + } + + HelperGroup *helperGroup = select_group(task); + + if (helperGroup == NULL) + { + log_error("Key not belong to this server"); + task->set_error(-EC_OUT_OF_KEY_RANGE, "GroupCollect::task_notify", "Key not belong to this server"); + task->reply_notify(); + } + else if (helperGroup == GROUP_DUMMY) + { + task->mark_as_black_hole(); + task->reply_notify(); + } + else if (helperGroup == GROUP_READONLY) + { + log_debug("try to do non read op on a key which belongs db which is migrating"); + task->set_error(-EC_SERVER_ERROR, "helper collect", "try to do non read op on a key which belongs db which is migrating"); + task->reply_notify(); + } + else + { + if (task->request_type() == TaskTypeWrite && guardReply != NULL) + task->push_reply_dispatcher(guardReply); + helperGroup->task_notify(task); + } + + stat_helper_group_queue_count(groups[0], dbConfig[0]->machineCnt * GROUPS_PER_MACHINE); + stat_helper_group_cur_max_queue_count(task->request_type()); +} + +int GroupCollect::load_config(DbConfig *cfg, int keysize, int idx) +{ + dbConfig[0] = cfg; + int ret = 0; + + if ((ret = build_master_group_mapping(idx)) != 0) + { + log_error("build master group map error, ret: %d", ret); + return ret; + } + if ((ret = build_helper_object(idx)) != 0) + { + log_error("build helper object error, ret: %d", ret); + return ret; + } + + if (dbConfig[0]->slaveGuard) + { + guard = new KeyGuard(keysize, dbConfig[0]->slaveGuard); + guardReply = new GuardNotify(this); + } + return 0; +} + +int GroupCollect::renew_config(struct DbConfig *cfg) +{ + dbConfig[1] = cfg; + std::swap(dbConfig[0], dbConfig[1]); + dbConfig[1]->Destroy(); + dbConfig[1] = NULL; + return 0; +} + +int GroupCollect::Attach(PollThread *thread, int idx) +{ + if (idx == 0) + TaskDispatcher::attach_thread(thread); + for (int i = 0; i < dbConfig[idx]->machineCnt * GROUPS_PER_MACHINE; i++) + { + if (groups[idx][i]) + groups[idx][i]->Attach(owner, &taskQueueAllocator); + } + return 0; +} + +void GroupCollect::set_timer_handler(TimerList *recv, TimerList *conn, TimerList *retry, int idx) +{ + if (idx == 0) + { + recvList = recv; + connList = conn; + retryList = retry; + } + for (int i = 0; i < dbConfig[idx]->machineCnt; i++) + { + if (dbConfig[idx]->mach[i].helperType == DUMMY_HELPER) + continue; + for (int j = 0; j < GROUPS_PER_MACHINE; j++) + { + if (dbConfig[idx]->mach[i].gprocs[j] == 0) + continue; + if (groups[idx][i * GROUPS_PER_MACHINE + j]) + groups[idx][i * GROUPS_PER_MACHINE + j]->set_timer_handler(recv, conn, retry); + } + } +} + +int GroupCollect::disable_commit_group(int idx) +{ + if (groups[idx] == NULL) + return 0; + for (int i = 2; + i < dbConfig[idx]->machineCnt * GROUPS_PER_MACHINE; + i += GROUPS_PER_MACHINE) + { + DELETE(groups[idx][i]); + } + return 0; +} + +void GroupCollect::stat_helper_group_queue_count(HelperGroup **groups, unsigned group_count) +{ + unsigned total_queue_count = 0; + unsigned total_queue_max_count = 0; + + for (unsigned i = 0; i < group_count; ++i) + { + if (groups[i]) + { + total_queue_count += groups[i]->queue_count(); + total_queue_max_count += groups[i]->queue_max_count(); + } + } + + statQueueCurCount = total_queue_count; + statQueueMaxCount = total_queue_max_count; + return; +} + +int GroupCollect::get_queue_cur_max_count(int iColumn) +{ + int maxcount = 0; + if ((iColumn < 0) || (iColumn >= GROUPS_PER_MACHINE)) + { + return maxcount; + } + + for (int row = 0; row < dbConfig[0]->machineCnt; row++) + { + /*read组是在group矩阵的第一列*/ + HelperGroup *readGroup = groups[0][GROUPS_PER_MACHINE * row + iColumn]; + if (NULL == readGroup) + { + continue; + } + if (readGroup->queue_count() > maxcount) + { + + maxcount = readGroup->queue_count(); + log_debug("the group queue maxcount is %d ", maxcount); + } + } + return maxcount; +} +/*传入请求类型，每次只根据请求类型统计响应的值*/ +void GroupCollect::stat_helper_group_cur_max_queue_count(int iRequestType) +{ + /*根据请求类型分辨不出来是主读还是备读(和Workload配置有关)，只好同时即统计主读组又统计备读组了*/ + /*除非遍历group矩阵里的指针值和selectgroup后的group指针比较，然后再对比矩阵列，这个更复杂*/ + if (TaskTypeRead == iRequestType) + { + statReadQueueCurMaxCount = get_queue_cur_max_count(MASTER_READ_GROUP_COLUMN); + statSlaveReadQueueMaxCount = get_queue_cur_max_count(SLAVE_READ_GROUP_COLUMN); + } + if (TaskTypeWrite == iRequestType) + { + statWriteQueueMaxCount = get_queue_cur_max_count(MASTER_WRITE_GROUP_COLUMN); + } + + if (TaskTypeCommit == iRequestType) + { + statCommitQueueCurMaxCount = get_queue_cur_max_count(MASTER_COMMIT_GROUP_COLUMN); + } +} + +void GroupCollect::collect_notify_helper_reload_config(TaskRequest *task) +{ + unsigned int uiGroupNum = 0; + unsigned int uiNullGroupNum = 0; + + for (int machineid = 0; machineid < dbConfig[0]->machineCnt; ++machineid) + { + HelperGroup **ptr = &groups[0][machineid * GROUPS_PER_MACHINE]; + for (int groupid = 0; groupid < GROUPS_PER_MACHINE; ++groupid) + { + ++uiGroupNum; + HelperGroup *pHelperGroup = ptr[groupid]; + if (NULL == pHelperGroup || GROUP_DUMMY == pHelperGroup || GROUP_READONLY == pHelperGroup) + { + ++uiNullGroupNum; + continue; + } + pHelperGroup->task_notify(task); + } + } + if (uiGroupNum == uiNullGroupNum) + { + log_error("not have available helpergroup, please check!"); + task->set_error(-EC_NOT_HAVE_AVAILABLE_HELPERGROUP, "helper collect", "not have available helpergroup"); + } + log_error("groupcollect notify work helper reload config finished!"); + task->reply_notify(); +} diff --git a/src/search_local/index_storage/common/helper_collect.h b/src/search_local/index_storage/common/helper_collect.h new file mode 100644 index 0000000..31e89eb --- /dev/null +++ b/src/search_local/index_storage/common/helper_collect.h @@ -0,0 +1,106 @@ +/* + * ===================================================================================== + * + * Filename: helper_collect.h + * + * Description: database collection helper. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __HELPER_COLLECT_H_ +#define __HELPER_COLLECT_H__ + +#include "dbconfig.h" +#include "poll_thread.h" +#include "request_base.h" +#include "watchdog_listener.h" +#include + +class HelperGroup; +class TimerList; +class KeyGuard; + +enum +{ + MASTER_READ_GROUP_COLUMN = 0, + MASTER_WRITE_GROUP_COLUMN = 1, + MASTER_COMMIT_GROUP_COLUMN = 2, + SLAVE_READ_GROUP_COLUMN = 3, +}; +class GroupCollect : public TaskDispatcher +{ +public: + GroupCollect(); + ~GroupCollect(); + + int load_config(struct DbConfig *cfg, int ks, int idx = 0); + int renew_config(struct DbConfig *cfg); + void collect_notify_helper_reload_config(TaskRequest *task); + int build_master_group_mapping(int idx = 0); + int build_helper_object(int idx = 0); + int notify_watch_dog(StartHelperPara *para); + int Cleanup(); + int Cleanup2(); + int start_listener(TaskRequest *task); + bool is_commit_full(TaskRequest *task); + bool Dispatch(TaskRequest *t); + int Attach(PollThread *thread, int idx = 0); + void set_timer_handler(TimerList *recv, TimerList *conn, TimerList *retry, int idx = 0); + int disable_commit_group(int idx = 0); + DbConfig *get_db_config(TaskRequest *task); + int migrate_db(TaskRequest *t); + int switch_db(TaskRequest *t); + + int has_dummy_machine(void) const { return hasDummyMachine; } + +private: + virtual void task_notify(TaskRequest *); + HelperGroup *select_group(TaskRequest *t); + + void stat_helper_group_queue_count(HelperGroup **group, unsigned group_count); + void stat_helper_group_cur_max_queue_count(int iRequestType); + int get_queue_cur_max_count(int iColumn); + +private: + struct DbConfig *dbConfig[2]; + + int hasDummyMachine; + + HelperGroup **groups[2]; +#define GMAP_NONE -1 +#define GMAP_DUMMY -2 +#define GROUP_DUMMY ((HelperGroup *)-2) +#define GROUP_READONLY ((HelperGroup *)-3) + short *groupMap[2]; + ReplyDispatcher *guardReply; + LinkQueue::allocator taskQueueAllocator; + + TimerList *recvList; + TimerList *connList; + TimerList *retryList; + + std::vector newDb; + std::map new2old; + int tableNo; + +public: + KeyGuard *guard; + +private: + StatItemU32 statQueueCurCount; /*所有组当前总的队列大小*/ + StatItemU32 statQueueMaxCount; /*所有组配置总的队列大小*/ + StatItemU32 statReadQueueCurMaxCount; /*所有机器所有主读组当前最大的队列大小*/ + StatItemU32 statWriteQueueMaxCount; /*所有机器所有写组当前最大的队列大小*/ + StatItemU32 statCommitQueueCurMaxCount; /*所有机器所有提交组当前最大的队列大小*/ + StatItemU32 statSlaveReadQueueMaxCount; /*所有机器所有备读组当前最大的队列大小*/ +}; + +#endif diff --git a/src/search_local/index_storage/common/helper_group.cc b/src/search_local/index_storage/common/helper_group.cc new file mode 100644 index 0000000..214eff7 --- /dev/null +++ b/src/search_local/index_storage/common/helper_group.cc @@ -0,0 +1,422 @@ +/* + * ===================================================================================== + * + * Filename: helper_group.cc + * + * Description: database collection group helper. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include + +#include "list.h" +#include "dbconfig.h" +#include "helper_client.h" +#include "helper_group.h" +#include "log.h" +#include "unix_socket.h" + +static StatItemU32 statHelperExpireCount; + +static void IncHelperExpireCount() +{ + statHelperExpireCount = statmgr.get_item_u32(DATA_SOURCE_EXPIRE_REQ); + statHelperExpireCount++; +} + +class HelperClientList : public ListObject +{ +public: + HelperClientList() : helper(NULL) {} + ~HelperClientList() { DELETE(helper); } + HelperClient *helper; +}; + +HelperGroup::HelperGroup(const char *s, const char *name_, int hc, int qs, + int statIndex) : TaskDispatcher(NULL), + queueSize(qs), + helperCount(0), + helperMax(hc), + readyHelperCnt(0), + fallback(NULL), + average_delay(0) /*默认时延为0*/ +{ + sockpath = strdup(s); + freeHelper.InitList(); + helperList = new HelperClientList[hc]; + + recvList = connList = retryList = NULL; + + strncpy(name, name_, sizeof(name) - 1); + name[sizeof(name) - 1] = '0'; + + statTime[0] = statmgr.get_sample(statIndex); + statTime[1] = statmgr.get_sample(statIndex + 1); + statTime[2] = statmgr.get_sample(statIndex + 2); + statTime[3] = statmgr.get_sample(statIndex + 3); + statTime[4] = statmgr.get_sample(statIndex + 4); + statTime[5] = statmgr.get_sample(statIndex + 5); +} + +HelperGroup::~HelperGroup() +{ + DELETE_ARRAY(helperList); + free(sockpath); +} + +void HelperGroup::record_response_delay(unsigned int t) +{ + if (t <= 0) + t = 1; + + if (average_delay == 0) + average_delay = t; + + double N = 20E6 / (average_delay + t); + + if ((unsigned)N > 200000) + N = 200000; /* 2w/s */ + if ((unsigned)N < 5) + N = 5; /* 0.5/s */ + + average_delay = ((N - 1) / N) * average_delay + t / N; +} + +void HelperGroup::add_ready_helper() +{ + if (readyHelperCnt == 0) + { + log_info("helper_group-%s switch to ONLINE mode", sockpath); + /* force flush task */ + attach_ready_timer(owner); + } + + if (readyHelperCnt < helperMax) + readyHelperCnt++; + + if (readyHelperCnt == helperMax) + log_debug("%s", "all client ready"); +} + +void HelperGroup::dec_ready_helper() +{ + if (readyHelperCnt > 0) + readyHelperCnt--; + if (readyHelperCnt == 0) + { + log_error("helper_group-%s all clients invalid, switch to OFFLINE mode", sockpath); + /* reply all task */ + attach_ready_timer(owner); + } +} + +void HelperGroup::record_process_time(int cmd, unsigned int usec) +{ + static const unsigned char cmd2type[] = + { + /*Nop*/ 0, + /*result_code*/ 0, + /*DTCResultSet*/ 0, + /*HelperAdmin*/ 0, + /*Get*/ 1, + /*Purge*/ 0, + /*Insert*/ 2, + /*Update*/ 3, + /*Delete*/ 4, + /*Replace*/ 5, + /*Flush*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Replicate*/ 1, + /*LocalMigrate*/ 1, + }; + statTime[0].push(usec); + int t = cmd2type[cmd]; + if (t) + statTime[t].push(usec); + + /* 计算新的平均时延 */ + record_response_delay(usec); +} + +int HelperGroup::Attach(PollThread *thread, LinkQueue::allocator *a) +{ + owner = thread; + for (int i = 0; i < helperMax; i++) + { + helperList[i].helper = new HelperClient(owner, this, i); + helperList[i].helper->Reconnect(); + } + + queue.SetAllocator(a); + return 0; +} + +int HelperGroup::connect_helper(int fd) +{ + struct sockaddr_un unaddr; + socklen_t addrlen; + + addrlen = init_unix_socket_address(&unaddr, sockpath); + return connect(fd, (struct sockaddr *)&unaddr, addrlen); +} + +void HelperGroup::queue_back_task(TaskRequest *task) +{ + if (queue.Unshift(task) < 0) + { + task->set_error(-EC_SERVER_ERROR, __FUNCTION__, "insufficient memory"); + task->reply_notify(); + } +} + +void HelperGroup::request_completed(HelperClient *h) +{ + HelperClientList *h0 = &helperList[h->helperIdx]; + if (h0->ListEmpty()) + { + h0->ListAdd(freeHelper); + helperCount--; + attach_ready_timer(owner); + } +} + +void HelperGroup::connection_reset(HelperClient *h) +{ + HelperClientList *h0 = &helperList[h->helperIdx]; + if (!h0->ListEmpty()) + { + h0->list_del(); + helperCount++; + } +} + +void HelperGroup::check_queue_expire(void) +{ + attach_ready_timer(owner); +} + +void HelperGroup::process_task(TaskRequest *task) +{ + if (DRequest::ReloadConfig == task->request_code() && TaskTypeHelperReloadConfig == task->request_type()) + { + process_reload_config(task); + return; + } + HelperClientList *h0 = freeHelper.NextOwner(); + HelperClient *helper = h0->helper; + + log_debug("process task....."); + if (helper->support_batch_key()) + task->mark_field_set_with_key(); + + Packet *packet = new Packet; + if (packet->encode_forward_request(task) != 0) + { + delete packet; + log_error("[2][task=%d]request error: %m", task->Role()); + task->set_error(-EC_BAD_SOCKET, "ForwardRequest", NULL); + task->reply_notify(); + } + else + { + h0->ResetList(); + helperCount++; + + helper->attach_task(task, packet); + } +} + +void HelperGroup::flush_task(uint64_t now) +{ + //check timeout for helper client + while (1) + { + TaskRequest *task = queue.Front(); + if (task == NULL) + break; + + if (readyHelperCnt == 0) + { + log_debug("no available helper, up stream server maybe offline"); + queue.Pop(); + task->set_error(-EC_UPSTREAM_ERROR, __FUNCTION__, + "no available helper, up stream server maybe offline"); + task->reply_notify(); + continue; + } + else if (task->is_expired(now)) + { + log_debug("%s", "task is expired in HelperGroup queue"); + IncHelperExpireCount(); + queue.Pop(); + task->set_error(-ETIMEDOUT, __FUNCTION__, "task is expired in HelperGroup queue"); + task->reply_notify(); + } + else if (!freeHelper.ListEmpty()) + { + queue.Pop(); + process_task(task); + } + else if (fallback && fallback->has_free_helper()) + { + queue.Pop(); + fallback->process_task(task); + } + else + { + break; + } + } +} + +void HelperGroup::timer_notify(void) +{ + uint64_t v = GET_TIMESTAMP() / 1000; + attach_timer(retryList); + flush_task(v); +} + +int HelperGroup::accept_new_request_fail(TaskRequest *task) +{ + unsigned work_client = helperMax; + unsigned queue_size = queue.Count(); + + /* queue至少排队work_client个任务 */ + if (queue_size <= work_client) + return 0; + + uint64_t wait_time = queue_size * (uint64_t)average_delay; + wait_time /= work_client; /* us */ + wait_time /= 1000; /* ms */ + + uint64_t now = GET_TIMESTAMP() / 1000; + + if (task->is_expired(now + wait_time)) + return 1; + + return 0; +} + +void HelperGroup::task_notify(TaskRequest *task) +{ + log_debug("HelperGroup::task_notify()"); + + if (DRequest::ReloadConfig == task->request_code() && TaskTypeHelperReloadConfig == task->request_type()) + { + group_notify_helper_reload_config(task); + return; + } + + uint64_t now = GET_TIMESTAMP() / 1000; /* ms*/ + flush_task(now); + + if (readyHelperCnt == 0) + { + log_debug("no available helper, up stream server maybe offline"); + task->set_error(-EC_UPSTREAM_ERROR, __FUNCTION__, "no available helper, up stream server maybe offline"); + task->reply_notify(); + } + else if (task->is_expired(now)) + { + log_debug("task is expired when sched to HelperGroup"); + IncHelperExpireCount(); + //modify error message + task->set_error(-ETIMEDOUT, __FUNCTION__, "task is expired when sched to HelperGroup,by DTC"); + task->reply_notify(); + } + else if (!freeHelper.ListEmpty()) + { + /* has free helper, sched task */ + process_task(task); + } + else + { + if (fallback && fallback->has_free_helper()) + { + fallback->process_task(task); + } + else if (accept_new_request_fail(task)) + { + /* helper 响应变慢，主动踢掉task */ + log_debug("HelperGroup response is slow, give up current task"); + IncHelperExpireCount(); + task->set_error(-EC_SERVER_BUSY, __FUNCTION__, "DB response is very slow, give up current task"); + task->reply_notify(); + } + else if (!queue_full()) + { + queue.Push(task); + } + else + { + /* no free helper */ + task->set_error(-EC_SERVER_BUSY, __FUNCTION__, "No available helper connections"); + log_error("No available helper queue slot,count=%d, max=%d", + queue.Count(), queueSize); + + task->reply_notify(); + } + } +} + +void HelperGroup::dump_state(void) +{ + log_info("HelperGroup %s count %d/%d", Name(), helperCount, helperMax); + int i; + for (i = 0; i < helperMax; i++) + { + log_info("helper %d state %s\n", i, helperList[i].helper->state_string()); + } +} + +void HelperGroup::group_notify_helper_reload_config(TaskRequest *task) +{ + //进入到这一步，helper应该是全部处于空闲状态的 + if (!freeHelper.ListEmpty()) + process_task(task); + else if (fallback && fallback->has_free_helper()) + fallback->process_task(task); + else + { + log_error("not have available helper client, please check!"); + task->set_error(-EC_NOT_HAVE_AVAILABLE_HELPERCLIENT, __FUNCTION__, "not have available helper client"); + } +} + +void HelperGroup::process_reload_config(TaskRequest *task) +{ + typedef std::vector HELPERCLIENTLISTVEC; + typedef std::vector::iterator HELPERCLIENTVIT; + HELPERCLIENTLISTVEC clientListVec; + HelperClientList *head = freeHelper.ListOwner(); + for (HelperClientList *pos = head->NextOwner(); pos != head; pos = pos->NextOwner()) + { + clientListVec.push_back(pos); + } + for (HELPERCLIENTVIT vit = clientListVec.begin(); vit != clientListVec.end(); ++vit) + { + HelperClientList *pHList = (*vit); + HelperClient *pHelper = pHList->helper; + pHList->ResetList(); + helperCount++; + pHelper->client_notify_helper_reload_config(); + } + + log_error("helpergroup [%s] notify work helper reload config finished!", Name()); +} diff --git a/src/search_local/index_storage/common/helper_group.h b/src/search_local/index_storage/common/helper_group.h new file mode 100644 index 0000000..c6f41ae --- /dev/null +++ b/src/search_local/index_storage/common/helper_group.h @@ -0,0 +1,115 @@ +/* + * ===================================================================================== + * + * Filename: helper_group.h + * + * Description: database collection group helper. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __HELPER_GROUP_H__ +#define __HELPER_GROUP_H__ + +#include "lqueue.h" +#include "poll_thread.h" + +#include "list.h" +#include "value.h" +#include "request_base.h" +#include "stat_dtc.h" + +class HelperClient; +class HelperClientList; +class TaskRequest; + +class HelperGroup : private TimerObject, public TaskDispatcher +{ +public: + HelperGroup(const char *sockpath, const char *name, int hc, int qs, + int statIndex); + ~HelperGroup(); + + int queue_full(void) const { return queue.Count() >= queueSize; } + int queue_empty(void) const { return queue.Count() == 0; } + int has_free_helper(void) const { return queue.Count() == 0 && freeHelper.ListEmpty(); } + + /* process or queue a task */ + virtual void task_notify(TaskRequest *); + + int Attach(PollThread *, LinkQueue::allocator *); + int connect_helper(int); + const char *sock_path(void) const { return sockpath; } + + void set_timer_handler(TimerList *recv, TimerList *conn, TimerList *retry) + { + recvList = recv; + connList = conn; + retryList = retry; + attach_timer(retryList); + } + + void queue_back_task(TaskRequest *); + void request_completed(HelperClient *); + void connection_reset(HelperClient *); + void check_queue_expire(void); + void dump_state(void); + + void add_ready_helper(); + void dec_ready_helper(); + +private: + virtual void timer_notify(void); + /* trying pop task and process */ + void flush_task(uint64_t time); + /* process a task, must has free helper */ + void process_task(TaskRequest *t); + const char *Name() const { return name; } + void record_response_delay(unsigned int t); + int accept_new_request_fail(TaskRequest *); + void group_notify_helper_reload_config(TaskRequest *task); + void process_reload_config(TaskRequest *task); + +public: + TimerList *recvList; + TimerList *connList; + TimerList *retryList; + +private: + LinkQueue queue; + int queueSize; + + int helperCount; + int helperMax; + int readyHelperCnt; + char *sockpath; + char name[24]; + + HelperClientList *helperList; + ListObject freeHelper; + +public: + HelperGroup *fallback; + +public: + void record_process_time(int type, unsigned int msec); + /* queue当前长度 */ + int queue_count(void) const { return queue.Count(); } + /* queue最大长度*/ + int queue_max_count(void) const { return queueSize; } + +private: + /* 平均请求时延 */ + double average_delay; + + StatSample statTime[6]; +}; + +#endif diff --git a/src/search_local/index_storage/common/hlist.h b/src/search_local/index_storage/common/hlist.h new file mode 100644 index 0000000..ea28d5e --- /dev/null +++ b/src/search_local/index_storage/common/hlist.h @@ -0,0 +1,201 @@ +/* + * ===================================================================================== + * + * Filename: hlist.h + * + * Description: list operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _HASH_LIST_H +#define _H4SH_LIST_H + +/* base on /usr/src/include/linux/list.h */ + +#include + +#define prefetch(data) (void)1 + +/* + * These are non-NULL pointers that will result in page faults + * under normal circumstances, used to verify that nobody uses + * non-initialized list entries. + */ +#define LIST_POISON1 ((struct hlist_node *)0x00100100) +#define LIST_POISON2 ((struct hlist_node **)0x00200200) + +/* + * Double linked lists with a single pointer list head. + * Mostly useful for hash tables where the two pointer list head is + * too wasteful. + * You lose the ability to access the tail in O(1). + */ + +struct hlist_head +{ + struct hlist_node *first; +}; + +struct hlist_node +{ + struct hlist_node *next, **pprev; +}; + +#define HLIST_HEAD_INIT \ + { \ + .first = NULL \ + } +#define HLIST_HEAD(name) struct hlist_head name = {.first = NULL} +#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) +static inline void INIT_HLIST_NODE(struct hlist_node *h) +{ + h->next = NULL; + h->pprev = NULL; +} + +static inline int hlist_unhashed(const struct hlist_node *h) +{ + return !h->pprev; +} + +static inline int hlist_empty(const struct hlist_head *h) +{ + return !h->first; +} + +static inline void __hlist_del(struct hlist_node *n) +{ + struct hlist_node *next = n->next; + struct hlist_node **pprev = n->pprev; + *pprev = next; + if (next) + next->pprev = pprev; +} + +static inline void hlist_del(struct hlist_node *n) +{ + __hlist_del(n); + n->next = LIST_POISON1; + n->pprev = LIST_POISON2; +} + +static inline void hlist_del_init(struct hlist_node *n) +{ + if (!hlist_unhashed(n)) + { + __hlist_del(n); + INIT_HLIST_NODE(n); + } +} + +static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) +{ + struct hlist_node *first = h->first; + n->next = first; + if (first) + first->pprev = &n->next; + h->first = n; + n->pprev = &h->first; +} + +/* next must be != NULL */ +static inline void hlist_add_before(struct hlist_node *n, + struct hlist_node *next) +{ + n->pprev = next->pprev; + n->next = next; + next->pprev = &n->next; + *(n->pprev) = n; +} + +static inline void hlist_add_after(struct hlist_node *n, + struct hlist_node *next) +{ + next->next = n->next; + n->next = next; + next->pprev = &n->next; + + if (next->next) + next->next->pprev = &next->next; +} + +/** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) ); }) + +#define hlist_entry(ptr, type, member) container_of(ptr, type, member) + +#define hlist_for_each(pos, head) \ + for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \ + pos = pos->next) + +#define hlist_for_each_safe(pos, n, head) \ + for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \ + pos = n) + +/** + * hlist_for_each_entry - iterate over list of given type + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry(tpos, pos, head, member) \ + for (pos = (head)->first; \ + pos && ({ prefetch(pos->next); 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ + pos = pos->next) + +/** + * hlist_for_each_entry_continue - iterate over a hlist continuing after current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_continue(tpos, pos, member) \ + for (pos = (pos)->next; \ + pos && ({ prefetch(pos->next); 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ + pos = pos->next) + +/** + * hlist_for_each_entry_from - iterate over a hlist continuing from current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_from(tpos, pos, member) \ + for (; pos && ({ prefetch(pos->next); 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ + pos = pos->next) + +/** + * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @n: another &struct hlist_node to use as temporary storage + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ + for (pos = (head)->first; \ + pos && ({ n = pos->next; 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ + pos = n) + +#endif diff --git a/src/search_local/index_storage/common/hotback_task.cc b/src/search_local/index_storage/common/hotback_task.cc new file mode 100644 index 0000000..dd81869 --- /dev/null +++ b/src/search_local/index_storage/common/hotback_task.cc @@ -0,0 +1,62 @@ +/* + * ===================================================================================== + * + * Filename: hotback_task.cc + * + * Description: hot back task. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "hotback_task.h" +#include "mem_check.h" +#include +#include +HotBackTask::HotBackTask() : m_Type(0), m_Flag(0), m_pPackedKey(NULL), m_PackedKeyLen(0), m_pValue(NULL), m_ValueLen(0) +{ +} + +HotBackTask::~HotBackTask() +{ + FREE_IF(m_pPackedKey); + FREE_IF(m_pValue); +} + +void HotBackTask::set_packed_key(char *pPackedKey, int keyLen) +{ + if ((NULL == pPackedKey) || (0 == keyLen)) + { + return; + } + + m_pPackedKey = (char *)MALLOC(keyLen); + if (NULL == m_pPackedKey) + { + return; + } + m_PackedKeyLen = keyLen; + memcpy(m_pPackedKey, pPackedKey, m_PackedKeyLen); +} + +void HotBackTask::set_value(char *pValue, int valueLen) +{ + if ((NULL == pValue) || (0 == valueLen)) + { + return; + } + + m_pValue = (char *)MALLOC(valueLen); + if (NULL == m_pPackedKey) + { + return; + } + m_ValueLen = valueLen; + memcpy(m_pValue, pValue, m_ValueLen); +} diff --git a/src/search_local/index_storage/common/hotback_task.h b/src/search_local/index_storage/common/hotback_task.h new file mode 100644 index 0000000..ef32fe4 --- /dev/null +++ b/src/search_local/index_storage/common/hotback_task.h @@ -0,0 +1,77 @@ +/* + * ===================================================================================== + * + * Filename: hotback_task.h + * + * Description: hot back task. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __HOTBACK_TASK_H__ +#define __HOTBACK_TASK_H__ + +struct HotBackTask +{ +private: + /*do nothing, only forbiden use */ + HotBackTask(const HotBackTask &other); + HotBackTask &operator=(const HotBackTask &other); + +public: + HotBackTask(); + ~HotBackTask(); + + void set_packed_key(char *pPackedKey, int keyLen); + void set_value(char *pValue, int valueLen); + void set_type(int type) + { + m_Type = type; + } + void set_flag(int flag) + { + m_Flag = flag; + } + char *get_packed_key() + { + return m_pPackedKey; + } + char *get_value() + { + return m_pValue; + } + int get_type() + { + return m_Type; + } + int get_flag() + { + return m_Flag; + } + int get_packed_key_len() + { + return m_PackedKeyLen; + } + int get_value_len() + { + return m_ValueLen; + } + +private: + int m_Type; + int m_Flag; + char *m_pPackedKey; + int m_PackedKeyLen; + char *m_pValue; + int m_ValueLen; + + +}; +#endif \ No newline at end of file diff --git a/src/search_local/index_storage/common/interface.h b/src/search_local/index_storage/common/interface.h new file mode 100644 index 0000000..3d5b4fc --- /dev/null +++ b/src/search_local/index_storage/common/interface.h @@ -0,0 +1,26 @@ +/* + * ===================================================================================== + * + * Filename: interface.h + * + * Description: interface define + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __DTC_INTERFACE_DEFINE_H__ +#define __DTC_INTERFACE_DEFINE_H__ + +#ifndef __INTERFACE__ +#define __INTERFACE__ class +#endif + +#endif diff --git a/src/search_local/index_storage/common/journal_id.h b/src/search_local/index_storage/common/journal_id.h new file mode 100644 index 0000000..a33d8bc --- /dev/null +++ b/src/search_local/index_storage/common/journal_id.h @@ -0,0 +1,99 @@ +/* + * ===================================================================================== + * + * Filename: journal_id.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_JOURNAL_ID +#define __DTC_JOURNAL_ID + +#include + +struct JournalID +{ + uint32_t serial; + uint32_t offset; + + JournalID() + { + serial = 0; + offset = 0; + } + + JournalID(const JournalID &v) + { + serial = v.serial; + offset = v.offset; + } + + JournalID(uint32_t s, uint32_t o) + { + serial = s; + offset = o; + } + + JournalID &operator=(const JournalID &v) + { + serial = v.serial; + offset = v.offset; + return *this; + } + + uint32_t Serial() const + { + return serial; + } + + uint32_t Offset() const + { + return offset; + } + + /* + * 对外接口全部打包为uint64_t, 方便操作。 + */ + JournalID &operator=(const uint64_t v) + { + serial = v >> 32; + offset = v & 0xFFFFFFFFULL; + return *this; + } + + JournalID(uint64_t v) + { + serial = v >> 32; + offset = v & 0xFFFFFFFFULL; + } + + operator uint64_t() const + { + uint64_t v = serial; + v <<= 32; + v += offset; + return v; + } + + int Zero() const + { + return serial == 0 && offset == 0; + } + + int GE(const JournalID &v) + { + return serial > v.serial || (serial == v.serial && offset >= v.offset); + } + +} __attribute__((packed)); + +#endif diff --git a/src/search_local/index_storage/common/key_dist.h b/src/search_local/index_storage/common/key_dist.h new file mode 100644 index 0000000..82e78fe --- /dev/null +++ b/src/search_local/index_storage/common/key_dist.h @@ -0,0 +1,28 @@ +/* + * ===================================================================================== + * + * Filename: key_dist.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "value.h" + +class KeyDist +{ +public: + KeyDist() {} + virtual ~KeyDist() = 0; + virtual unsigned int HashKey(const DTCValue *) = 0; + virtual int key2_db_id(const DTCValue *) = 0; + virtual int key2_table(const DTCValue *, char *, int) = 0; +}; diff --git a/src/search_local/index_storage/common/key_guard.cc b/src/search_local/index_storage/common/key_guard.cc new file mode 100644 index 0000000..58b0788 --- /dev/null +++ b/src/search_local/index_storage/common/key_guard.cc @@ -0,0 +1,125 @@ +/* + * ===================================================================================== + * + * Filename: key_guard.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "mem_check.h" + +#include "key_guard.h" + +static int keycmp_1(const char *a, const char *b, int ks) +{ + return *a - *b; +} + +static int keycmp_2(const char *a, const char *b, int ks) +{ + return *(uint16_t *)a - *(uint16_t *)b; +} + +static int keycmp_4(const char *a, const char *b, int ks) +{ + return *(uint32_t *)a - *(uint32_t *)b; +} + +static int keycmp_n(const char *a, const char *b, int ks) +{ + return memcmp(a, b, ks); +} + +static int keycmp_b(const char *a, const char *b, int ks) +{ + return *a != *b ? 1 : memcmp(a + 1, b + 1, *a); +} + +KeyGuard::KeyGuard(int ks, int exp) +{ + keySize = ks; + timeout = exp; + switch (keySize) + { + case 0: + keycmp = keycmp_b; + case 1: + keycmp = keycmp_1; + case 2: + keycmp = keycmp_2; + case 4: + keycmp = keycmp_4; + default: + keycmp = keycmp_n; + } + for (int i = 0; i < HASHBASE; i++) + { + INIT_LIST_HEAD(hashList + i); + } + INIT_LIST_HEAD(&lru); +}; + +KeyGuard::~KeyGuard() +{ +} + +void KeyGuard::try_expire(void) +{ + unsigned int deadline = time(NULL) - timeout; + while (!list_empty(&lru)) + { + struct keyslot *k = list_entry(lru.next, struct keyslot, tlist); + if (k->timestamp >= deadline) + break; + list_del(&k->hlist); + list_del(&k->tlist); + FREE(k); + } +} + +void KeyGuard::add_key(unsigned long barHash, const char *ptrKey) +{ + struct list_head *h = hashList + ((unsigned long)barHash) % HASHBASE; + struct keyslot *k; + list_for_each_entry(k, h, hlist) + { + if (keycmp(ptrKey, k->data, keySize) == 0) + { + k->timestamp = time(NULL); + _list_move_tail(&k->tlist, &lru); + return; + } + } + + int size = keySize; + if (size == 0) + size = 1 + *(unsigned char *)ptrKey; + k = (struct keyslot *)MALLOC(offsetof(struct keyslot, data) + size); + memcpy(k->data, ptrKey, size); + k->timestamp = time(NULL); + list_add(&k->hlist, h); + list_add(&k->tlist, &lru); +} + +bool KeyGuard::in_set(unsigned long barHash, const char *ptrKey) +{ + struct list_head *h = hashList + ((unsigned long)barHash) % HASHBASE; + struct keyslot *k; + list_for_each_entry(k, h, hlist) + { + if (keycmp(ptrKey, k->data, keySize) == 0) + return true; + } + + return false; +} diff --git a/src/search_local/index_storage/common/key_guard.h b/src/search_local/index_storage/common/key_guard.h new file mode 100644 index 0000000..86bfd09 --- /dev/null +++ b/src/search_local/index_storage/common/key_guard.h @@ -0,0 +1,51 @@ +/* + * ===================================================================================== + * + * Filename: key_guard.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "timer_list.h" +#include "list.h" + +class KeyGuard +{ +public: + KeyGuard(int keysize, int timeout); + ~KeyGuard(); + + void add_key(unsigned long barHash, const char *ptrKey); + bool in_set(unsigned long barHash, const char *ptrKey); + +private: + void try_expire(void); + +private: + enum + { + HASHBASE = 65536 + }; + int keySize; + int timeout; + struct list_head hashList[HASHBASE]; + struct list_head lru; + int (*keycmp)(const char *a, const char *b, int ks); + + struct keyslot + { + struct list_head hlist; + struct list_head tlist; + unsigned int timestamp; + char data[0]; + } __attribute__((__aligned__(1))); +}; diff --git a/src/search_local/index_storage/common/key_route.cc b/src/search_local/index_storage/common/key_route.cc new file mode 100644 index 0000000..86101d9 --- /dev/null +++ b/src/search_local/index_storage/common/key_route.cc @@ -0,0 +1,764 @@ +/* + * ===================================================================================== + * + * Filename: key_route.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "key_route.h" +#include "file_backed_key_set.h" +#include "admin_tdef.h" +#include "field.h" + +KeyRoute::KeyRoute(PollThread *owner, int keyFormat) : TaskDispatcher(owner), m_cacheOutput(owner), m_remoteOutput(owner), m_keyFormat(keyFormat), m_iCSState(CS_NOT_STARTED), m_strCSAddr("") +{ +} + +void KeyRoute::task_notify(TaskRequest *t) +{ + if (t->request_code() == DRequest::SvrAdmin) + { + log_debug("admin_code:%d", t->requestInfo.admin_code()); + switch (t->requestInfo.admin_code()) + { + case DRequest::ServerAdminCmd::ReloadClusterNodeList: + process_reload(t); + return; + case DRequest::ServerAdminCmd::SetClusterNodeState: + process_node_state_change(t); + return; + case DRequest::ServerAdminCmd::change_node_address: + process_change_node_address(t); + return; + case DRequest::ServerAdminCmd::GetClusterState: + process_get_cluster_state(t); + return; + case DRequest::ServerAdminCmd::Migrate: + process_migrate(t); + return; + case DRequest::ServerAdminCmd::Cascade: + process_cascade(t); + return; + default: + m_cacheOutput.task_notify(t); + return; + } + } + + if (t->request_code() == DRequest::Nop) + { + //just pass through + m_cacheOutput.task_notify(t); + return; + } + + if (t->packed_key() == NULL) + { + t->set_error(-EC_BAD_OPERATOR, "Key Route", "Batch Request Fast Path Not Supported"); + t->reply_notify(); + return; + } + + if (CS_CASCADING == m_iCSState) + { + log_debug("cascadeing state, addr [%s]", m_strCSAddr.c_str()); + if (m_strCSAddr.empty()) + { + t->set_error(-EC_SERVER_ERROR, "key route", "the remote cascade addr is NULL"); + t->reply_notify(); + return; + } + t->set_remote_addr(m_strCSAddr.c_str()); + m_remoteOutput.task_notify(t); + return; + } + + std::string selected = select_node(t->packed_key()); + log_debug("route to:%s self is:%s", selected.c_str(), m_selfName.c_str()); + const char *packedKey = t->packed_key(); + int keyLen = m_keyFormat > 0 ? m_keyFormat : (*(unsigned char *)packedKey) + 1; + + if (selected == m_selfName) + { + m_cacheOutput.task_notify(t); + return; + } + + if (accept_key(selected, packedKey, keyLen)) + { + t->set_remote_addr(name_to_addr(selected)); + t->mark_as_pass_thru(); + m_remoteOutput.task_notify(t); + return; + } + else if (is_migrating(selected, packedKey, keyLen)) + { + if (t->request_code() != DRequest::Get) + { + t->set_error(-EC_SERVER_ERROR, "Key Route", "try to do non read op on a key while it is migrating"); + t->reply_notify(); + return; + } + } + else + { + MigrationStateMap::iterator i = m_serverMigrationState.find(selected); + if (i->second.state == MS_MIGRATE_READONLY) + { + if (t->request_code() != DRequest::Get) + { + t->set_error(-EC_SERVER_READONLY, "Key Route", "migrate readonly"); + t->reply_notify(); + return; + } + } + } + m_cacheOutput.task_notify(t); +} + +//Migrate command need set DTC address in KeyRoute. +//processing method in cache_admin +void KeyRoute::process_migrate(TaskRequest *t) +{ + std::string selected = select_node(t->packed_key()); + + log_debug("migrate to %s", selected.c_str()); + if (selected == m_selfName) + { + const DTCFieldValue *ui = t->request_operation(); + if (ui && ui->field_value(0) && + (ui->field_value(0)->s64 & 0xFF) == DTCMigrate::FROM_SERVER) + { + m_cacheOutput.task_notify(t); + return; + } + + log_info("key belongs to self,no migrate operate"); + t->set_error(-EC_STATE_ERROR, "Key Route", "key belongs to self"); + t->reply_notify(); + return; + } + + MigrationStateMap::iterator iter = m_serverMigrationState.find(selected); + if (iter == m_serverMigrationState.end()) + { + t->set_error(-EC_STATE_ERROR, "Key Route", + "internal error: node not in map!"); + t->reply_notify(); + log_error("internal error: %s not in serverMigrationState", selected.c_str()); + return; + } + + if (iter->second.state != MS_MIGRATING && + iter->second.state != MS_MIGRATE_READONLY) + { + t->set_error(-EC_STATE_ERROR, "Key Route", "state not migratable"); + t->reply_notify(); + log_error("%s not migrating while key migrated!", selected.c_str()); + return; + } + t->set_remote_addr(name_to_addr(selected)); + m_cacheOutput.task_notify(t); +} + +std::string KeyRoute::select_node(const char *packedKey) +{ + int keyLen = m_keyFormat > 0 ? m_keyFormat : (*(unsigned char *)packedKey) + 1; + std::string selected; + //agent don't know how long the key is, so use u64 always + if (m_keyFormat == 4) + { + uint64_t v = *(uint32_t *)packedKey; + return m_selector.Select((const char *)&v, sizeof(uint64_t)); + } + + return m_selector.Select(packedKey, keyLen); +} + +int KeyRoute::key_migrated(const char *key) +{ + std::string selected = select_node(key); + MigrationStateMap::iterator iter = m_serverMigrationState.find(selected); + if (iter == m_serverMigrationState.end()) + { + log_crit("internal error: %s not in serverMigrationState", selected.c_str()); + return -1; + } + + if (iter->second.state != MS_MIGRATING && + iter->second.state != MS_MIGRATE_READONLY) + { + log_crit("%s not migrating while key migrated!", selected.c_str()); + return -1; + } + + return iter->second.migrated->Migrated(key); +} + +int KeyRoute::key_migrating(const char *key) +{ + std::string selected = select_node(key); + MigrationStateMap::iterator iter = m_serverMigrationState.find(selected); + if (iter == m_serverMigrationState.end()) + { + log_crit("internal error: %s not in serverMigrationState", selected.c_str()); + return -1; + } + + if (iter->second.state != MS_MIGRATING && + iter->second.state != MS_MIGRATE_READONLY) + { + log_crit("%s not migrating while key migrated!", selected.c_str()); + return -1; + } + + return iter->second.migrated->Insert(key); +} + +bool KeyRoute::accept_key(const std::string &node, const char *key, int len) +{ + MigrationStateMap::iterator iter = m_serverMigrationState.find(node); + if (iter == m_serverMigrationState.end()) + { + //this should never happen! + log_crit("can't find %s in state map!", node.c_str()); + //forward it either way + return true; + } + + if (iter->second.state == MS_NOT_STARTED) + return false; + + if (iter->second.state == MS_COMPLETED) + return true; + + return iter->second.migrated->Contains(key); +} + +bool KeyRoute::is_migrating(const std::string &node, const char *key, int len) +{ + MigrationStateMap::iterator iter = m_serverMigrationState.find(node); + if (iter == m_serverMigrationState.end()) + { + //this should never happen! + log_crit("can't find %s in state map!", node.c_str()); + //forward it either way + return false; + } + //migrate direciton stat is migrating or migrate readonly, should judge this key is in migrating + if ((iter->second.state == MS_MIGRATING) || (iter->second.state == MS_MIGRATE_READONLY)) + { + return iter->second.migrated->is_migrating(key); + } + return false; +} +bool KeyRoute::is_same_cluster_config(std::map &dtcClusterMap, const std::string &strDtcName) +{ + if (strDtcName != m_selfName) + { + log_error("cluster dtc name is not equal, now name[%s], previous name[%s]", strDtcName.c_str(), m_selfName.c_str()); + return false; + } + if (dtcClusterMap.size() != m_serverMigrationState.size()) + { + log_error("cluster dtc num is not equal, now size[%ld], previous size[%ld]", dtcClusterMap.size(), m_serverMigrationState.size()); + return false; + } + MigrationStateMap::const_iterator migrateIter = m_serverMigrationState.begin(); + for (; migrateIter != m_serverMigrationState.end(); migrateIter++) + { + std::map::const_iterator dtcmapIter = dtcClusterMap.find(migrateIter->first); + if (dtcmapIter != dtcClusterMap.end()) + { + if (migrateIter->second.addr != dtcmapIter->second) + { + log_error("cluster dtc addis not equal,dtc name[%s] , previous size[%s], now size[%s]", migrateIter->first.c_str(), migrateIter->second.addr.c_str(), dtcmapIter->second.c_str()); + return false; + } + } + else + { + log_error("previous dtc[%s] is not in now dtc cluster", migrateIter->first.c_str()); + return false; + } + } + return true; +} +void KeyRoute::process_reload(TaskRequest *t) +{ + //在迁移未完成前不允许切换表 + if (migration_inprogress()) + { + + RowValue row(t->table_definition()); + t->update_row(row); + log_debug("get new table, len %d", row[3].bin.len); + std::string strSelfName; + std::map dtcClusterMap; + if (!ClusterConfig::parse_cluster_config(strSelfName, dtcClusterMap, row[3].bin.ptr, row[3].bin.len)) + { + t->set_error(-EC_BAD_INVALID_FIELD, "Key Route", "invalid table"); + t->reply_notify(); + return; + } + if (!is_same_cluster_config(dtcClusterMap, strSelfName)) + { + t->set_error(-EC_STATE_ERROR, "Key Route", "Reload not possible while migration in-progress"); + t->reply_notify(); + } + else + { + t->prepare_result_no_limit(); + t->reply_notify(); + } + + return; + } + + if (!t->request_operation()) + { + t->set_error(-EC_DATA_NEEDED, "Key Route", "table needed"); + t->reply_notify(); + return; + } + + RowValue row(t->table_definition()); + t->update_row(row); + + //the table should be placed in the last field + log_debug("get new table, len %d", row[3].bin.len); + + std::string strSelfName; + std::vector nodes; + if (!ClusterConfig::parse_cluster_config(strSelfName, &nodes, row[3].bin.ptr, row[3].bin.len)) + { + t->set_error(-EC_BAD_INVALID_FIELD, "Key Route", "invalid table"); + t->reply_notify(); + return; + } + if (!ClusterConfig::save_cluster_config(&nodes, strSelfName)) + { + t->set_error(-EC_SERVER_ERROR, "Key Route", "wirte config to file error"); + t->reply_notify(); + return; + } + + m_serverMigrationState.clear(); + m_selector.Clear(); + + log_debug("got new table"); + for (std::vector::iterator i = nodes.begin(); + i != nodes.end(); ++i) + { + log_debug("\tnode %s : %s, self? %s", i->name.c_str(), + i->addr.c_str(), i->self ? "yes" : "no"); + ServerMigrationState s; + s.addr = i->addr; + s.state = i->self ? MS_COMPLETED : MS_NOT_STARTED; + s.migrated = NULL; + m_serverMigrationState[i->name] = s; + if (i->self) + m_selfName = i->name; + m_selector.add_node(i->name.c_str()); + } + + save_state_to_file(); + + t->prepare_result_no_limit(); + t->reply_notify(); +} + +static const char *migrateStateStr[] = { + "MIGRATION NOT STARTED", + "MIGRATING", + "MIGRATION READONLY", + "MIGRATION COMPLETED", +}; + +void KeyRoute::process_get_cluster_state(TaskRequest *t) +{ + t->prepare_result_no_limit(); + for (MigrationStateMap::iterator iter = m_serverMigrationState.begin(); + iter != m_serverMigrationState.end(); ++iter) + { + std::string result = iter->first; + if (m_selfName == iter->first) + result += " * "; + result += "\t"; + result += iter->second.addr; + result += "\t\t"; + + if (iter->second.state < MS_NOT_STARTED || iter->second.state > MS_COMPLETED) + result += "!!!INVALID STATE!!!"; + else + result += migrateStateStr[iter->second.state]; + + RowValue row(t->table_definition()); + row[3].Set(result.c_str(), result.size()); + t->append_row(row); + } + + t->reply_notify(); +} + +void KeyRoute::process_change_node_address(TaskRequest *t) +{ + if (!t->request_operation()) + { + t->set_error(-EC_DATA_NEEDED, "Key Route", "name/address needed"); + t->reply_notify(); + return; + } + + RowValue row(t->table_definition()); + t->update_row(row); + std::string name(row[2].bin.ptr, row[2].bin.len); + std::string addr(row[3].bin.ptr, row[3].bin.len); + + MigrationStateMap::iterator iter = m_serverMigrationState.find(name); + if (iter == m_serverMigrationState.end()) + { + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "invalid node name"); + t->reply_notify(); + return; + } + if (!ClusterConfig::change_node_address(name, addr)) + { + t->set_error(-EC_SERVER_ERROR, "Key Route", "change node address in config file error"); + t->reply_notify(); + return; + } + + iter->second.addr = addr; + t->prepare_result_no_limit(); + t->reply_notify(); +} + +void KeyRoute::process_node_state_change(TaskRequest *t) +{ + if (!t->request_operation()) + { + t->set_error(-EC_DATA_NEEDED, "Key Route", "node name needed."); + t->reply_notify(); + return; + } + + RowValue row(t->table_definition()); + t->update_row(row); + std::string name(row[2].bin.ptr, row[2].bin.len); + int newState = row[1].u64; + MigrationStateMap::iterator iter = m_serverMigrationState.find(name); + if (newState > MS_COMPLETED || iter == m_serverMigrationState.end()) + { + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "invalid name/state"); + t->reply_notify(); + return; + } + + if (iter->second.state == newState) + { + //nothing changed + t->prepare_result_no_limit(); + t->reply_notify(); + return; + } + + if (name == m_selfName) + { + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "can't change self's state"); + t->reply_notify(); + return; + } + + switch ((iter->second.state << 4) | newState) + { + case (MS_NOT_STARTED << 4) | MS_MIGRATE_READONLY: + log_warning("node[%s] state changed from NOT_STARTED to MIGRATE_READONLY", name.c_str()); + //fall through + case (MS_NOT_STARTED << 4) | MS_MIGRATING: + iter->second.migrated = new FileBackedKeySet(key_list_file_name(name).c_str(), t->table_definition()->key_format()); + if (iter->second.migrated->Open() < 0) + { + delete iter->second.migrated; + iter->second.migrated = NULL; + t->set_error(-EC_SERVER_ERROR, "Key Route", "open file backed key set failed"); + t->reply_notify(); + return; + } + iter->second.state = newState; + break; + + case (MS_NOT_STARTED << 4) | MS_COMPLETED: + //when we add a node to the cluster, only data belongs to the new node need to be migrated + //other nodes are untouched. + //this is the case. + log_info("node[%s] state changed from NOT_STARTED to COMLETED", name.c_str()); + iter->second.state = newState; + break; + + case (MS_MIGRATING << 4) | MS_NOT_STARTED: + case (MS_MIGRATE_READONLY << 4) | MS_NOT_STARTED: + //XXX: should we allow these changes? + log_warning("node[%s] state changed from %s to NOT_STARTED!", + name.c_str(), + iter->second.state == MS_MIGRATING ? "MIGRATING" : "MIGRATE_READONLY"); + iter->second.migrated->Clear(); + delete iter->second.migrated; + iter->second.state = newState; + break; + + case (MS_MIGRATE_READONLY << 4) | MS_MIGRATING: + log_warning("node[%s] state changed from MIGRATE_READONLY to MIGRATING", + name.c_str()); + //fall through + case (MS_MIGRATING << 4) | MS_MIGRATE_READONLY: + //normal change + iter->second.state = newState; + break; + + case (MS_MIGRATING << 4) | MS_COMPLETED: + log_warning("node[%s] state changed from MIGRATING to COMPLETED", + name.c_str()); + //fall through + case (MS_MIGRATE_READONLY << 4) | MS_COMPLETED: + iter->second.migrated->Clear(); + delete iter->second.migrated; + iter->second.state = newState; + break; + + case (MS_COMPLETED << 4) | MS_NOT_STARTED: + log_warning("node[%s] state changed from COMPLETED to NOT_STARTED", + name.c_str()); + iter->second.state = newState; + break; + + case (MS_COMPLETED << 4) | MS_MIGRATING: + case (MS_COMPLETED << 4) | MS_MIGRATE_READONLY: + //XXX: should we allow these changes? + log_warning("node[%s] state changed from COMPLATED to %s!", + name.c_str(), + newState == MS_MIGRATING ? "MIGRATING" : "MIGRATE_READONLY"); + iter->second.migrated = new FileBackedKeySet(key_list_file_name(name).c_str(), t->table_definition()->key_format()); + if (iter->second.migrated->Open() < 0) + { + delete iter->second.migrated; + iter->second.migrated = NULL; + t->set_error(-EC_SERVER_ERROR, "Key Route", "open file backed key set failed\n"); + t->reply_notify(); + return; + } + iter->second.state = newState; + break; + + default: + log_crit("invalid state/new state value : %d --> %d", + iter->second.state, newState); + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "invalid state"); + t->reply_notify(); + return; + } + + //state changed, save it to file + save_state_to_file(); + t->prepare_result_no_limit(); + t->reply_notify(); +} + +bool KeyRoute::migration_inprogress() +{ + for (MigrationStateMap::iterator iter = m_serverMigrationState.begin(); + iter != m_serverMigrationState.end(); ++iter) + { + if (iter->second.state == MS_MIGRATING || + iter->second.state == MS_MIGRATE_READONLY) + return true; + } + + return false; +} + +static const char *state_file_name = "../data/cluster.stat"; + +void KeyRoute::save_state_to_file() +{ + FILE *f = fopen(state_file_name, "wb"); + if (!f) + { + log_crit("open %s for write failed, %m", state_file_name); + return; + } + + int n = m_serverMigrationState.size(); + fwrite(&n, sizeof(n), 1, f); + for (MigrationStateMap::iterator iter = m_serverMigrationState.begin(); + iter != m_serverMigrationState.end(); ++iter) + { + int len = iter->first.size(); + fwrite(&len, sizeof(len), 1, f); + fwrite(iter->first.c_str(), 1, len, f); + fwrite(&iter->second.state, sizeof(iter->second.state), 1, f); + } + + fclose(f); +} + +void KeyRoute::Init(const std::vector &nodes) +{ + for (std::vector::const_iterator i = nodes.begin(); + i != nodes.end(); ++i) + { + log_info("\tnode %s : %s, self? %s", i->name.c_str(), + i->addr.c_str(), i->self ? "yes" : "no"); + ServerMigrationState s; + s.addr = i->addr; + s.state = i->self ? MS_COMPLETED : MS_NOT_STARTED; + s.migrated = NULL; + m_serverMigrationState[i->name] = s; + if (i->self) + m_selfName = i->name; + m_selector.add_node(i->name.c_str()); + } +} + +int KeyRoute::load_node_state_if_any() +{ + FILE *f = fopen(state_file_name, "rb"); + if (!f) //it is ok if the state file is not there + return 0; + + log_info("begin loading stat file ..."); + + int n = 0; + assert(fread(&n, sizeof(n), 1, f) == 1); + if (n != (int)m_serverMigrationState.size()) + { + log_crit("state file and config file mismatch!"); + fclose(f); + return -1; + } + + char buf[256]; //name can't exceeds 255 + for (int i = 0; i < n; ++i) + { + int len = 0; + assert(fread(&len, sizeof(len), 1, f) == 1); + if (len == 0 || len > (int)sizeof(buf) - 1) + { + log_crit("invalid state file!"); + fclose(f); + return -1; + } + assert((int)fread(buf, 1, len, f) == len); + buf[len] = 0; + int state = 0; + assert(fread(&state, sizeof(state), 1, f) == 1); + if (state < MS_NOT_STARTED || state > MS_COMPLETED) + { + log_crit("invalid state in state file!"); + fclose(f); + return -1; + } + + MigrationStateMap::iterator iter = m_serverMigrationState.find(buf); + if (iter == m_serverMigrationState.end()) + { + log_crit("name %s in state file doesn't exist in config!", buf); + fclose(f); + return -1; + } + + iter->second.state = state; + log_info("%s %s", iter->first.c_str(), migrateStateStr[state]); + if (state == MS_MIGRATING || state == MS_MIGRATE_READONLY) + { + iter->second.migrated = new FileBackedKeySet( + key_list_file_name(iter->first).c_str(), m_keyFormat); + if (iter->second.migrated->Load() < 0) + { + log_crit("load %s migrated key list failed", buf); + fclose(f); + return -1; + } + } + } + + //we should got EOF + assert(fread(buf, 1, 1, f) == 0); + fclose(f); + + log_info("load state file completed."); + return 0; +} + +void KeyRoute::process_cascade(TaskRequest *t) +{ + if (!t->request_operation()) + { + t->set_error(-EC_DATA_NEEDED, "Key Route", "cascade addr needed."); + t->reply_notify(); + return; + } + + RowValue row(t->table_definition()); + t->update_row(row); + std::string strAddr(row[2].bin.ptr, row[2].bin.len); + int newState = row[1].u64; + + if (newState < CS_NOT_STARTED || newState >= CS_MAX) + { + log_error("cascade mistach invalid state/new state value : %d --> %d", m_iCSState, newState); + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "mistach invalid state"); + t->reply_notify(); + return; + } + + if (m_iCSState == newState) + { + log_error("nothing to changed state/new state value : %d --> %d!", m_iCSState, newState); + t->prepare_result_no_limit(); + t->reply_notify(); + return; + } + + switch ((m_iCSState << 4) | newState) + { + case (CS_NOT_STARTED << 4) | CS_CASCADING: + { + log_error("switch cascade state CS_NOT_STARTED to CS_CASCADING, addr [%s]", strAddr.c_str()); + m_iCSState = newState; + m_strCSAddr = strAddr; + break; + } + case (CS_CASCADING << 4) | CS_NOT_STARTED: + { + log_error("switch cascade state CS_CASCADING to CS_NOT_STARTED, addr [%s]", strAddr.c_str()); + m_iCSState = newState; + //切换为未开始状态时，将对端地址置空 + m_strCSAddr = ""; + break; + } + default: + { + log_error("cascade invalid state/new state value : %d --> %d", m_iCSState, newState); + t->set_error(-EC_BAD_RAW_DATA, "Key Route", "invalid state"); + t->reply_notify(); + return; + } + } + t->prepare_result_no_limit(); + t->reply_notify(); + return; +} diff --git a/src/search_local/index_storage/common/key_route.h b/src/search_local/index_storage/common/key_route.h new file mode 100644 index 0000000..2a3901e --- /dev/null +++ b/src/search_local/index_storage/common/key_route.h @@ -0,0 +1,116 @@ +/* + * ===================================================================================== + * + * Filename: key_route.h + * + * Description: routing from key to agent/dtc. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef KEY_ROUTE_H__ +#define KEY_ROUTE_H__ + +#include +#include +#include + +#include "request_base.h" +#include "task_request.h" +#include "consistent_hash_selector.h" +#include "parse_cluster_config.h" + +class FileBackedKeySet; + +class KeyRoute : public TaskDispatcher +{ +public: + KeyRoute(PollThread *owner, int keyFormat); + + void bind_remote_helper(TaskDispatcher *r) + { + m_remoteOutput.bind_dispatcher(r); + } + void bind_cache(TaskDispatcher *c) + { + m_cacheOutput.bind_dispatcher(c); + } + int key_migrated(const char *key); + int key_migrating(const char *key); + + void Init(const std::vector &nodes); + void task_notify(TaskRequest *t); + int load_node_state_if_any(); + +private: + RequestOutput m_cacheOutput; + RequestOutput m_remoteOutput; + int m_keyFormat; + + void process_reload(TaskRequest *t); + void process_node_state_change(TaskRequest *t); + void process_change_node_address(TaskRequest *t); + void process_migrate(TaskRequest *t); + void process_get_cluster_state(TaskRequest *t); + bool accept_key(const std::string &node, const char *key, int len); + bool is_migrating(const std::string &node, const char *key, int len); + bool is_same_cluster_config(std::map &dtcClusterMap, const std::string &strDtcName); + const char *name_to_addr(const std::string &node) + { + return m_serverMigrationState[node].addr.c_str(); + } + + std::string key_list_file_name(const std::string &name) + { + return "../data/" + name + ".migrated"; + } + + std::string select_node(const char *key); + + bool migration_inprogress(); + void save_state_to_file(); + void process_cascade(TaskRequest *t); + + enum MigrationState + { + MS_NOT_STARTED, + MS_MIGRATING, + MS_MIGRATE_READONLY, + MS_COMPLETED, + }; + + struct ServerMigrationState + { + std::string addr; + int state; + FileBackedKeySet *migrated; + + ServerMigrationState() : state(MS_NOT_STARTED), migrated(NULL) {} + }; + + typedef std::map MigrationStateMap; + MigrationStateMap m_serverMigrationState; + std::string m_selfName; + ConsistentHashSelector m_selector; + + //级联状态枚举值，不可与上面的迁移状态枚举值重复 + enum CascadeState + { + CS_NOT_STARTED = 100, + CS_CASCADING, + CS_MAX, + }; + //本DTC级联状态 + int m_iCSState; + //级联对端DTC地址，例：10.191.147.188:12000/tcp + std::string m_strCSAddr; +}; + +#endif diff --git a/src/search_local/index_storage/common/keycodec.h b/src/search_local/index_storage/common/keycodec.h new file mode 100644 index 0000000..576e631 --- /dev/null +++ b/src/search_local/index_storage/common/keycodec.h @@ -0,0 +1,102 @@ +/* + * ===================================================================================== + * + * Filename: keycodec.h + * + * Description: key decode relevant operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __DTC_KEY_CODEC_H +#define __DTC_KEY_CODEC_H + +#include +#include +#include "namespace.h" + +DTC_BEGIN_NAMESPACE + +class KeyCodec +{ +public: + KeyCodec(const unsigned t) + { + set_key_type(t); + } + + ~KeyCodec() {} + + void set_key_type(const unsigned t) + { + type = t; + } + unsigned key_length(const char *ptr) + { + return type > 0 ? type : *(unsigned char *)ptr; + } + unsigned total_length(const char *ptr) + { + return type > 0 ? key_length(ptr) : key_length(ptr) + 1; + } + const char *key_pointer(const char *ptr) + { + return type > 0 ? ptr : ptr + 1; + } + int key_compare(const char *a, const char *b) + { + switch (type) + { + case 1: + return *(uint8_t *)a - *(uint8_t *)b; + case 2: + return *(uint16_t *)a - *(uint16_t *)b; + case 4: + return *(uint32_t *)a - *(uint32_t *)b; + } + + return memcmp(a, b, total_length(a)); + } + + unsigned key_hash(const char *ptr) + { + unsigned hash = 0x123; + unsigned len = total_length(ptr); + + do + { + unsigned char c = *ptr++; + //c = icase_hash(c); + hash = hash * 101 + c; + } while (--len); + + return hash; + } + +private: + /* + * This removes bit 5 if bit 6 is set. (from git name-hash.c) + * + * That will make US-ASCII characters hash to their upper-case + * equivalent. We could easily do this one whole word at a time, + * but that's for future worries. + */ + static inline unsigned char icase_hash(unsigned char c) + { + return c & ~((c & 0x40) >> 1); + } + +private: + unsigned type; +}; + +DTC_END_NAMESPACE +#endif diff --git a/src/search_local/index_storage/common/list.h b/src/search_local/index_storage/common/list.h new file mode 100644 index 0000000..cfb9c8b --- /dev/null +++ b/src/search_local/index_storage/common/list.h @@ -0,0 +1,323 @@ +/* + * ===================================================================================== + * + * Filename: list.h + * + * Description: Simple doubly linked list implementation. + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _H_LIST_H_ +#define _H_LIST_H_ +#include +#include + +__BEGIN_DECLS +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ +#define __builtin_prefetch(x, y, z) (void)1 +struct list_head +{ + struct list_head *next, *prev; +}; +typedef struct list_head list_head_t; + +#define LIST_HEAD_INIT(name) \ + { \ + &(name), &(name) \ + } + +#define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +#define INIT_LIST_HEAD(ptr) \ + do \ + { \ + (ptr)->next = (ptr); \ + (ptr)->prev = (ptr); \ + } while (0) + +/* + * Insert a p entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_add(struct list_head *p, + struct list_head *prev, + struct list_head *next) +{ + next->prev = p; + p->next = next; + p->prev = prev; + prev->next = p; +} + +/** + * list_add - add a p entry + * @p: p entry to be added + * @head: list head to add it after + * + * Insert a p entry after the specified head. + * This is good for implementing stacks. + */ +static inline void list_add(struct list_head *p, struct list_head *head) +{ + __list_add(p, head, head->next); +} +/** + * list_add_tail - add a p entry + * @p: p entry to be added + * @head: list head to add it before + * + * Insert a p entry before the specified head. + * This is useful for implementing queues. + */ +static inline void list_add_tail(struct list_head *p, struct list_head *head) +{ + __list_add(p, head->prev, head); +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_del(struct list_head *prev, struct list_head *next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty on entry does not return true after this, the entry is in an undefined state. + */ +static inline void list_del(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + entry->next = 0; + entry->prev = 0; +} + +/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +static inline void list_del_init(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + INIT_LIST_HEAD(entry); +} + +/** + * list_move - delete from one list and add as another's head + * @list: the entry to move + * @head: the head that will precede our entry + */ +static inline void list_move(struct list_head *list, struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add(list, head); +} +/** + * list_move_tail - delete from one list and add as another's tail + * @list: the entry to move + * @head: the head that will follow our entry + */ +static inline void _list_move_tail(struct list_head *list, + struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add_tail(list, head); +} + +/** + * list_empty - tests whether a list is empty + * @head: the list to test. + */ +static inline int list_empty(const struct list_head *head) +{ + return head->next == head; +} + +static inline void __list_splice(struct list_head *list, + struct list_head *head) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +/** + * list_splice - join two lists + * @list: the p list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(struct list_head *list, struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head); +} + +/** + * list_splice_init - join two lists and reinitialise the emptied list. + * @list: the p list to add. + * @head: the place to add it in the first list. + * + * The list at @list is reinitialised + */ +static inline void list_splice_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + { + __list_splice(list, head); + INIT_LIST_HEAD(list); + } +} + +#ifndef offsetof +#if __GNUC__ >= 4 +#define offsetof(type, member) __builtin_offsetof(type, member) +#else +#define offsetof(type, member) (unsigned long)(&((type *)0)->member) +#endif +#endif + +/** + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + */ +#define list_entry(ptr, type, member) \ + ((type *)((char *)(ptr)-offsetof(type, member))) + +/** + * list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each(pos, head) \ + for (pos = (head)->next, __builtin_prefetch(pos->next, 0, 1); \ + pos != (head); \ + pos = pos->next, __builtin_prefetch(pos->next, 0, 1)) + +#define __list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) +/** + * list_for_each_prev - iterate over a list backwards + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each_prev(pos, head) \ + for (pos = (head)->prev, __builtin_prefetch(pos->prev, 0, 1); \ + pos != (head); \ + pos = pos->prev, __builtin_prefetch(pos->prev, 0, 1)) + +#define __list_for_each_prev(pos, head) \ + for (pos = (head)->prev; pos != (head); \ + pos = pos->prev) + +/** + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop counter. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * list_for_each_entry - iterate over list of given type + * @pos: the type * to use as a loop counter. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + __builtin_prefetch(pos->member.next, 0, 1); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member), \ + __builtin_prefetch(pos->member.next, 0, 1)) + +__END_DECLS +#ifdef __cplusplus + +// internal, used by ListObject<> only +class ListHead +{ +public: + struct list_head objlist; + + inline void InitList(void) { INIT_LIST_HEAD(&objlist); } + inline void ResetList(void) { list_del_init(&objlist); } + inline int ListEmpty(void) const { return list_empty(&objlist); } + inline ListHead *ListNext(void) { return list_entry(objlist.next, ListHead, objlist); } + inline ListHead *ListPrev(void) { return list_entry(objlist.prev, ListHead, objlist); } + + inline void ListAdd(ListHead &n) { list_add(&objlist, &n.objlist); } + inline void ListAdd(ListHead *n) { list_add(&objlist, &n->objlist); } + inline void ListAddTail(ListHead &n) { list_add_tail(&objlist, &n.objlist); } + inline void ListAddTail(ListHead *n) { list_add_tail(&objlist, &n->objlist); } + inline void list_del(void) { ResetList(); } + inline void ListMove(ListHead &n) { list_move(&objlist, &n.objlist); } + inline void ListMove(ListHead *n) { list_move(&objlist, &n->objlist); } + inline void list_move_tail(ListHead &n) { _list_move_tail(&objlist, &n.objlist); } + inline void list_move_tail(ListHead *n) { _list_move_tail(&objlist, &n->objlist); } + inline void FreeList(void) + { + while (!ListEmpty()) + ListNext()->ResetList(); + } +}; + +// T is container class +// I is list index, if container inherit multiple ListObject +template +class ListObject : public ListHead +{ +public: + inline ListObject(void) { InitList(); } + inline ~ListObject(void) { ResetList(); } + inline ListObject *ListNext(void) { return (ListObject *)ListHead::ListNext(); } + inline ListObject *ListPrev(void) { return (ListObject *)ListHead::ListPrev(); } + // inline T *ListOwner(void) { return static_cast(this); } + inline T *ListOwner(void) { return (T *)this; } + inline T *NextOwner(void) { return ListNext()->ListOwner(); } + inline T *PrevOwner(void) { return ListPrev()->ListOwner(); } +}; + +#endif +#endif diff --git a/src/search_local/index_storage/common/listener.cc b/src/search_local/index_storage/common/listener.cc new file mode 100644 index 0000000..a8b5550 --- /dev/null +++ b/src/search_local/index_storage/common/listener.cc @@ -0,0 +1,110 @@ +/* + * ===================================================================================== + * + * Filename: listener.cc + * + * Description: listener module. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include "listener.h" +#include "poll_thread.h" +#include "decoder_base.h" +#include "log.h" +#include + +extern int gMaxConnCnt; + +DTCListener::DTCListener(const SocketAddress *s) +{ + sockaddr = s; + bind = 0; + window = 0; + outPeer = NULL; +} + +DTCListener::~DTCListener() +{ + if (sockaddr && sockaddr->socket_family() == AF_UNIX && netfd >= 0) + { + if (sockaddr->Name()[0] == '/') + { + unlink(sockaddr->Name()); + } + } +} + +int DTCListener::Bind(int blog, int rbufsz, int wbufsz) +{ + if (bind) + return 0; + + if ((netfd = sock_bind(sockaddr, blog, rbufsz, wbufsz, 1 /*reuse*/, 1 /*nodelay*/, 1 /*defer_accept*/)) == -1) + return -1; + + bind = 1; + + return 0; +} + +int DTCListener::Attach(DecoderUnit *unit, int blog, int rbufsz, int wbufsz) +{ + if (Bind(blog, rbufsz, wbufsz) != 0) + return -1; + + outPeer = unit; + if (sockaddr->socket_type() == SOCK_DGRAM) + { + if (unit->process_dgram(netfd) < 0) + return -1; + else + netfd = dup(netfd); + return 0; + } + enable_input(); + return attach_poller(unit->owner_thread()); +} + +void DTCListener::input_notify(void) +{ + log_debug("enter input_notify!!!!!!!!!!!!!!!!!"); + int newfd = -1; + socklen_t peerSize; + struct sockaddr peer; + + while (true) + { + peerSize = sizeof(peer); + newfd = accept(netfd, &peer, &peerSize); + + if (newfd == -1) + { + if (errno != EINTR && errno != EAGAIN) + log_notice("[%s]accept failed, fd=%d, %m", sockaddr->Name(), netfd); + + break; + } + + if (outPeer->process_stream(newfd, window, &peer, peerSize) < 0) + close(newfd); + } +} diff --git a/src/search_local/index_storage/common/listener.h b/src/search_local/index_storage/common/listener.h new file mode 100644 index 0000000..53e0ad1 --- /dev/null +++ b/src/search_local/index_storage/common/listener.h @@ -0,0 +1,46 @@ +/* + * ===================================================================================== + * + * Filename: listener.h + * + * Description: listener module. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __LISTENER_H__ +#define __LISTENER_H__ +#include "poller.h" +#include "net_addr.h" + +int unix_sock_bind(const char *path, int backlog = 0); +int udp_sock_bind(const char *addr, uint16_t port, int rbufsz, int wbufsz); +int sock_bind(const char *addr, uint16_t port, int backlog = 0); + +class DecoderUnit; +class DTCListener : public PollerObject +{ +public: + DTCListener(const SocketAddress *); + ~DTCListener(); + + int Bind(int blog, int rbufsz, int wbufsz); + virtual void input_notify(void); + virtual int Attach(DecoderUnit *, int blog = 0, int rbufsz = 0, int wbufsz = 0); + void set_request_window(int w) { window = w; }; + int FD(void) const { return netfd; } + +private: + DecoderUnit *outPeer; + const SocketAddress *sockaddr; + int bind; + int window; +}; +#endif diff --git a/src/search_local/index_storage/common/listener_bind.cc b/src/search_local/index_storage/common/listener_bind.cc new file mode 100644 index 0000000..9e52812 --- /dev/null +++ b/src/search_local/index_storage/common/listener_bind.cc @@ -0,0 +1,161 @@ +/* + * ===================================================================================== + * + * Filename: listener_bind.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unix_socket.h" +#include "log.h" + +int sock_bind(const char *addr, uint16_t port, int backlog) +{ + struct sockaddr_in inaddr; + int reuse_addr = 1; + int netfd; + + bzero(&inaddr, sizeof(struct sockaddr_in)); + inaddr.sin_family = AF_INET; + inaddr.sin_port = htons(port); + + const char *end = strchr(addr, ':'); + if (end) + { + char *p = (char *)alloca(end - addr + 1); + memcpy(p, addr, end - addr); + p[end - addr] = '\0'; + addr = p; + } + if (strcmp(addr, "*") != 0 && + inet_pton(AF_INET, addr, &inaddr.sin_addr) <= 0) + { + log_error("invalid address %s:%d", addr, port); + return -1; + } + + if ((netfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) + { + log_error("make tcp socket error, %m"); + return -1; + } + + setsockopt(netfd, SOL_SOCKET, SO_REUSEADDR, &reuse_addr, sizeof(reuse_addr)); + setsockopt(netfd, SOL_TCP, TCP_NODELAY, &reuse_addr, sizeof(reuse_addr)); + reuse_addr = 60; + /* 避免没有请求的空连接唤醒epoll浪费cpu资源 */ + setsockopt(netfd, SOL_TCP, TCP_DEFER_ACCEPT, &reuse_addr, sizeof(reuse_addr)); + + if (bind(netfd, (struct sockaddr *)&inaddr, sizeof(struct sockaddr)) == -1) + { + log_error("bind tcp %s:%u failed, %m", addr, port); + close(netfd); + return -1; + } + + if (listen(netfd, backlog) == -1) + { + log_error("listen tcp %s:%u failed, %m", addr, port); + close(netfd); + return -1; + } + + log_info("listen on tcp %s:%u", addr, port); + return netfd; +} + +int udp_sock_bind(const char *addr, uint16_t port, int rbufsz, int wbufsz) +{ + struct sockaddr_in inaddr; + int netfd; + + bzero(&inaddr, sizeof(struct sockaddr_in)); + inaddr.sin_family = AF_INET; + inaddr.sin_port = htons(port); + + const char *end = strchr(addr, ':'); + if (end) + { + char *p = (char *)alloca(end - addr + 1); + memcpy(p, addr, end - addr); + p[end - addr] = '\0'; + addr = p; + } + if (strcmp(addr, "*") != 0 && + inet_pton(AF_INET, addr, &inaddr.sin_addr) <= 0) + { + log_error("invalid address %s:%d", addr, port); + return -1; + } + + if ((netfd = socket(AF_INET, SOCK_DGRAM, 0)) == -1) + { + log_error("make udp socket error, %m"); + return -1; + } + + if (bind(netfd, (struct sockaddr *)&inaddr, sizeof(struct sockaddr)) == -1) + { + log_error("bind udp %s:%u failed, %m", addr, port); + close(netfd); + return -1; + } + + if (rbufsz) + setsockopt(netfd, SOL_SOCKET, SO_RCVBUF, &rbufsz, sizeof(rbufsz)); + if (wbufsz) + setsockopt(netfd, SOL_SOCKET, SO_SNDBUF, &wbufsz, sizeof(wbufsz)); + + log_info("listen on udp %s:%u", addr, port); + return netfd; +} + +int unix_sock_bind(const char *path, int backlog) +{ + struct sockaddr_un unaddr; + int netfd; + + socklen_t addrlen = init_unix_socket_address(&unaddr, path); + if ((netfd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) + { + log_error("%s", "make unix socket error, %m"); + return -1; + } + + if (bind(netfd, (struct sockaddr *)&unaddr, addrlen) == -1) + { + log_error("bind unix %s failed, %m", path); + close(netfd); + return -1; + } + + if (listen(netfd, backlog) == -1) + { + log_error("listen unix %s failed, %m", path); + close(netfd); + return -1; + } + + log_info("listen on unix %s, fd=%d", path, netfd); + return netfd; +} diff --git a/src/search_local/index_storage/common/listener_pool.cc b/src/search_local/index_storage/common/listener_pool.cc new file mode 100644 index 0000000..c1cf874 --- /dev/null +++ b/src/search_local/index_storage/common/listener_pool.cc @@ -0,0 +1,195 @@ +/* + * ===================================================================================== + * + * Filename: listener_pool.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "listener_pool.h" +#include "listener.h" +#include "client_unit.h" +#include "poll_thread.h" +#include "unix_socket.h" +#include "task_multiplexer.h" +#include "task_request.h" +#include "config.h" +#include "log.h" + +// this file is obsoleted, new one is agent_listen_pool.[hc] +extern DTCTableDefinition *gTableDef[]; + +ListenerPool::ListenerPool(void) +{ + memset(listener, 0, sizeof(listener)); + memset(thread, 0, sizeof(thread)); + memset(decoder, 0, sizeof(decoder)); +} + +int ListenerPool::init_decoder(int n, int idle, TaskDispatcher *out) +{ + if (thread[n] == NULL) + { + char name[16]; + snprintf(name, sizeof(name), "inc%d", n); + thread[n] = new PollThread(name); + if (thread[n]->initialize_thread() == -1) + return -1; + try + { + decoder[n] = new DTCDecoderUnit(thread[n], gTableDef, idle); + } + catch (int err) + { + DELETE(decoder[n]); + return -1; + } + if (decoder[n] == NULL) + return -1; + + TaskMultiplexer *taskMultiplexer = new TaskMultiplexer(thread[n]); + taskMultiplexer->bind_dispatcher(out); + decoder[n]->bind_dispatcher(taskMultiplexer); + } + + return 0; +} + +int ListenerPool::Bind(DTCConfig *gc, TaskDispatcher *out) +{ + bool hasBindAddr = false; + int idle = gc->get_int_val("cache", "idle_timeout", 100); + if (idle < 0) + { + log_notice("idle_timeout invalid, use default value: 100"); + idle = 100; + } + int single = gc->get_int_val("cache", "SingleIncomingThread", 0); + int backlog = gc->get_int_val("cache", "MaxListenCount", 256); + int win = gc->get_int_val("cache", "MaxRequestWindow", 0); + + for (int i = 0; i < MAXLISTENERS; i++) + { + const char *errmsg; + char bindStr[32]; + char bindPort[32]; + int rbufsz; + int wbufsz; + + if (i == 0) + { + snprintf(bindStr, sizeof(bindStr), "BindAddr"); + snprintf(bindPort, sizeof(bindPort), "BindPort"); + } + else + { + snprintf(bindStr, sizeof(bindStr), "BindAddr%d", i); + snprintf(bindPort, sizeof(bindPort), "BindPort%d", i); + } + + const char *addrStr = gc->get_str_val("cache", bindStr); + if (addrStr == NULL) + continue; + errmsg = sockaddr[i].set_address(addrStr, gc->get_str_val("cache", bindPort)); + if (errmsg) + { + log_error("bad BindAddr%d/BindPort%d: %s\n", i, i, errmsg); + continue; + } + + int n = single ? 0 : i; + if (sockaddr[i].socket_type() == SOCK_DGRAM) + { // DGRAM listener + rbufsz = gc->get_int_val("cache", "UdpRecvBufferSize", 0); + wbufsz = gc->get_int_val("cache", "UdpSendBufferSize", 0); + } + else + { + // STREAM socket listener + rbufsz = wbufsz = 0; + } + + listener[i] = new DTCListener(&sockaddr[i]); + listener[i]->set_request_window(win); + if (listener[i]->Bind(backlog, rbufsz, wbufsz) != 0) + { + if (i == 0) + { + log_crit("Error bind unix-socket"); + return -1; + } + else + { + continue; + } + } + + if (init_decoder(n, idle, out) != 0) + return -1; + if (listener[i]->Attach(decoder[n], backlog) < 0) + return -1; + hasBindAddr = true; + } + if (!hasBindAddr) + { + log_crit("Must has a BindAddr"); + return -1; + } + + for (int i = 0; i < MAXLISTENERS; i++) + { + if (thread[i] == NULL) + continue; + thread[i]->running_thread(); + } + return 0; +} + +ListenerPool::~ListenerPool(void) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (thread[i]) + { + thread[i]->interrupt(); + //delete thread[i]; + } + DELETE(listener[i]); + DELETE(decoder[i]); + } +} + +int ListenerPool::Match(const char *name, int port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (listener[i] == NULL) + continue; + if (sockaddr[i].Match(name, port)) + return 1; + } + return 0; +} + +int ListenerPool::Match(const char *name, const char *port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (listener[i] == NULL) + continue; + if (sockaddr[i].Match(name, port)) + return 1; + } + return 0; +} diff --git a/src/search_local/index_storage/common/listener_pool.h b/src/search_local/index_storage/common/listener_pool.h new file mode 100644 index 0000000..c9dca0b --- /dev/null +++ b/src/search_local/index_storage/common/listener_pool.h @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: listener_pool.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "daemon.h" +#include "net_addr.h" + +// this file is obsoleted, new one is agent_listen_pool.[hc] +class DTCListener; +class PollThread; +class DecoderUnit; +class TaskRequest; +class DTCDecoderUnit; +template +class TaskDispatcher; + +class ListenerPool +{ +private: + SocketAddress sockaddr[MAXLISTENERS]; + DTCListener *listener[MAXLISTENERS]; + PollThread *thread[MAXLISTENERS]; + DTCDecoderUnit *decoder[MAXLISTENERS]; + + int init_decoder(int n, int idle, TaskDispatcher *out); + +public: + ListenerPool(); + ~ListenerPool(); + int Bind(DTCConfig *, TaskDispatcher *); + + int Match(const char *, int = 0); + int Match(const char *, const char *); +}; diff --git a/src/search_local/index_storage/common/localip.c b/src/search_local/index_storage/common/localip.c new file mode 100644 index 0000000..f367055 --- /dev/null +++ b/src/search_local/index_storage/common/localip.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +unsigned int get_local_ip(void) +{ + volatile static unsigned int cached_localip = INADDR_ANY; + int fd; + struct ifconf ifc; + struct ifreq *ifr = NULL; + int n = 0; + int i; + + if (cached_localip != 0) + return cached_localip; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd < 0) + return INADDR_ANY; + + ifc.ifc_len = 0; + ifc.ifc_req = NULL; + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + { + ifr = (struct ifreq *)alloca(ifc.ifc_len > 128 ? ifc.ifc_len : 128); + ifc.ifc_req = ifr; + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + n = ifc.ifc_len / sizeof(struct ifreq); + } + close(fd); + + for (i = 0; i < n; i++) + { + if (ifr[i].ifr_addr.sa_family == AF_INET) + { + unsigned char *p = (unsigned char *)&((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr; + if (p[0] == 10 || (p[0] == 172 && (p[1] >= 16 && p[1] <= 31)) || + (p[0] == 192 && p[1] == 168)) + + return cached_localip = *(unsigned int *)p; + } + } + + return INADDR_ANY; +} diff --git a/src/search_local/index_storage/common/lock.h b/src/search_local/index_storage/common/lock.h new file mode 100644 index 0000000..c8b085a --- /dev/null +++ b/src/search_local/index_storage/common/lock.h @@ -0,0 +1,83 @@ +/* + * ===================================================================================== + * + * Filename: lock.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_LOCK_H__ +#define __DTC_LOCK_H__ + +#include +#include "namespace.h" + +DTC_BEGIN_NAMESPACE + +class Mutex +{ + friend class condition; + +public: + inline Mutex(void) + { + ::pthread_mutex_init(&_mutex, 0); + } + + inline void lock(void) + { + ::pthread_mutex_lock(&_mutex); + } + + inline void unlock(void) + { + ::pthread_mutex_unlock(&_mutex); + } + + inline ~Mutex(void) + { + ::pthread_mutex_destroy(&_mutex); + } + +private: + Mutex(const Mutex &m); + Mutex &operator=(const Mutex &m); + +private: + pthread_mutex_t _mutex; +}; + +/** + * * definition of ScopedLock; + * **/ +class ScopedLock +{ + friend class condition; + +public: + inline ScopedLock(Mutex &mutex) : _mutex(mutex) + { + _mutex.lock(); + } + + inline ~ScopedLock(void) + { + _mutex.unlock(); + } + +private: + Mutex &_mutex; +}; + +DTC_END_NAMESPACE + +#endif //__DTC_LOCK_H__ diff --git a/src/search_local/index_storage/common/lock_free_queue.h b/src/search_local/index_storage/common/lock_free_queue.h new file mode 100644 index 0000000..ada4f72 --- /dev/null +++ b/src/search_local/index_storage/common/lock_free_queue.h @@ -0,0 +1,48 @@ +/* + * ===================================================================================== + * + * Filename: lock_free_queue.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef LOCK_FREE_QUEUE_H +#define LOCK_FREE_QUEUE_H + +#include + +#define LOCK_FREE_QUEUE_DEFAULT_SIZE 65536 +#define CAS(a_ptr, a_oldVal, a_newVal) __sync_bool_compare_and_swap(a_ptr, a_oldVal, a_newVal) + +template +class LockFreeQueue +{ +public: + LockFreeQueue(); + ~LockFreeQueue(); + bool en_queue(const ELEM_T &data); + bool de_queue(ELEM_T &data); + uint32_t Size(); + uint32_t queue_size(); + +private: + ELEM_T *m_theQueue; + uint32_t m_queueSize; + volatile uint32_t m_readIndex; + volatile uint32_t m_writeIndex; + volatile uint32_t m_maximumReadIndex; + inline uint32_t count_to_index(uint32_t count); +}; + +#include "LockFreeQueue_Imp.h" + +#endif diff --git a/src/search_local/index_storage/common/lock_free_queue_imp.h b/src/search_local/index_storage/common/lock_free_queue_imp.h new file mode 100644 index 0000000..f418775 --- /dev/null +++ b/src/search_local/index_storage/common/lock_free_queue_imp.h @@ -0,0 +1,107 @@ +/* + * ===================================================================================== + * + * Filename: lock_free_queue_imp.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "LockFreeQueue.h" +#include + +template +LockFreeQueue::LockFreeQueue() : m_readIndex(0), + m_writeIndex(0), + m_maximumReadIndex(0) +{ + + m_queueSize = 65536; + if (Q_SIZE > 65536) + { + do + { + m_queueSize = m_queueSize << 1; + } while (m_queueSize < Q_SIZE); + m_queueSize = m_queueSize >> 1; + } + m_theQueue = new ELEM_T[m_queueSize]; +} + +template +LockFreeQueue::~LockFreeQueue() +{ + delete[] m_theQueue; +} + +template +uint32_t LockFreeQueue::count_to_index(uint32_t count) +{ + return (count % m_queueSize); +} + +template +uint32_t LockFreeQueue::Size() +{ + uint32_t m_currentWriteIndex = m_writeIndex; + uint32_t m_currentReadIndex = m_readIndex; + if (m_currentWriteIndex >= m_currentReadIndex) + { + return (m_currentWriteIndex - m_currentReadIndex); + } + else + { + return (m_queueSize + m_currentWriteIndex - m_currentReadIndex); + } +} + +template +uint32_t LockFreeQueue::queue_size() +{ + return m_queueSize; +} + +template +bool LockFreeQueue::en_queue(const ELEM_T &data) +{ + uint32_t currentReadIndex; + uint32_t currentWriteIndex; + do + { + currentReadIndex = m_readIndex; + currentWriteIndex = m_writeIndex; + if (count_to_index(currentWriteIndex + 1) == count_to_index(currentReadIndex)) + return false; + } while (false == CAS(&m_writeIndex, currentWriteIndex, currentWriteIndex + 1)); + m_theQueue[count_to_index(currentWriteIndex)] = data; + while (false == CAS(&m_maximumReadIndex, currentWriteIndex, currentWriteIndex + 1)) + { + sched_yield(); + } + return true; +} + +template +bool LockFreeQueue::de_queue(ELEM_T &data) +{ + uint32_t currentMaximumReadIndex; + uint32_t currentReadIndex; + do + { + currentMaximumReadIndex = m_maximumReadIndex; + currentReadIndex = m_readIndex; + if (count_to_index(currentMaximumReadIndex) == count_to_index(currentReadIndex)) + return false; + data = m_theQueue[count_to_index(currentReadIndex)]; + if (true == CAS(&m_readIndex, currentReadIndex, currentReadIndex + 1)) + return true; + } while (1); +} diff --git a/src/search_local/index_storage/common/log.cc b/src/search_local/index_storage/common/log.cc new file mode 100644 index 0000000..610159c --- /dev/null +++ b/src/search_local/index_storage/common/log.cc @@ -0,0 +1,386 @@ +/* + * ===================================================================================== + * + * Filename: log.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "compiler.h" +#include "log.h" +#include "daemon.h" +#include "config.h" +#include "stat_dtc.h" + +#define LOGSIZE 4096 +#define MSGSIZE 4096 +#define REMOTELOG_ERR_TYPE 14 +int __log_level__ = 6; +int __business_id__ = 0; + +static int noconsole = 0; +static int logfd = -1; +static int logday = 0; +static char log_dir[128] = "../log"; +static char appname[32] = ""; +static int (*alert_hook)(const char *, int); + +//remote log area +static struct sockaddr_in remote_log_addr; +static int sockaddr_length = 0; + +StatItemU32 *__statLogCount; +unsigned int __localStatLogCnt[8]; + +void _set_remote_log_config_(const char *addr, int port, int businessid) +{ + if (NULL == addr) + { + return; + } + if (0 == port) + { + return; + } + __business_id__ = businessid; + bzero((void *)&remote_log_addr, sizeof(remote_log_addr)); + remote_log_addr.sin_family = AF_INET; + remote_log_addr.sin_port = htons(port); + int ret = inet_pton(AF_INET, addr, &remote_log_addr.sin_addr); + if (1 != ret) + { + + return; + } + sockaddr_length = sizeof(remote_log_addr); + return; +} + +static inline int _build_remote_log_socket_() +{ + int log_fd = socket(AF_INET, SOCK_DGRAM, 0); + if (log_fd <= 0) + { + + return -1; + } + return log_fd; +} + +#if HAS_TLS +//thread name +static __thread const char *threadname __TLS_MODEL; +void _set_log_thread_name_(const char *n) { threadname = n; } +//remote log fd +static __thread int remote_log_fd; +int _set_remote_log_fd_() +{ + + int fd = _build_remote_log_socket_(); + if (fd > 0) + { + remote_log_fd = fd; + return 0; + } + else + { + return -1; + } +} +#else +//thread name +static pthread_key_t namekey; +static pthread_once_t nameonce = PTHREAD_ONCE_INIT; +static void _init_namekey_(void) { pthread_key_create(&namekey, NULL); } +void _set_log_thread_name_(const char *n) { pthread_setspecific(namekey, n); } +//remote log fd +static pthread_key_t fdkey; +static pthread_once_t fdonce = PTHREAD_ONCE_INIT; +static void _init_fdkey_(void) { pthread_key_create(&fdkey, NULL); } +int _set_remote_log_fd_() +{ + int fd = _build_remote_log_socket_(); + if (fd > 0) + { + pthread_setspecific(fdkey, fd); + return 0; + } + else + { + return -1; + } +} +#endif + +// clean logfd when module unloaded +__attribute__((__destructor__)) static void clean_logfd(void) +{ + if (logfd >= 0) + { + close(logfd); + logfd = -1; + } +} + +void _init_log_(const char *app, const char *dir) +{ +#if !HAS_TLS + pthread_once(&nameonce, _init_namekey_); +#endif + __statLogCount = NULL; // 暂时只在本地统计 + memset(__localStatLogCnt, 0, sizeof(__localStatLogCnt)); + + strncpy(appname, app, sizeof(appname) - 1); + + if (dir) + { + strncpy(log_dir, dir, sizeof(log_dir) - 1); + } + mkdir(log_dir, 0777); + if (access(log_dir, W_OK | X_OK) < 0) + { + log_error("logdir(%s): Not writable", log_dir); + } + + logfd = open("/dev/null", O_WRONLY); + if (logfd < 0) + logfd = dup(2); + fcntl(logfd, F_SETFD, FD_CLOEXEC); +} + +void _set_log_level_(int l) +{ + if (l >= 0) + __log_level__ = l > 4 ? l : 4; +} + +void _set_log_alert_hook_(int (*alert_func)(const char *, int)) +{ + alert_hook = alert_func; +} + +void _write_log_( + int level, + const char *filename, + const char *funcname, + int lineno, + const char *format, ...) +{ + // save errno + int savedErrNo = errno; + int off = 0; + int msgoff = 0; + char buf[LOGSIZE]; + char logfile[256]; +#if !HAS_TLS + const char *threadname; +#endif + + if (appname[0] == 0) + return; + + if (level < 0) + level = 0; + else if (level > 7) + level = 7; + if (__statLogCount == NULL) + ++__localStatLogCnt[level]; + else + ++__statLogCount[level]; + + // construct prefix + struct tm tm; + time_t now = time(NULL); + localtime_r(&now, &tm); +#if HAS_TLS +#else + pthread_once(&nameonce, _init_namekey_); + threadname = (const char *)pthread_getspecific(namekey); +#endif + if (filename == NULL) + { + if (threadname) + { + off = snprintf(buf, LOGSIZE, + "<%d>[%02d:%02d:%02d] %s: ", + level, + tm.tm_hour, tm.tm_min, tm.tm_sec, + threadname); + } + else + { + off = snprintf(buf, LOGSIZE, + "<%d>[%02d:%02d:%02d] pid[%d]: ", + level, + tm.tm_hour, tm.tm_min, tm.tm_sec, + _gettid_()); + } + } + else + { + filename = basename(filename); + if (threadname) + off = snprintf(buf, LOGSIZE, + "<%d>[%02d:%02d:%02d] %s: %s(%d)[%s]: ", + level, + tm.tm_hour, tm.tm_min, tm.tm_sec, + threadname, + filename, lineno, funcname); + else + off = snprintf(buf, LOGSIZE, + "<%d>[%02d:%02d:%02d] pid[%d]: %s(%d)[%s]: ", + level, + tm.tm_hour, tm.tm_min, tm.tm_sec, + _gettid_(), + filename, lineno, funcname); + } + + if (off >= LOGSIZE) + off = LOGSIZE - 1; + + { + int today = tm.tm_year * 1000 + tm.tm_yday; + if (logfd >= 0 && today != logday) + { + int fd; + + logday = today; + snprintf(logfile, sizeof(logfile), + "%s/%s.error%04d%02d%02d.log", log_dir, appname, + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday); + fd = open(logfile, O_CREAT | O_LARGEFILE | O_APPEND | O_WRONLY, 0644); + if (fd >= 0) + { + dup2(fd, logfd); + close(fd); + fcntl(logfd, F_SETFD, FD_CLOEXEC); + } + } + } + + // remember message body start point + msgoff = off; + + { + // formatted message + va_list ap; + va_start(ap, format); + // restore errno + errno = savedErrNo; + off += vsnprintf(buf + off, LOGSIZE - off, format, ap); + va_end(ap); + } + + if (off >= LOGSIZE) + off = LOGSIZE - 1; + if (buf[off - 1] != '\n') + { + buf[off++] = '\n'; + } + + int unused; + + if (logfd >= 0) + { + unused = write(logfd, buf, off); + } + if (level <= 3) + { + remote_log(0, NULL, REMOTELOG_ERR_TYPE, 0, buf, now, 0, 0, off); //error optype is 14 + } + + if (level <= 6 && !noconsole) + { + // debug don't send to console/stderr + unused = fwrite(buf + 3, 1, off - 3, stderr); + if (unused <= 0) + { + // disable console if write error + noconsole = 1; + } + } + + if (alert_hook && level <= 1 /* emerg,alert */) + { + if (alert_hook(buf + msgoff, off - msgoff - 1) < 0) + { + // attr report error, log a warning message + buf[1] = '4'; // 4 is warning level + // replace message body + off = msgoff + snprintf(buf + msgoff, LOGSIZE - msgoff, "%s", "report to attr failed\n"); + // log another line + unused = fwrite(buf + 3, 1, off - 3, stderr); + if (logfd >= 0) + { + unused = write(logfd, buf, off); + } + } + } +} + +struct CRemoteLog +{ + int cmd; + int magic; + int len; + char body[0]; +}; + +void remote_log(int type, const char *key, int op_type, int op_result, char *content, long op_time, int cmd, int magic, int contentlen) +{ + if (0 == sockaddr_length) + { + return; + } +#if !HAS_TLS + int remote_log_fd; + pthread_once(&fdonce, _init_fdkey_); + remote_log_fd = *(int *)pthread_getspecific(fdkey); +#endif + if (REMOTELOG_ERR_TYPE == op_type) + { + if ((contentlen > 0) && ('\n' == content[contentlen - 1])) + { + content[contentlen - 1] = 0; + } + } + int off = 0; + char msg[MSGSIZE]; + memset(msg, 0, MSGSIZE); + CRemoteLog *pRemoteLog = (CRemoteLog *)msg; + int body_max_len = MSGSIZE - sizeof(CRemoteLog); + off = snprintf(pRemoteLog->body, body_max_len, "{\"routekey\":%d,\"tableconfig\":\"%s\",\"fieldvalues\":[{\"bid\":%d,\"ukey\":\"%s\",\"op_type\":%d,\"content\":\"%s\",\"op_time\":%ld,\"op_result\":%d}]}", + __business_id__, type ? "dtc_business.xml" : "dtc_error.xml", __business_id__, key == NULL ? "" : key, op_type, content == NULL ? "" : content, op_time, op_result); + + pRemoteLog->cmd = cmd; + pRemoteLog->magic = magic; + pRemoteLog->len = off; + + //printf("send fd is %d", remote_log_fd); + + if (remote_log_fd > 0) + { + sendto(remote_log_fd, msg, sizeof(CRemoteLog) + off, 0, (const sockaddr *)&remote_log_addr, sockaddr_length); + } +} diff --git a/src/search_local/index_storage/common/log.h b/src/search_local/index_storage/common/log.h new file mode 100644 index 0000000..3605650 --- /dev/null +++ b/src/search_local/index_storage/common/log.h @@ -0,0 +1,104 @@ +/* + * ===================================================================================== + * + * Filename: log.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTCLOG_H__ +#define __DTCLOG_H__ + +#include +__BEGIN_DECLS + +#include + +extern int __log_level__; +extern int __business_id__; +#define log_bare(lvl, fmt, args...) _write_log_(lvl, NULL, NULL, 0, fmt, ##args) +#define log_generic(lvl, fmt, args...) _write_log_(lvl, __FILE__, __FUNCTION__, __LINE__, fmt, ##args) +#define log_emerg(fmt, args...) log_generic(0, fmt, ##args) +#define log_alert(fmt, args...) log_generic(1, fmt, ##args) +#define log_crit(fmt, args...) log_generic(2, fmt, ##args) +#define log_error(fmt, args...) log_generic(3, fmt, ##args) +#define log_warning(fmt, args...) \ + do \ + { \ + if (__log_level__ >= 4) \ + log_generic(4, fmt, ##args); \ + } while (0) +#define log_notice(fmt, args...) \ + do \ + { \ + if (__log_level__ >= 5) \ + log_generic(5, fmt, ##args); \ + } while (0) +#define log_info(fmt, args...) \ + do \ + { \ + if (__log_level__ >= 6) \ + log_generic(6, fmt, ##args); \ + } while (0) +#define log_debug(fmt, args...) \ + do \ + { \ + if (__log_level__ >= 7) \ + log_generic(7, fmt, ##args); \ + } while (0) + +#define error_log(fmt, args...) log_error(fmt, ##args) + +#if __cplusplus +extern void _init_log_(const char *app, const char *dir = NULL); +#else +extern void _init_log_(const char *app, const char *dir); +#endif +extern void _init_log_alerter_(void); +extern void _init_log_stat_(void); +extern void _set_log_level_(int); +extern void _set_log_alert_hook_(int (*alert_func)(const char *, int)); +extern void _set_log_thread_name_(const char *n); +extern void _write_log_(int, const char *, const char *, int, const char *, ...) __attribute__((format(printf, 5, 6))); +extern int _set_remote_log_fd_(); +extern void remote_log(int type, const char *key, int op_type, int op_result, char *content, long op_time, int cmd, int magic, int contentlen); +extern void _set_business_id_(int); + +#include +#include +#ifndef __NR_gettid +#endif +static inline int _gettid_(void) +{ + return syscall(__NR_gettid); +} + +#include +static inline unsigned int GET_MSEC(void) +{ + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000 + tv.tv_usec / 1000; +} +#define INIT_MSEC(v) v = GET_MSEC() +#define CALC_MSEC(v) v = GET_MSEC() - (v) +static inline unsigned int GET_USEC(void) +{ + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; +} +#define INIT_USEC(v) v = GET_USEC() +#define CALC_USEC(v) v = GET_USEC() - (v) + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/log_alert.cc b/src/search_local/index_storage/common/log_alert.cc new file mode 100644 index 0000000..c914643 --- /dev/null +++ b/src/search_local/index_storage/common/log_alert.cc @@ -0,0 +1,73 @@ +/* + * ===================================================================================== + * + * Filename: log_alert.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "log.h" +#include "config.h" +#include "daemon.h" + +#include + +static char iname[1024]; +static int inamelen; +static unsigned int attrid; + +static int alert_func_attr(const char *msg, int len) +{ + int ret = 0; + if (attrid > 600) /* 二级网管允许的最小告警id */ + { + if (len + inamelen >= (int)sizeof(iname)) + { + len = sizeof(iname) - inamelen; + } + memcpy(iname + inamelen, msg, len); + if (adv_attr_set(attrid, inamelen + len, iname)) + ret = -1; + } + return ret; +} + +static int scan_process_name(char name[], size_t size) +{ + char str[1024] = {0}; + + if (readlink("/proc/self/cwd", str, 1023) < 0) + return -1; + + snprintf(name, size, "%s", basename(dirname(str))); + return 0; +} + +void _init_log_alerter_(void) +{ + attrid = gConfig->get_int_val("cache", "OpenningFDAttrID", 0); + if (attrid > 600) /* 二级网管允许的最小告警id */ + { + /* 实例名称 */ + if (scan_process_name(iname, sizeof(iname))) + snprintf(iname, sizeof(iname), "dtc"); + inamelen = strlen(iname); + // append ": ", report as: "iname: msg" + iname[inamelen++] = ':'; + iname[inamelen++] = ' '; + + _set_log_alert_hook_(alert_func_attr); + log_info("log_alert send to attr id=%u\n", attrid); + } +} diff --git a/src/search_local/index_storage/common/log_stat.cc b/src/search_local/index_storage/common/log_stat.cc new file mode 100644 index 0000000..33c107a --- /dev/null +++ b/src/search_local/index_storage/common/log_stat.cc @@ -0,0 +1,42 @@ +/* + * ===================================================================================== + * + * Filename: log_stat.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "log.h" +#include "stat_dtc.h" +extern StatItemU32 *__statLogCount; +extern unsigned int __localStatLogCnt[8]; + +void _init_log_stat_(void) +{ + __statLogCount = new StatItemU32[8]; + for (unsigned i = 0; i < 8; i++) + { + __statLogCount[i] = statmgr.get_item_u32(LOG_COUNT_0 + i); + __statLogCount[i].set(__localStatLogCnt[i]); + } +} + +// prevent memory leak if module unloaded +// unused-yet because this file never link into module +__attribute__((__constructor__)) static void clean_logstat(void) +{ + if (__statLogCount != NULL) + { + delete __statLogCount; + __statLogCount = NULL; + } +} diff --git a/src/search_local/index_storage/common/lqueue.h b/src/search_local/index_storage/common/lqueue.h new file mode 100644 index 0000000..d3ea092 --- /dev/null +++ b/src/search_local/index_storage/common/lqueue.h @@ -0,0 +1,196 @@ +/* + * ===================================================================================== + * + * Filename: lqueue.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __LQUEUE_H__ +#define __LQUEUE_H__ + +#include +#include "compiler.h" + +template +class LinkQueue +{ +protected: + struct slot_t + { + slot_t *next; + T data; + }; + +public: + class allocator + { + public: + inline allocator(int mc = 1024) + { + freeList = NULL; + freecount = 0; + freemax = mc > 60000 ? 60000 : mc; + } + inline ~allocator() + { + free_cache(); + } + + private: + friend class LinkQueue; + unsigned short freecount; + unsigned short freemax; + slot_t *freeList; + + __INLINE void free_cache() + { + while (freeList) + { + slot_t *s = freeList; + freeList = s->next; + freecount--; + free(s); + } + } + __INLINE slot_t *getslot(void) + { + slot_t *s; + if (likely(freeList)) + { + s = freeList; + freeList = s->next; + freecount--; + } + else + { + s = (slot_t *)malloc(sizeof(slot_t)); + } + return s; + } + __INLINE void putslot(slot_t *s) + { + if (unlikely(freecount >= freemax)) + { + free(s); + } + else + { + s->next = freeList; + freeList = s; + freecount++; + } + } + }; + +public: + inline LinkQueue(allocator *a = 0) + { + count = 0; + head = NULL; + tail = &head; + alloc = a; + } + + inline ~LinkQueue() + { + while (head) + { + slot_t *s = head; + head = s->next; + free(s); + } + head = NULL; + tail = NULL; + } + + __INLINE void SetAllocator(allocator *a) { alloc = a; } + __INLINE int queue_empty() const + { + return count == 0; + } + + __INLINE int Count() const + { + return count; + } + + __INLINE T Front(void) const + { + return head == NULL ? NULL : head->data; + } + + __INLINE int Push(T p) + { + slot_t *s = getslot(); + if (unlikely(s == NULL)) + return -1; + + count++; + s->data = p; + s->next = NULL; + *tail = s; + tail = &s->next; + return 0; + } + + __INLINE int Unshift(T p) + { + slot_t *s = getslot(); + if (s == NULL) + return -1; + count++; + + s->data = p; + s->next = head; + if (head == NULL) + tail = &s->next; + head = s; + return 0; + } + + __INLINE T Pop() + { + if (head == NULL) + return (T)0; + slot_t *s = head; + head = s->next; + if (head == NULL) + tail = &head; + T ret = s->data; + putslot(s); + count--; + return ret; + } + +protected: + allocator *alloc; + slot_t *head; + slot_t **tail; + int count; + +private: + __INLINE slot_t *getslot(void) + { + if (alloc) + return alloc->getslot(); + return (slot_t *)malloc(sizeof(slot_t)); + } + __INLINE void putslot(slot_t *p) + { + if (alloc) + return alloc->putslot(p); + return free(p); + } +}; + +#endif diff --git a/src/search_local/index_storage/common/markup_stl.cc b/src/search_local/index_storage/common/markup_stl.cc new file mode 100644 index 0000000..6a749fe --- /dev/null +++ b/src/search_local/index_storage/common/markup_stl.cc @@ -0,0 +1,2030 @@ +// MarkupSTL.cpp: implementation of the MarkupSTL class. +// +// Markup Release 6.3 +// Copyright (C) 1999-2002 First Objective Software, Inc. All rights reserved +// Go to www.firstobject.com for the latest CMarkup and EDOM documentation +// Use in commercial applications requires written permission +// This software is provided "as is", with no warranty. +#ifdef _CGIHOST_PROCESS +#include "../include/tsfnew.h" +#endif + +#include "markup_stl.h" +#include +#include +using namespace std; + +#ifdef _DEBUG +#undef THIS_FILE +static char THIS_FILE[] = __FILE__; +#define new DEBUG_NEW +#endif + +//[CMARKUPDEV +// To add a version tag to new documents, define MARKUP_VERSIONTAG +#ifdef MARKUP_VERSIONTAG +#define x_VERSIONTAG "\r\n" +#endif +//]CMARKUPDEV + +void MarkupSTL::operator=(const MarkupSTL &markup) +{ + m_iPosParent = markup.m_iPosParent; + m_iPos = markup.m_iPos; + m_iPosChild = markup.m_iPosChild; + m_iPosFree = markup.m_iPosFree; + m_nNodeType = markup.m_nNodeType; + //[CMARKUPDEV + m_nNodeOffset = markup.m_nNodeOffset; + m_nNodeLength = markup.m_nNodeLength; + //]CMARKUPDEV + m_aPos = markup.m_aPos; + m_strDoc = markup.m_strDoc; + MARKUP_SETDEBUGSTATE; +} + +bool MarkupSTL::set_doc(const char *szDoc) +{ + // Reset indexes + m_iPosFree = 1; + reset_pos(); + m_mapSavedPos.clear(); + + // Set document text + if (szDoc) + m_strDoc = szDoc; + else + m_strDoc.erase(); + + // Starting size of position array: 1 element per 64 bytes of document + // Tight fit when parsing small doc, only 0 to 2 reallocs when parsing large doc + // Start at 8 when creating new document + uint32_t nStartSize = m_strDoc.size() / 64 + 8; + if (m_aPos.size() < nStartSize) + m_aPos.resize(nStartSize); + + // Parse document + bool bWellFormed = false; + if (m_strDoc.size()) + { + m_aPos[0].Clear(); + int iPos = x_ParseElem(0); + if (iPos > 0) + { + m_aPos[0].iElemChild = iPos; + bWellFormed = true; + } + } + + // Clear indexes if parse failed or empty document + if (!bWellFormed) + { + m_aPos[0].Clear(); + m_iPosFree = 1; + } + + reset_pos(); + return bWellFormed; +}; + +bool MarkupSTL::is_well_formed() +{ + if (m_aPos.size() && m_aPos[0].iElemChild) + return true; + return false; +} + +bool MarkupSTL::Load(const char *szFileName) +{ + // Load document from file + bool bResult = false; + FILE *fp = fopen(szFileName, "rb"); + if (fp) + { + // Determine file length + fseek(fp, 0L, SEEK_END); + int nFileLen = ftell(fp); + fseek(fp, 0L, SEEK_SET); + + // Load string + allocator mem; + allocator::pointer pBuffer = mem.allocate(nFileLen + 1, NULL); + if (fread(pBuffer, nFileLen, 1, fp) == 1) + { + pBuffer[nFileLen] = '\0'; + bResult = set_doc(pBuffer); + } + fclose(fp); + mem.deallocate(pBuffer, 1); + } + if (!bResult) + { + set_doc(NULL); + } + MARKUP_SETDEBUGSTATE; + return bResult; +} + +bool MarkupSTL::Save(const char *szFileName) +{ + // Save document to file + bool bResult = false; + FILE *fp = fopen(szFileName, "wb"); + if (fp) + { + // Save string + int nFileLen = m_strDoc.size(); + if (!nFileLen) + bResult = true; + else if (fwrite(m_strDoc.c_str(), nFileLen, 1, fp) == 1) + bResult = true; + fclose(fp); + } + return bResult; +} + +bool MarkupSTL::find_elem(const char *szName) +{ + // Change current position only if found + // + if (m_aPos.size()) + { + int iPos = x_FindElem(m_iPosParent, m_iPos, szName); + if (iPos) + { + // Assign new position + x_SetPos(m_aPos[iPos].iElemParent, iPos, 0); + return true; + } + } + return false; +} + +bool MarkupSTL::find_child_elem(const char *szName) +{ + // Change current child position only if found + // + // Shorthand: call this with no current main position + // means find child under root element + if (!m_iPos) + find_elem(); + + int iPosChild = x_FindElem(m_iPos, m_iPosChild, szName); + if (iPosChild) + { + // Assign new position + int iPos = m_aPos[iPosChild].iElemParent; + x_SetPos(m_aPos[iPos].iElemParent, iPos, iPosChild); + return true; + } + + return false; +} + +//[CMARKUPDEV +int MarkupSTL::find_node(int nType) +{ + // Change current node position only if a node is found + // If nType is 0 find any node, otherwise find node of type nType + // Return type of node or 0 if not found + // If found node is an element, change m_iPos + + // Determine where in document to start scanning for node + int nTypeFound = 0; + int nNodeOffset = m_nNodeOffset; + if (m_nNodeType > 1) + { + // By-pass current node + nNodeOffset += m_nNodeLength; + } + else + { + // Set position to begin looking for node + nNodeOffset = 0; // default to start of document + if (m_iPos) + { + // After element + nNodeOffset = m_aPos[m_iPos].nEndR + 1; + } + else + { + // Immediately after start tag of parent + if (m_aPos[m_iPosParent].is_empty_element()) + return 0; + if (m_iPosParent) + nNodeOffset = m_aPos[m_iPosParent].nStartR + 1; + } + } + + // Get nodes until we find what we're looking for + int iPosNew = m_iPos; + TokenPos token(m_strDoc.c_str()); + token.nNext = nNodeOffset; + do + { + nNodeOffset = token.nNext; + nTypeFound = x_ParseNode(token); + if (!nTypeFound) + return 0; + if (nTypeFound == MNT_ELEMENT) + { + if (iPosNew) + iPosNew = m_aPos[iPosNew].iElemNext; + else + iPosNew = m_aPos[m_iPosParent].iElemChild; + if (!iPosNew) + return 0; + if (!nType || (nType & nTypeFound)) + { + // Found node, move position to this element + x_SetPos(m_iPosParent, iPosNew, 0); + return m_nNodeType; + } + token.nNext = m_aPos[iPosNew].nEndR + 1; + } + } while (nType && !(nType & nTypeFound)); + + m_iPos = iPosNew; + m_iPosChild = 0; + m_nNodeOffset = nNodeOffset; + m_nNodeLength = token.nNext - nNodeOffset; + m_nNodeType = nTypeFound; + MARKUP_SETDEBUGSTATE; + return m_nNodeType; +} + +bool MarkupSTL::remove_node() +{ + if (m_iPos || m_nNodeLength) + { + x_RemoveNode(m_iPosParent, m_iPos, m_nNodeType, m_nNodeOffset, m_nNodeLength); + m_iPosChild = 0; + MARKUP_SETDEBUGSTATE; + return true; + } + return false; +} +//]CMARKUPDEV + +string MarkupSTL::get_tag_name() const +{ + // Return the tag name at the current main position + string strTagName; + + //[CMARKUPDEV + // This method is primarily for elements, however + // it does return something for certain other nodes + if (m_nNodeLength) + { + switch (m_nNodeType) + { + case MNT_PROCESSING_INSTRUCTION: + { + // Processing instruction returns target + // For the document version it returns "xml" + TokenPos token(m_strDoc.c_str()); + token.nNext = m_nNodeOffset + 2; + if (x_FindToken(token)) + strTagName = x_GetToken(token); + } + break; + case MNT_COMMENT: + strTagName = "#comment"; + break; + case MNT_CDATA_SECTION: + strTagName = "#cdata-section"; + break; + case MNT_DOCUMENT_TYPE: + { + // Document type returns document type name + TokenPos token(m_strDoc.c_str()); + token.nNext = m_nNodeOffset + 2; + // Find second token + if (x_FindToken(token) && x_FindToken(token)) + strTagName = x_GetToken(token); + } + break; + case MNT_TEXT: + case MNT_WHITESPACE: + strTagName = "#text"; + break; + } + return strTagName; + } + //]CMARKUPDEV + + if (m_iPos) + strTagName = x_GetTagName(m_iPos); + return strTagName; +} + +bool MarkupSTL::into_elem() +{ + // If there is no child position and into_elem is called it will succeed in release 6.3 + // (A subsequent call to find_elem will find the first element) + // The following short-hand behavior was never part of EDOM and was misleading + // It would find a child element if there was no current child element position and go into it + // It is removed in release 6.3, this change is NOT backwards compatible! + // if ( ! m_iPosChild ) + // find_child_elem(); + + if (m_iPos && m_nNodeType == MNT_ELEMENT) + { + x_SetPos(m_iPos, m_iPosChild, 0); + return true; + } + return false; +} + +bool MarkupSTL::out_of_elem() +{ + // Go to parent element + if (m_iPosParent) + { + x_SetPos(m_aPos[m_iPosParent].iElemParent, m_iPosParent, m_iPos); + return true; + } + return false; +} + +string MarkupSTL::get_attrib_name(int n) const +{ + // Return nth attribute name of main position + if (!m_iPos || m_nNodeType != MNT_ELEMENT) + return ""; + + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[m_iPos].nStartL + 1; + for (int nAttrib = 0; nAttrib <= n; ++nAttrib) + if (!x_FindAttrib(token)) + return ""; + + // Return substring of document + return x_GetToken(token); +} + +bool MarkupSTL::save_pos(const char *szPosName) +{ + // Save current element position in saved position map + if (szPosName) + { + SavedPos savedpos; + savedpos.iPosParent = m_iPosParent; + savedpos.iPos = m_iPos; + savedpos.iPosChild = m_iPosChild; + string strPosName = szPosName; + m_mapSavedPos[strPosName] = savedpos; + return true; + } + return false; +} + +bool MarkupSTL::restore_pos(const char *szPosName) +{ + // Restore element position if found in saved position map + if (szPosName) + { + string strPosName = szPosName; + mapSavedPosT::const_iterator iterSavePos = m_mapSavedPos.find(strPosName); + if (iterSavePos != m_mapSavedPos.end()) + { + SavedPos savedpos = (*iterSavePos).second; + x_SetPos(savedpos.iPosParent, savedpos.iPos, savedpos.iPosChild); + return true; + } + } + return false; +} + +bool MarkupSTL::get_offsets(int &nStart, int &nEnd) const +{ + // Return document offsets of current main position element + // This is not part of EDOM but is used by the Markup project + if (m_iPos) + { + nStart = m_aPos[m_iPos].nStartL; + nEnd = m_aPos[m_iPos].nEndR; + return true; + } + return false; +} + +string MarkupSTL::get_child_sub_doc() const +{ + if (m_iPosChild) + { + int nL = m_aPos[m_iPosChild].nStartL; + int nR = m_aPos[m_iPosChild].nEndR + 1; + TokenPos token(m_strDoc.c_str()); + token.nNext = nR; + if (!x_FindToken(token) || m_strDoc[token.nL] == '<') + nR = token.nL; + return m_strDoc.substr(nL, nR - nL); + } + return ""; +} + +bool MarkupSTL::remove_elem() +{ + // Remove current main position element + if (m_iPos && m_nNodeType == MNT_ELEMENT) + { + int iPos = x_RemoveElem(m_iPos); + x_SetPos(m_iPosParent, iPos, 0); + return true; + } + return false; +} + +bool MarkupSTL::remove_child_elem() +{ + // Remove current child position element + if (m_iPosChild) + { + int iPosChild = x_RemoveElem(m_iPosChild); + x_SetPos(m_iPosParent, m_iPos, iPosChild); + return true; + } + return false; +} + +//[CMARKUPDEV +string MarkupSTL::find_get_data(const char *szName) +{ + if (find_elem(szName)) + return x_GetData(m_iPos); + return ""; +} + +bool MarkupSTL::find_set_data(const char *szName, const char *szData, int nCDATA) +{ + if (find_elem(szName)) + return x_SetData(m_iPos, szData, nCDATA); + return false; +} + +// Base64 methods +string MarkupSTL::EncodeBase64(const unsigned char *pBuffer, int nBufferLen) +{ + // Return Base64 string + // 1 byte takes 2, 2:3, 3:4, 4:6, 5:7, 6:8, 7:10.... + int nLenBase64 = nBufferLen + nBufferLen / 3 + 3; + nLenBase64 += (nLenBase64 / 64) * 2 + 4; // CRLFs + string strBase64; + strBase64.resize(nLenBase64); + + // Start with CRLF + int nOffsetBase64 = 0; + strBase64[nOffsetBase64++] = '\r'; + strBase64[nOffsetBase64++] = '\n'; + + // Loop through pBuffer 3 bytes at a time + const char *pCodes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + int nOffsetBuffer = 0; + unsigned int n3BufferBytes; + int nTop; + int nPad = 0; + while (nOffsetBuffer < nBufferLen) + { + // Set n3BufferBytes + n3BufferBytes = pBuffer[nOffsetBuffer++]; + n3BufferBytes = n3BufferBytes << 8; + if (nOffsetBuffer < nBufferLen) + n3BufferBytes |= pBuffer[nOffsetBuffer++]; + else + ++nPad; + n3BufferBytes = n3BufferBytes << 8; + if (nOffsetBuffer < nBufferLen) + n3BufferBytes |= pBuffer[nOffsetBuffer++]; + else + ++nPad; + + // Fill strBase64 + nTop = nOffsetBase64 + 4; + while (nTop != nOffsetBase64) + { + --nTop; + if (nPad == 0 || nTop < nOffsetBase64 + 4 - nPad) + strBase64[nTop] = pCodes[n3BufferBytes & 0x3f]; + else + strBase64[nTop] = '='; + n3BufferBytes = n3BufferBytes >> 6; + } + nOffsetBase64 += 4; + + if (!(nOffsetBase64 % 66) || nPad) + { + strBase64[nOffsetBase64++] = '\r'; + strBase64[nOffsetBase64++] = '\n'; + } + } + + strBase64.resize(nOffsetBase64); + return strBase64; +} + +int MarkupSTL::DecodeBase64(const string &strBase64, unsigned char *pBuffer, int nBufferLen) +{ + // Return length of binary buffer after decoding + // If nBufferLen is 0, returns long enough length without decoding + // so that you can allocate a large enough buffer + // 1 byte takes 2, 2:3, 3:4, 4:6, 5:7, 6:8, 7:10.... + // Otherwise, returns actual decoded length as long as it fits inside nBufferLen + int nLenBase64 = strBase64.size(); + if (!nBufferLen) + return (nLenBase64 / 4) * 3 + 3; + + // Loop through Base64, 4 bytes at a time + const unsigned char *pBase64 = (const unsigned char *)strBase64.c_str(); + int nOffsetBase64 = 0; + int nOffsetBuffer = 0; + unsigned int n3BufferBytes; + int nTop; + int nCount; + int nCode; + int nPad = 0; + while (nOffsetBase64 < nLenBase64) + { + // Set n3BufferBytes + nCount = 0; + n3BufferBytes = 0; + while (nCount != 4 && nOffsetBase64 < nLenBase64) + { + // Check encoded character and bypass if whitespace + nCode = pBase64[nOffsetBase64++]; + if (nCode == '\r' || nCode == '\n') + continue; + + // Shift previous bits up before OR-ing 6 lower sig bits + n3BufferBytes = n3BufferBytes << 6; + + // Use ASCII codes for fast conversion + if (nCode > 96) //(a-z) + n3BufferBytes |= (nCode - 71); + else if (nCode > 64) //(A-Z) + n3BufferBytes |= (nCode - 65); + else if (nCode == 61) //(=) + ++nPad; + else if (nCode > 47) //(0-9) + n3BufferBytes |= (nCode + 4); + else if (nCode == 47) //(/) + n3BufferBytes |= 63; + else if (nCode == 43) //(+) + n3BufferBytes |= 62; + ++nCount; + } + + if (nCount == 4) + { + // Fill pBuffer + nTop = nOffsetBuffer + 3; + if (nTop - nPad > nBufferLen) + return 0; + while (nTop != nOffsetBuffer) + { + --nTop; + if (nPad == 0 || nTop < nOffsetBuffer + 3 - nPad) + pBuffer[nTop] = (unsigned char)(n3BufferBytes & 0xff); + n3BufferBytes = n3BufferBytes >> 8; + } + nOffsetBuffer += 3 - nPad; + } + } + + return nOffsetBuffer; +} +//]CMARKUPDEV + +////////////////////////////////////////////////////////////////////// +// Private Methods +////////////////////////////////////////////////////////////////////// + +int MarkupSTL::x_GetFreePos() +{ + // + // This returns the index of the next unused ElemPos in the array + // + if ((uint32_t)m_iPosFree == m_aPos.size()) + m_aPos.resize(m_iPosFree + m_iPosFree / 2); + ++m_iPosFree; + return m_iPosFree - 1; +} + +int MarkupSTL::x_ReleasePos() +{ + // + // This decrements the index of the next unused ElemPos in the array + // allowing the element index returned by GetFreePos() to be reused + // + --m_iPosFree; + return 0; +} + +int MarkupSTL::x_ParseError(const char *szError, const char *szName) +{ + if (szName) + { + char szFormat[300]; + snprintf(szFormat, 299, szError, szName); + m_strError = szFormat; + } + else + m_strError = szError; + x_ReleasePos(); + return -1; +} + +int MarkupSTL::x_ParseElem(int iPosParent) +{ + // This is either called by set_doc, x_AddSubDoc, or itself recursively + // m_aPos[iPosParent].nEndL is where to start parsing for the child element + // This returns the new position if a tag is found, otherwise zero + // In all cases we need to get a new ElemPos, but release it if unused + // + int iPos = x_GetFreePos(); + m_aPos[iPos].nStartL = m_aPos[iPosParent].nEndL; + m_aPos[iPos].iElemParent = iPosParent; + m_aPos[iPos].iElemChild = 0; + m_aPos[iPos].iElemNext = 0; + + // Start Tag + // A loop is used to ignore all remarks tags and special tags + // i.e. , and + // So any tag beginning with ? or ! is ignored + // Loop past ignored tags + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[iPosParent].nEndL; + string strName; + while (strName.empty()) + { + // Look for left angle bracket of start tag + m_aPos[iPos].nStartL = token.nNext; + if (!x_FindChar(token.szDoc, m_aPos[iPos].nStartL, '<')) + return x_ParseError("Element tag not found"); + + // Set parent's End tag to start looking from here (or later) + m_aPos[iPosParent].nEndL = m_aPos[iPos].nStartL; + + // Determine whether this is an element, or bypass other type of node + token.nNext = m_aPos[iPos].nStartL + 1; + if (x_FindToken(token)) + { + if (token.bIsString) + return x_ParseError("Tag starts with quote"); + char cFirstChar = m_strDoc[token.nL]; + if (cFirstChar == '?' || cFirstChar == '!') + { + token.nNext = m_aPos[iPos].nStartL; + if (!x_ParseNode(token)) + return x_ParseError("Invalid node"); + } + else if (cFirstChar != '/') + { + strName = x_GetToken(token); + // Look for end of tag + if (!x_FindChar(token.szDoc, token.nNext, '>')) + return x_ParseError("End of tag not found"); + } + else + return x_ReleasePos(); // probably end tag of parent + } + else + return x_ParseError("Abrupt end within tag"); + } + m_aPos[iPos].nStartR = token.nNext; + + // Is ending mark within start tag, i.e. empty element? + if (m_strDoc[m_aPos[iPos].nStartR - 1] == '/') + { + // Empty element + // Close tag left is set to ending mark, and right to open tag right + m_aPos[iPos].nEndL = m_aPos[iPos].nStartR - 1; + m_aPos[iPos].nEndR = m_aPos[iPos].nStartR; + } + else // look for end tag + { + // Element probably has contents + // Determine where to start looking for left angle bracket of end tag + // This is done by recursively parsing the contents of this element + int iInner, iInnerPrev = 0; + m_aPos[iPos].nEndL = m_aPos[iPos].nStartR + 1; + while ((iInner = x_ParseElem(iPos)) > 0) + { + // Set links to iInner + if (iInnerPrev) + m_aPos[iInnerPrev].iElemNext = iInner; + else + m_aPos[iPos].iElemChild = iInner; + iInnerPrev = iInner; + + // Set offset to reflect child + m_aPos[iPos].nEndL = m_aPos[iInner].nEndR + 1; + } + if (iInner == -1) + return -1; + + // Look for left angle bracket of end tag + if (!x_FindChar(token.szDoc, m_aPos[iPos].nEndL, '<')) + return x_ParseError("End tag of %s element not found", strName.c_str()); + + // Look through tokens of end tag + token.nNext = m_aPos[iPos].nEndL + 1; + int nTokenCount = 0; + while (x_FindToken(token)) + { + ++nTokenCount; + if (!token.bIsString) + { + // Is first token not an end slash mark? + if (nTokenCount == 1 && m_strDoc[token.nL] != '/') + return x_ParseError("Expecting end tag of element %s", strName.c_str()); + + else if (nTokenCount == 2 && !token.Match(strName.c_str())) + return x_ParseError("End tag does not correspond to %s", strName.c_str()); + + // Else is it a right angle bracket? + else if (m_strDoc[token.nL] == '>') + break; + } + } + + // Was a right angle bracket not found? + if (!token.szDoc[token.nL] || nTokenCount < 2) + return x_ParseError("End tag not completed for element %s", strName.c_str()); + m_aPos[iPos].nEndR = token.nL; + } + + // Successfully parsed element (and contained elements) + return iPos; +} + +bool MarkupSTL::x_FindChar(const char *szDoc, int &nChar, char c) +{ + // static function + const char *pChar = &szDoc[nChar]; + while (*pChar && *pChar != c) + pChar += 1; + nChar = pChar - szDoc; + if (!*pChar) + return false; + /* + while ( szDoc[nChar] && szDoc[nChar] != c ) + nChar += _tclen( &szDoc[nChar] ); + if ( ! szDoc[nChar] ) + return false; + */ + return true; +} + +bool MarkupSTL::x_FindToken(MarkupSTL::TokenPos &token) +{ + // Starting at token.nNext, bypass whitespace and find the next token + // returns true on success, members of token point to token + // returns false on end of document, members point to end of document + const char *szDoc = token.szDoc; + int nChar = token.nNext; + token.bIsString = false; + + // By-pass leading whitespace + while (szDoc[nChar] && strchr(" \t\n\r", szDoc[nChar])) + ++nChar; + if (!szDoc[nChar]) + { + // No token was found before end of document + token.nL = nChar; + token.nR = nChar; + token.nNext = nChar; + return false; + } + + // Is it an opening quote? + char cFirstChar = szDoc[nChar]; + if (cFirstChar == '\"' || cFirstChar == '\'') + { + token.bIsString = true; + + // Move past opening quote + ++nChar; + token.nL = nChar; + + // Look for closing quote + x_FindChar(token.szDoc, nChar, cFirstChar); + + // Set right to before closing quote + token.nR = nChar - 1; + + // Set nChar past closing quote unless at end of document + if (szDoc[nChar]) + ++nChar; + } + else + { + // Go until special char or whitespace + token.nL = nChar; + while (szDoc[nChar] && !strchr(" \t\n\r<>=\\/?!", szDoc[nChar])) + nChar += 1; + + // Adjust end position if it is one special char + if (nChar == token.nL) + ++nChar; // it is a special char + token.nR = nChar - 1; + } + + // nNext points to one past last char of token + token.nNext = nChar; + return true; +} + +string MarkupSTL::x_GetToken(const MarkupSTL::TokenPos &token) const +{ + // The token contains indexes into the document identifying a small substring + // Build the substring from those indexes and return it + if (token.nL > token.nR) + return ""; + return m_strDoc.substr(token.nL, (token.nR - token.nL + (((uint32_t)(token.nR) < m_strDoc.size()) ? 1 : 0))); +} + +int MarkupSTL::x_FindElem(int iPosParent, int iPos, const char *szPath) +{ + // If szPath is NULL or empty, go to next sibling element + // Otherwise go to next sibling element with matching path + // + /* + if ( iPos ) + iPos = m_aPos[iPos].iElemNext; + else + iPos = m_aPos[iPosParent].iElemChild; + + // Finished here if szPath not specified + if ( szPath == NULL || !szPath[0] ) + return iPos; + + // Search + TokenPos token( m_strDoc.c_str() ); + while ( iPos ) + { + // Compare tag name + token.nNext = m_aPos[iPos].nStartL + 1; + x_FindToken( token ); // Locate tag name + if ( token.Match(szPath) ) + return iPos; + iPos = m_aPos[iPos].iElemNext; + } + return 0; + */ + + //[CMARKUPDEV + // + // Example relative paths: + // + // "ITEM/QTY" first QTY child element of next ITEM element + // "ITEM/*" first child element of next ITEM element + // + // Example absolute paths: + // + // "/" root + // "/ORDER" root must be an ORDER element + // "/*/ITEM" first ITEM child element of root + // + int iPathOffset = 0; + if (szPath && szPath[0] == '/') + { + // Go to root + iPos = m_aPos[0].iElemChild; + iPosParent = 0; + ++iPathOffset; + } + else if (iPos) + iPos = m_aPos[iPos].iElemNext; + else + iPos = m_aPos[iPosParent].iElemChild; + + // Finished here if szPath not specified + if (szPath == NULL || !szPath[iPathOffset]) + return iPos; + + // Search + TokenPos token(m_strDoc.c_str()); + while (iPos) + { + // Compare tag name + token.nNext = m_aPos[iPos].nStartL + 1; + x_FindToken(token); // Locate tag name + if (szPath[iPathOffset] == '*') + { + // Wildcard + if (szPath[iPathOffset + 1] == '/') + { + iPathOffset += 2; + iPosParent = iPos; + iPos = m_aPos[iPosParent].iElemChild; + } + else + return iPos; + } + else if (token.Match(&szPath[iPathOffset])) + { + // Matched tag name + int nLen = token.nR - token.nL + 1; + if (szPath[iPathOffset + nLen] == '/') + { + iPathOffset += nLen + 1; + iPosParent = iPos; + iPos = m_aPos[iPosParent].iElemChild; + } + else + return iPos; + } + else + iPos = m_aPos[iPos].iElemNext; + } + return 0; + //]CMARKUPDEV +} + +int MarkupSTL::x_ParseNode(MarkupSTL::TokenPos &token) +{ + // Call this with token.nNext set to the start of the node + // This returns the node type and token.nNext set to the char after the node + // If the node is not found or an element, token.nR is not determined + // White space between elements is a text node + int nTypeFound = 0; + const char *szDoc = token.szDoc; + token.nL = token.nNext; + if (szDoc[token.nL] == '<') + { + // Started with <, could be: + // comment + // dtd + // processing instruction + // cdata section + // element + // + if (!szDoc[token.nL + 1] || !szDoc[token.nL + 2]) + return 0; + char cFirstChar = szDoc[token.nL + 1]; + const char *szEndOfNode = NULL; + if (cFirstChar == '?') + { + nTypeFound = MNT_PROCESSING_INSTRUCTION; // processing instruction + szEndOfNode = "?>"; + } + else if (cFirstChar == '!') + { + char cSecondChar = szDoc[token.nL + 2]; + if (cSecondChar == '[') + { + nTypeFound = MNT_CDATA_SECTION; + szEndOfNode = "]]>"; + } + else if (cSecondChar == '-') + { + nTypeFound = MNT_COMMENT; + szEndOfNode = "-->"; + } + else + { + // Document type requires tokenizing because of strings and brackets + nTypeFound = 0; + int nBrackets = 0; + while (x_FindToken(token)) + { + if (!token.bIsString) + { + char cChar = szDoc[token.nL]; + if (cChar == '[') + ++nBrackets; + else if (cChar == ']') + --nBrackets; + else if (nBrackets == 0 && cChar == '>') + { + nTypeFound = MNT_DOCUMENT_TYPE; + break; + } + } + } + if (!nTypeFound) + return 0; + } + } + else if (cFirstChar == '/') + { + // End tag means no node found within parent element + return 0; + } + else + { + nTypeFound = MNT_ELEMENT; + } + + // Search for end of node if not found yet + if (szEndOfNode) + { + const char *pEnd = strstr(&szDoc[token.nNext], szEndOfNode); + if (!pEnd) + return 0; // not well-formed + token.nNext = (pEnd - szDoc) + strlen(szEndOfNode); + } + } + else if (szDoc[token.nL]) + { + // It is text or whitespace because it did not start with < + nTypeFound = MNT_WHITESPACE; + if (x_FindToken(token)) + { + if (szDoc[token.nL] == '<') + token.nNext = token.nL; + else + { + nTypeFound = MNT_TEXT; + x_FindChar(token.szDoc, token.nNext, '<'); + } + } + } + return nTypeFound; +} + +string MarkupSTL::x_GetTagName(int iPos) const +{ + // Return the tag name at specified element + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[iPos].nStartL + 1; + if (!iPos || !x_FindToken(token)) + return ""; + + // Return substring of document + return x_GetToken(token); +} + +bool MarkupSTL::x_FindAttrib(MarkupSTL::TokenPos &token, const char *szAttrib) const +{ + // If szAttrib is NULL find next attrib, otherwise find named attrib + // Return true if found + int nAttrib = 0; + for (int nCount = 0; x_FindToken(token); ++nCount) + { + if (!token.bIsString) + { + // Is it the right angle bracket? + if (m_strDoc[token.nL] == '>' || m_strDoc[token.nL] == '/') + break; // attrib not found + + // Equal sign + if (m_strDoc[token.nL] == '=') + continue; + + // Potential attribute + if (!nAttrib && nCount) + { + // Attribute name search? + if (!szAttrib || !szAttrib[0]) + return true; // return with token at attrib name + + // Compare szAttrib + if (token.Match(szAttrib)) + nAttrib = nCount; + } + } + else if (nAttrib && nCount == nAttrib + 2) + { + return true; + } + } + + // Not found + return false; +} + +string MarkupSTL::x_GetAttrib(int iPos, const char *szAttrib) const +{ + // Return the value of the attrib at specified element + if (!iPos || m_nNodeType != MNT_ELEMENT) + return ""; + + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[iPos].nStartL + 1; + if (szAttrib && x_FindAttrib(token, szAttrib)) + return x_TextFromDoc(token.nL, token.nR - (((uint32_t)(token.nR) < m_strDoc.size()) ? 0 : 1)); + return ""; +} + +bool MarkupSTL::x_SetAttrib(int iPos, const char *szAttrib, int nValue) +{ + // Convert integer to string and call set_child_attrib + char szVal[25]; + snprintf(szVal, 24, "%d", nValue); + return x_SetAttrib(iPos, szAttrib, szVal); +} + +bool MarkupSTL::x_SetAttrib(int iPos, const char *szAttrib, const char *szValue) +{ + // Set attribute in iPos element + if (!iPos || m_nNodeType != MNT_ELEMENT) + return false; + + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[iPos].nStartL + 1; + int nInsertAt, nReplace = 0; + string strInsert; + if (x_FindAttrib(token, szAttrib)) + { + // Decision: for empty value leaving attrib="" instead of removing attrib + // Replace value only + strInsert = x_TextToDoc(szValue, true); + nInsertAt = token.nL; + nReplace = token.nR - token.nL + 1; + } + else + { + // Insert string name value pair + string strFormat; + strFormat = " "; + strFormat += szAttrib; + strFormat += "=\""; + strFormat += x_TextToDoc(szValue, true); + strFormat += "\""; + strInsert = strFormat; + + // take into account whether it is an empty element + nInsertAt = m_aPos[iPos].nStartR - (m_aPos[iPos].is_empty_element() ? 1 : 0); + } + + x_DocChange(nInsertAt, nReplace, strInsert); + int nAdjust = strInsert.size() - nReplace; + m_aPos[iPos].nStartR += nAdjust; + m_aPos[iPos].adjust_end(nAdjust); + x_Adjust(iPos, nAdjust); + MARKUP_SETDEBUGSTATE; + return true; +} + +//[CMARKUPDEV +bool MarkupSTL::x_RemoveAttrib(int iPos, const char *szAttrib) +{ + // Find attribute in iPos element and remove it + if (iPos && m_nNodeType == MNT_ELEMENT) + { + TokenPos token(m_strDoc.c_str()); + token.nNext = m_aPos[iPos].nStartL + 1; + while (x_FindAttrib(token)) + { + // Compare szAttrib + if (token.Match(szAttrib)) + { + int nInsertAt = token.nL - 1; // preceding whitespace + if (x_FindToken(token) && m_strDoc[token.nL] == '=' && x_FindToken(token) && token.bIsString) + { + int nReplace = token.nR - nInsertAt + 2; + x_DocChange(nInsertAt, nReplace, ""); + m_aPos[iPos].nStartR -= nReplace; + m_aPos[iPos].adjust_end(-nReplace); + x_Adjust(iPos, -nReplace); + MARKUP_SETDEBUGSTATE; + return true; + } + } + } + } + return false; +} +//]CMARKUPDEV + +bool MarkupSTL::x_CreateNode(string &strNode, int nNodeType, const char *szText) +{ + // Set strNode based on nNodeType and szText + // Return false if szText would jeopardize well-formed document + // + switch (nNodeType) + { + case MNT_CDATA_SECTION: + if (strstr(szText, "]]>") != NULL) + return false; + strNode = ""; + break; + //[CMARKUPDEV + case MNT_PROCESSING_INSTRUCTION: + strNode = ""; + break; + case MNT_COMMENT: + strNode = ""; + break; + case MNT_ELEMENT: + strNode = "<"; + strNode += szText; + strNode += "/>"; + break; + case MNT_TEXT: + case MNT_WHITESPACE: + strNode = x_TextToDoc(szText); + break; + case MNT_DOCUMENT_TYPE: + strNode = szText; + break; + //]CMARKUPDEV + } + return true; +} + +bool MarkupSTL::x_SetData(int iPos, const char *szData, int nCDATA) +{ + // Set data at specified position + // if nCDATA==1, set content of element to a CDATA Section + string strInsert; + + //[CMARKUPDEV + if (iPos == m_iPos && m_nNodeLength) + { + // Not an element + if (!x_CreateNode(strInsert, m_nNodeType, szData)) + return false; + x_DocChange(m_nNodeOffset, m_nNodeLength, strInsert); + x_AdjustForNode(m_iPosParent, iPos, strInsert.size() - m_nNodeLength); + m_nNodeLength = strInsert.size(); + MARKUP_SETDEBUGSTATE; + return true; + } + //]CMARKUPDEV + + // Set data in iPos element + if (!iPos || m_aPos[iPos].iElemChild) + return false; + + // Build strInsert from szData based on nCDATA + // If CDATA section not valid, use parsed text (PCDATA) instead + if (nCDATA != 0) + if (!x_CreateNode(strInsert, MNT_CDATA_SECTION, szData)) + nCDATA = 0; + if (nCDATA == 0) + strInsert = x_TextToDoc(szData); + + // Decide where to insert + int nInsertAt, nReplace; + if (m_aPos[iPos].is_empty_element()) + { + nInsertAt = m_aPos[iPos].nEndL; + nReplace = 1; + + // Pre-adjust since becomes data + string strTagName = x_GetTagName(iPos); + m_aPos[iPos].nStartR -= 1; + m_aPos[iPos].nEndL -= (1 + strTagName.size()); + string strFormat; + strFormat = ">"; + strFormat += strInsert; + strFormat += "", token.nNext); + if (nEndCDATA != (int32_t)(string::npos) && nEndCDATA < m_aPos[iPos].nEndL) + { + return m_strDoc.substr(token.nL + 9, nEndCDATA - token.nL - 9); + } + } + return x_TextFromDoc(m_aPos[iPos].nStartR + 1, m_aPos[iPos].nEndL - 1); + } + return ""; +} + +string MarkupSTL::x_TextToDoc(const char *szText, bool bAttrib) const +{ + // + // < less than + // & ampersand + // > greater than + // + // and for attributes: + // + // ' apostrophe or single quote + // " double quote + // + static const char *szaReplace[] = {"<", "&", ">", "'", """}; + const char *pFind = bAttrib ? "<&>\'\"" : "<&>"; + const char *pSource = szText; + string strResult; + int nLen = strlen(szText); + strResult.reserve(nLen + nLen / 10); + char cSource = *pSource; + char *pFound; + while (cSource) + { + if ((pFound = (char *)strchr(pFind, cSource)) != NULL) + { + pFound = (char *)szaReplace[pFound - pFind]; + strResult.append(pFound); + } + else + { + strResult += cSource; + } + cSource = *(++pSource); + } + return strResult; +} + +string MarkupSTL::x_TextFromDoc(int nLeft, int nRight) const +{ + // + // Conveniently the result is always the same or shorter in length + // + static const char *szaCode[] = {"lt;", "amp;", "gt;", "apos;", "quot;"}; + static int anCodeLen[] = {3, 4, 3, 5, 5}; + static const char *szSymbol = "<&>\'\""; + string strResult; + strResult.reserve(nRight - nLeft + 1); + const char *pSource = m_strDoc.c_str(); + int nChar = nLeft; + char cSource = pSource[nChar]; + while (nChar <= nRight) + { + if (cSource == '&') + { + // If no match is found it means XML doc is invalid + // no devastating harm done, ampersand code will just be left in result + for (int nMatch = 0; nMatch < 5; ++nMatch) + { + if (nChar <= nRight - anCodeLen[nMatch] && strncmp(szaCode[nMatch], &pSource[nChar + 1], anCodeLen[nMatch]) == 0) + { + cSource = szSymbol[nMatch]; + nChar += anCodeLen[nMatch]; + break; + } + } + } + strResult += cSource; + nChar++; + cSource = pSource[nChar]; + } + return strResult; +} + +void MarkupSTL::x_DocChange(int nLeft, int nReplace, const string &strInsert) +{ + // Insert strInsert int m_strDoc at nLeft replacing nReplace chars + // + //[CMARKUPDEV + // When creating a document, reduce reallocs by reserving string space + // Allow for 1.5 times the current allocation, with minimum of 200 + int nNewLength = strInsert.size() + m_strDoc.size() - nReplace; + int nAllocLen = m_strDoc.capacity(); + if (nNewLength > nAllocLen) + { + int nReserve = nAllocLen + nAllocLen / 2; + if (nReserve < nNewLength) + nReserve = nNewLength; + if (nReserve < 200) + nReserve = 200; + m_strDoc.reserve(nReserve); + } + //]CMARKUPDEV + m_strDoc.replace(nLeft, nReplace, strInsert); +} + +void MarkupSTL::x_Adjust(int iPos, int nShift, bool bAfterPos) +{ + // Loop through affected elements and adjust indexes + // Algorithm: + // 1. update children unless bAfterPos + // (if no children or bAfterPos is true, end tag of iPos not affected) + // 2. update next siblings and their children + // 3. go up until there is a next sibling of a parent and update end tags + // 4. step 2 + int iPosTop = m_aPos[iPos].iElemParent; + bool bPosFirst = bAfterPos; // mark as first to skip its children + while (iPos) + { + // Were we at containing parent of affected position? + bool bPosTop = false; + if (iPos == iPosTop) + { + // Move iPosTop up one towards root + iPosTop = m_aPos[iPos].iElemParent; + bPosTop = true; + } + + // Traverse to the next update position + if (!bPosTop && !bPosFirst && m_aPos[iPos].iElemChild) + { + // Depth first + iPos = m_aPos[iPos].iElemChild; + } + else if (m_aPos[iPos].iElemNext) + { + iPos = m_aPos[iPos].iElemNext; + } + else + { + // Look for next sibling of a parent of iPos + // When going back up, parents have already been done except iPosTop + while ((iPos = m_aPos[iPos].iElemParent) != 0 && iPos != iPosTop) + if (m_aPos[iPos].iElemNext) + { + iPos = m_aPos[iPos].iElemNext; + break; + } + } + bPosFirst = false; + + // Shift indexes at iPos + if (iPos != iPosTop) + m_aPos[iPos].adjust_start(nShift); + m_aPos[iPos].adjust_end(nShift); + } +} + +void MarkupSTL::x_LocateNew(int iPosParent, int &iPosRel, int &nOffset, int nLength, int nFlags) +{ + // Determine where to insert new element or node + // + bool bInsert = (nFlags & 1) ? true : false; + bool bHonorWhitespace = (nFlags & 2) ? true : false; + + int nStartL; + if (nLength) + { + // Located at a non-element node + if (bInsert) + nStartL = nOffset; + else + nStartL = nOffset + nLength; + } + else if (iPosRel) + { + // Located at an element + if (bInsert) // precede iPosRel + nStartL = m_aPos[iPosRel].nStartL; + else // follow iPosRel + nStartL = m_aPos[iPosRel].nEndR + 1; + } + else if (m_aPos[iPosParent].is_empty_element()) + { + // Parent has no separate end tag, so split empty element + nStartL = m_aPos[iPosParent].nStartR; + } + else + { + if (bInsert) // after start tag + nStartL = m_aPos[iPosParent].nStartR + 1; + else // before end tag + nStartL = m_aPos[iPosParent].nEndL; + } + + // Go up to start of next node, unless its splitting an empty element + if (!bHonorWhitespace && !m_aPos[iPosParent].is_empty_element()) + { + TokenPos token(m_strDoc.c_str()); + token.nNext = nStartL; + if (!x_FindToken(token) || m_strDoc[token.nL] == '<') + nStartL = token.nL; + } + + // Determine iPosBefore + int iPosBefore = 0; + if (iPosRel) + { + if (bInsert) + { + // Is iPosRel past first sibling? + int iPosPrev = m_aPos[iPosParent].iElemChild; + if (iPosPrev != iPosRel) + { + // Find previous sibling of iPosRel + while (m_aPos[iPosPrev].iElemNext != iPosRel) + iPosPrev = m_aPos[iPosPrev].iElemNext; + iPosBefore = iPosPrev; + } + } + else + { + iPosBefore = iPosRel; + } + } + else if (m_aPos[iPosParent].iElemChild) + { + if (!bInsert) + { + // Find last element under iPosParent + int iPosLast = m_aPos[iPosParent].iElemChild; + int iPosNext = iPosLast; + while (iPosNext) + { + iPosLast = iPosNext; + iPosNext = m_aPos[iPosNext].iElemNext; + } + iPosBefore = iPosLast; + } + } + + nOffset = nStartL; + iPosRel = iPosBefore; +} + +bool MarkupSTL::x_AddElem(const char *szName, const char *szValue, bool bInsert, bool bAddChild) +{ + if (bAddChild) + { + // Adding a child element under main position + if (!m_iPos) + return false; + } + else if (m_iPosParent == 0) + { + // Adding root element + if (is_well_formed()) + return false; + +//[CMARKUPDEV +#if defined(x_VERSIONTAG) + if (m_strDoc.is_empty()) + m_strDoc = x_VERSIONTAG; +#endif + //]CMARKUPDEV + + // Locate after any version and DTD + m_aPos[0].nEndL = m_strDoc.size(); + } + + // Locate where to add element relative to current node + int iPosParent, iPosBefore, nOffset = 0, nLength = 0; + if (bAddChild) + { + iPosParent = m_iPos; + iPosBefore = m_iPosChild; + } + else + { + iPosParent = m_iPosParent; + iPosBefore = m_iPos; + //[CMARKUPDEV + nOffset = m_nNodeOffset; + nLength = m_nNodeLength; + //]CMARKUPDEV + } + int nFlags = bInsert ? 1 : 0; + x_LocateNew(iPosParent, iPosBefore, nOffset, nLength, nFlags); + bool bEmptyParent = m_aPos[iPosParent].is_empty_element(); + if (bEmptyParent) + nOffset += 2; // include CRLF + + // Create element and modify positions of affected elements + // If no szValue is specified, an empty element is created + // i.e. either value or + // + int iPos = x_GetFreePos(); + m_aPos[iPos].nStartL = nOffset; + + // Set links + m_aPos[iPos].iElemParent = iPosParent; + m_aPos[iPos].iElemChild = 0; + m_aPos[iPos].iElemNext = 0; + if (iPosBefore) + { + // Link in after iPosBefore + m_aPos[iPos].iElemNext = m_aPos[iPosBefore].iElemNext; + m_aPos[iPosBefore].iElemNext = iPos; + } + else + { + // First child + m_aPos[iPos].iElemNext = m_aPos[iPosParent].iElemChild; + m_aPos[iPosParent].iElemChild = iPos; + } + + // Create string for insert + string strInsert; + int nLenName = strlen(szName); + int nLenValue = szValue ? strlen(szValue) : 0; + if (!nLenValue) + { + // empty element + strInsert = "<"; + strInsert += szName; + strInsert += "/>\r\n"; + m_aPos[iPos].nStartR = m_aPos[iPos].nStartL + nLenName + 2; + m_aPos[iPos].nEndL = m_aPos[iPos].nStartR - 1; + m_aPos[iPos].nEndR = m_aPos[iPos].nEndL + 1; + } + else + { + // value + string strValue = x_TextToDoc(szValue); + nLenValue = strValue.size(); + strInsert = "<"; + strInsert += szName; + strInsert += ">"; + strInsert += strValue; + strInsert += "\r\n"; + m_aPos[iPos].nStartR = m_aPos[iPos].nStartL + nLenName + 1; + m_aPos[iPos].nEndL = m_aPos[iPos].nStartR + nLenValue + 1; + m_aPos[iPos].nEndR = m_aPos[iPos].nEndL + nLenName + 2; + } + + // Insert + int nReplace = 0, nLeft = m_aPos[iPos].nStartL; + if (bEmptyParent) + { + string strParentTagName = x_GetTagName(iPosParent); + string strFormat; + strFormat = ">\r\n"; + strFormat += strInsert; + strFormat += " (len 4) becomes (len 11) + // In x_Adjust everything will be adjusted 11 - 4 = 7 + // But the nEndL of element A should only be adjusted 5 + m_aPos[iPosParent].nEndL -= (strParentTagName.size() + 1); + } + x_DocChange(nLeft, nReplace, strInsert); + x_Adjust(iPos, strInsert.size() - nReplace); + + if (bAddChild) + x_SetPos(m_iPosParent, iPosParent, iPos); + else + x_SetPos(iPosParent, iPos, 0); + return true; +} + +bool MarkupSTL::x_AddSubDoc(const char *szSubDoc, bool bInsert, bool bAddChild) +{ + // Add subdocument, parse, and modify positions of affected elements + // + int nOffset = 0, iPosParent, iPosBefore; + if (bAddChild) + { + // Add a subdocument under main position, after current child position + if (!m_iPos) + return false; + iPosParent = m_iPos; + iPosBefore = m_iPosChild; + } + else + { + iPosParent = m_iPosParent; + iPosBefore = m_iPos; + } + int nFlags = bInsert ? 1 : 0; + x_LocateNew(iPosParent, iPosBefore, nOffset, 0, nFlags); + bool bEmptyParent = m_aPos[iPosParent].is_empty_element(); + if (bEmptyParent) + nOffset += 2; // include CRLF + + // if iPosBefore is NULL, insert as first element under parent + int nParentEndLBeforeAdd = m_aPos[iPosParent].nEndL; + int iPosFreeBeforeAdd = m_iPosFree; + + // Skip version tag or DTD at start of subdocument + TokenPos token(szSubDoc); + int nNodeType = x_ParseNode(token); + while (nNodeType && nNodeType != MNT_ELEMENT) + { + token.szDoc = &szSubDoc[token.nNext]; + token.nNext = 0; + nNodeType = x_ParseNode(token); + } + string strInsert = token.szDoc; + + // Insert subdocument + m_aPos[iPosParent].nEndL = nOffset; + int nReplace = 0, nLeft = nOffset; + string strParentTagName; + if (bEmptyParent) + { + strParentTagName = x_GetTagName(iPosParent); + string strFormat; + strFormat = ">\r\n"; + strFormat += strInsert; + strFormat += ""; + if (bNewline) + strFormat += "\r\n"; + strFormat += strInsert; + strFormat += " +#include +#include +#include + +#ifdef _DEBUG +#define _DS(i) (i ? &(m_strDoc.c_str())[m_aPos[i].nStartL] : 0) +#define MARKUP_SETDEBUGSTATE \ + m_pMainDS = _DS(m_iPos); \ + m_pChildDS = _DS(m_iPosChild) +#else +#define MARKUP_SETDEBUGSTATE +#endif + +class MarkupSTL +{ +public: + MarkupSTL() { set_doc(NULL); }; + MarkupSTL(const char *szDoc) { set_doc(szDoc); }; + MarkupSTL(const MarkupSTL &markup) { *this = markup; }; + void operator=(const MarkupSTL &markup); + virtual ~MarkupSTL(){}; + + // Navigate + bool Load(const char *szFileName); + bool set_doc(const char *szDoc); + bool is_well_formed(); + bool find_elem(const char *szName = NULL); + bool find_child_elem(const char *szName = NULL); + bool into_elem(); + bool out_of_elem(); + void reset_child_pos() { x_SetPos(m_iPosParent, m_iPos, 0); }; + + void reset_main_pos() { x_SetPos(m_iPosParent, 0, 0); }; + void reset_pos() { x_SetPos(0, 0, 0); }; + std::string get_tag_name() const; + std::string get_child_tag_name() const { return x_GetTagName(m_iPosChild); }; + std::string get_data() const { return x_GetData(m_iPos); }; + std::string get_child_data() const { return x_GetData(m_iPosChild); }; + //[CMARKUPDEV + std::string find_get_data(const char *szName); + //]CMARKUPDEV + std::string get_attrib(const char *szAttrib) const { return x_GetAttrib(m_iPos, szAttrib); }; + std::string get_child_attrib(const char *szAttrib) const { return x_GetAttrib(m_iPosChild, szAttrib); }; + std::string get_attrib_name(int n) const; + bool save_pos(const char *szPosName = ""); + bool restore_pos(const char *szPosName = ""); + bool get_offsets(int &nStart, int &nEnd) const; + std::string get_error() const { return m_strError; }; + + enum MarkupNodeType + { + MNT_ELEMENT = 1, // 0x01 + MNT_TEXT = 2, // 0x02 + MNT_WHITESPACE = 4, // 0x04 + MNT_CDATA_SECTION = 8, // 0x08 + MNT_PROCESSING_INSTRUCTION = 16, // 0x10 + MNT_COMMENT = 32, // 0x20 + MNT_DOCUMENT_TYPE = 64, // 0x40 + MNT_EXCLUDE_WHITESPACE = 123, // 0x7b + }; + //[CMARKUPDEV + int find_node(int nType = 0); + int get_node_type() { return m_nNodeType; }; + bool add_node(int nType, const char *szText) { return x_AddNode(nType, szText, false); }; + bool insert_node(int nType, const char *szText) { return x_AddNode(nType, szText, true); }; + bool remove_node(); + //]CMARKUPDEV + + // Create + bool Save(const char *szFileName); + std::string get_doc() const { return m_strDoc; }; + bool add_elem(const char *szName, const char *szData = NULL) { return x_AddElem(szName, szData, false, false); }; + bool insert_elem(const char *szName, const char *szData = NULL) { return x_AddElem(szName, szData, true, false); }; + bool add_child_elem(const char *szName, const char *szData = NULL) { return x_AddElem(szName, szData, false, true); }; + bool insert_child_elem(const char *szName, const char *szData = NULL) { return x_AddElem(szName, szData, true, true); }; + bool add_attrib(const char *szAttrib, const char *szValue) { return x_SetAttrib(m_iPos, szAttrib, szValue); }; + bool add_child_attrib(const char *szAttrib, const char *szValue) { return x_SetAttrib(m_iPosChild, szAttrib, szValue); }; + bool add_attrib(const char *szAttrib, int nValue) { return x_SetAttrib(m_iPos, szAttrib, nValue); }; + bool add_child_attrib(const char *szAttrib, int nValue) { return x_SetAttrib(m_iPosChild, szAttrib, nValue); }; + bool add_child_sub_doc(const char *szSubDoc) { return x_AddSubDoc(szSubDoc, false, true); }; + bool insert_child_sub_doc(const char *szSubDoc) { return x_AddSubDoc(szSubDoc, true, true); }; + std::string get_child_sub_doc() const; + + // Modify + bool remove_elem(); + bool remove_child_elem(); + bool set_attrib(const char *szAttrib, const char *szValue) { return x_SetAttrib(m_iPos, szAttrib, szValue); }; + bool set_child_attrib(const char *szAttrib, const char *szValue) { return x_SetAttrib(m_iPosChild, szAttrib, szValue); }; + bool set_attrib(const char *szAttrib, int nValue) { return x_SetAttrib(m_iPos, szAttrib, nValue); }; + bool set_child_attrib(const char *szAttrib, int nValue) { return x_SetAttrib(m_iPosChild, szAttrib, nValue); }; + //[CMARKUPDEV + bool remove_attrib(const char *szAttrib) { return x_RemoveAttrib(m_iPos, szAttrib); }; + bool remove_child_attrib(const char *szAttrib) { return x_RemoveAttrib(m_iPosChild, szAttrib); }; + bool find_set_data(const char *szName, const char *szData, int nCDATA = 0); + //]CMARKUPDEV + bool set_data(const char *szData, int nCDATA = 0) { return x_SetData(m_iPos, szData, nCDATA); }; + bool set_child_data(const char *szData, int nCDATA = 0) { return x_SetData(m_iPosChild, szData, nCDATA); }; + + //[CMARKUPDEV + // Base64 + static std::string EncodeBase64(const unsigned char *pBuffer, int nBufferLen); + static int DecodeBase64(const std::string &strBase64, unsigned char *pBuffer, int nBufferLen); + //]CMARKUPDEV + +protected: +#ifdef _DEBUG + const char *m_pMainDS; + const char *m_pChildDS; +#endif + + std::string m_strDoc; + std::string m_strError; + + struct ElemPos + { + ElemPos() { Clear(); }; + ElemPos(const ElemPos &pos) { *this = pos; }; + bool is_empty_element() const { return (nStartR == nEndL + 1); }; + void Clear() + { + nStartL = 0; + nStartR = 0; + nEndL = 0; + nEndR = 0; + nReserved = 0; + iElemParent = 0; + iElemChild = 0; + iElemNext = 0; + }; + void adjust_start(int n) + { + nStartL += n; + nStartR += n; + }; + void adjust_end(int n) + { + nEndL += n; + nEndR += n; + }; + int nStartL; + int nStartR; + int nEndL; + int nEndR; + int nReserved; + int iElemParent; + int iElemChild; + int iElemNext; + }; + + typedef std::vector vectorElemPosT; + vectorElemPosT m_aPos; + int m_iPosParent; + int m_iPos; + int m_iPosChild; + int m_iPosFree; + int m_nNodeType; + //[CMARKUPDEV + int m_nNodeOffset; + int m_nNodeLength; + //]CMARKUPDEV + + struct TokenPos + { + TokenPos(const char *sz) + { + Clear(); + szDoc = sz; + }; + void Clear() + { + nL = 0; + nR = -1; + nNext = 0; + bIsString = false; + }; + bool Match(const char *szName) + { + int nLen = nR - nL + 1; +// To ignore case, define MARKUP_IGNORECASE +#ifdef MARKUP_IGNORECASE + return ((strnicmp(&szDoc[nL], szName, nLen) == 0) +#else + return ((strncmp(&szDoc[nL], szName, nLen) == 0) +#endif + && (szName[nLen] == '\0' || strchr(" =/[", szName[nLen]))); + }; + int nL; + int nR; + int nNext; + const char *szDoc; + bool bIsString; + }; + + struct SavedPos + { + int iPosParent; + int iPos; + int iPosChild; + }; + typedef std::map mapSavedPosT; + mapSavedPosT m_mapSavedPos; + + void x_SetPos(int iPosParent, int iPos, int iPosChild) + { + m_iPosParent = iPosParent; + m_iPos = iPos; + m_iPosChild = iPosChild; + m_nNodeType = iPos ? MNT_ELEMENT : 0; + //[CMARKUPDEV + m_nNodeOffset = 0; + m_nNodeLength = 0; + //]CMARKUPDEV + MARKUP_SETDEBUGSTATE; + }; + + int x_GetFreePos(); + int x_ReleasePos(); + + int x_ParseElem(int iPos); + int x_ParseError(const char *szError, const char *szName = NULL); + static bool x_FindChar(const char *szDoc, int &nChar, char c); + static bool x_FindToken(TokenPos &token); + std::string x_GetToken(const TokenPos &token) const; + int x_FindElem(int iPosParent, int iPos, const char *szPath); + std::string x_GetTagName(int iPos) const; + std::string x_GetData(int iPos) const; + std::string x_GetAttrib(int iPos, const char *szAttrib) const; + bool x_AddElem(const char *szName, const char *szValue, bool bInsert, bool bAddChild); + bool x_AddSubDoc(const char *szSubDoc, bool bInsert, bool bAddChild); + bool x_FindAttrib(TokenPos &token, const char *szAttrib = NULL) const; + bool x_SetAttrib(int iPos, const char *szAttrib, const char *szValue); + bool x_SetAttrib(int iPos, const char *szAttrib, int nValue); + //[CMARKUPDEV + bool x_RemoveAttrib(int iPos, const char *szAttrib); + bool x_AddNode(int nNodeType, const char *szText, bool bInsert); + void x_RemoveNode(int iPosParent, int &iPos, int &nNodeType, int &nNodeOffset, int &nNodeLength); + void x_AdjustForNode(int iPosParent, int iPos, int nShift); + //]CMARKUPDEV + bool x_CreateNode(std::string &strNode, int nNodeType, const char *szText); + void x_LocateNew(int iPosParent, int &iPosRel, int &nOffset, int nLength, int nFlags); + int x_ParseNode(TokenPos &token); + bool x_SetData(int iPos, const char *szData, int nCDATA); + int x_RemoveElem(int iPos); + void x_DocChange(int nLeft, int nReplace, const std::string &strInsert); + void x_PosInsert(int iPos, int nInsertLength); + void x_Adjust(int iPos, int nShift, bool bAfterPos = false); + std::string x_TextToDoc(const char *szText, bool bAttrib = false) const; + std::string x_TextFromDoc(int nLeft, int nRight) const; +}; + +#endif // !defined(AFX_MARKUPSTL_H__948A2705_9E68_11D2_A0BF_00105A27C570__INCLUDED_) diff --git a/src/search_local/index_storage/common/md5.c b/src/search_local/index_storage/common/md5.c new file mode 100644 index 0000000..2e24696 --- /dev/null +++ b/src/search_local/index_storage/common/md5.c @@ -0,0 +1,362 @@ +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +#ifdef _WIN32 +#define inline __inline +#endif +#define __USE_STRING_INLINES 1 +#include +#include +#include "md5.h" + +static void MD5Transform(uint32_t buf[4], uint32_t const in[16]); +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define byteReverse(x, y) /* */ +#else +static void byteReverse(unsigned char *buf, unsigned longs) +{ + uint32_t t; + do + { + t = (uint32_t)((unsigned)buf[3] << 8 | buf[2]) << 16 | + ((unsigned)buf[1] << 8 | buf[0]); + *(uint32_t *)buf = t; + buf += 4; + } while (--longs); +} +#endif + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +void MD5Init(struct MD5Context *ctx) +{ + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +void MD5Update(struct MD5Context *ctx, unsigned char const *buf, unsigned len) +{ + register uint32_t t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32_t)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if (t) + { + unsigned char *p = (unsigned char *)ctx->in + t; + + t = 64 - t; + if (len < t) + { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + buf += t; + len -= t; + } + /* Process data in 64-byte chunks */ + + while (len >= 64) + { + memcpy(ctx->in, buf, 64); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +void MD5Final(unsigned char digest[16], struct MD5Context *ctx) +{ + unsigned int count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) + { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } + else + { + /* Pad block to 56 bytes */ + memset(p, 0, count - 8); + } + byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + ((uint32_t *)ctx->in)[14] = ctx->bits[0]; + ((uint32_t *)ctx->in)[15] = ctx->bits[1]; + + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */ +} + +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + (w += f(x, y, z) + data, w = w << s | w >> (32 - s), w += x) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32_t buf[4], uint32_t const in[16]) +{ + register uint32_t a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +#if 0 +void MD5Digest( const unsigned char *msg, int len, unsigned char *digest) { + struct MD5Context ctx; + MD5Init (&ctx); + MD5Update (&ctx, msg, len); + MD5Final (digest, &ctx); +} + +void MD5DigestHex( const unsigned char *msg, int len, unsigned char *digest) { + struct MD5Context ctx; + unsigned char h[16]; + const char xdigit[] = "0123456789abcdef"; + int i; + + MD5Init (&ctx); + MD5Update (&ctx, msg, len); + MD5Final (h, &ctx); + for(i=0; i<16; i++) { + *digest++ = xdigit[h[i]>>4]; + *digest++ = xdigit[h[i]&0xf]; + } + *digest = '\0'; +} + +void MD5HMAC (const unsigned char *password, unsigned pass_len, + const unsigned char *challenge, unsigned chal_len, + unsigned char response[16]) +{ + int i; + unsigned char ipad[64]; + unsigned char opad[64]; + unsigned char hash_passwd[16]; + + struct MD5Context ctx; + + if (pass_len > sizeof (ipad)) + { + MD5Init (&ctx); + MD5Update (&ctx, password, pass_len); + MD5Final (hash_passwd, &ctx); + password = hash_passwd; pass_len = sizeof (hash_passwd); + } + + memset (ipad, 0, sizeof (ipad)); + memset (opad, 0, sizeof (opad)); + memcpy (ipad, password, pass_len); + memcpy (opad, password, pass_len); + + for (i=0; i<64; i++) { + ipad[i] ^= 0x36; + opad[i] ^= 0x5c; + } + + MD5Init (&ctx); + MD5Update (&ctx, ipad, sizeof (ipad)); + MD5Update (&ctx, challenge, chal_len); + MD5Final (response, &ctx); + + MD5Init (&ctx); + MD5Update (&ctx, opad, sizeof (opad)); + MD5Update (&ctx, response, 16); + MD5Final (response, &ctx); +} + +void MD5HMAC2 (const unsigned char *password, unsigned pass_len, + const unsigned char *challenge, unsigned chal_len, + const unsigned char *challenge2, unsigned chal_len2, + unsigned char response[16]) +{ + int i; + unsigned char ipad[64]; + unsigned char opad[64]; + unsigned char hash_passwd[16]; + + struct MD5Context ctx; + + if (pass_len > sizeof (ipad)) + { + MD5Init (&ctx); + MD5Update (&ctx, password, pass_len); + MD5Final (hash_passwd, &ctx); + password = hash_passwd; pass_len = sizeof (hash_passwd); + } + + memset (ipad, 0, sizeof (ipad)); + memset (opad, 0, sizeof (opad)); + memcpy (ipad, password, pass_len); + memcpy (opad, password, pass_len); + + for (i=0; i<64; i++) { + ipad[i] ^= 0x36; + opad[i] ^= 0x5c; + } + + MD5Init (&ctx); + MD5Update (&ctx, ipad, sizeof (ipad)); + MD5Update (&ctx, challenge, chal_len); + MD5Update (&ctx, challenge2, chal_len2); + MD5Final (response, &ctx); + + MD5Init (&ctx); + MD5Update (&ctx, opad, sizeof (opad)); + MD5Update (&ctx, response, 16); + MD5Final (response, &ctx); +} +#endif diff --git a/src/search_local/index_storage/common/md5.h b/src/search_local/index_storage/common/md5.h new file mode 100644 index 0000000..57d6c1c --- /dev/null +++ b/src/search_local/index_storage/common/md5.h @@ -0,0 +1,39 @@ +#ifndef _H_MD5_H +#define _H_MD5_H + +#include +#include +__BEGIN_DECLS + +#define MD5Init _TX_MD5Init_ +#define MD5Update _TX_MD5Update_ +#define MD5Final _TX_MD5Final_ +#define MD5Digest _TX_MD5Digest_ +#define MD5DigestHex _TX_MD5DigestHex_ +#define MD5HMAC _TX_MD5HMAC_ +#define MD5HMAC2 _TX_MD5HMAC2_ + +struct MD5Context +{ + uint32_t buf[4]; + uint32_t bits[2]; + uint8_t in[64]; +}; + +typedef struct MD5Context md5_t; + +extern void MD5Init(struct MD5Context *); +extern void MD5Update(struct MD5Context *, unsigned char const *, unsigned); +extern void MD5Final(unsigned char digest[16], struct MD5Context *ctx); +extern void MD5Digest(const unsigned char *msg, int len, unsigned char *digest); +extern void MD5DigestHex(const unsigned char *msg, int len, unsigned char *digest); +extern void MD5HMAC(const unsigned char *password, unsigned pass_len, + const unsigned char *challenge, unsigned chal_len, + unsigned char response[16]); +extern void MD5HMAC2(const unsigned char *password, unsigned pass_len, + const unsigned char *challenge, unsigned chal_len, + const unsigned char *challenge2, unsigned chal_len2, + unsigned char response[16]); + +__END_DECLS +#endif diff --git a/src/search_local/index_storage/common/mem_check.cc b/src/search_local/index_storage/common/mem_check.cc new file mode 100644 index 0000000..6985c22 --- /dev/null +++ b/src/search_local/index_storage/common/mem_check.cc @@ -0,0 +1,337 @@ +/* + * ===================================================================================== + * + * Filename: mem_check.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#if MEMCHECK +#include +#include "lock.h" +#include "log.h" + +#define HASHBASE 4097 + +static Mutex lock; + +struct location_t +{ + const char *ret; + int ln; +}; + +struct location_cmp +{ + bool operator()(const location_t &a, const location_t &b) const + { + return a.ret < b.ret || (a.ret == b.ret && a.ln < b.ln) ? true : false; + } +}; + +struct count_t +{ + unsigned long count; + unsigned long size; + void add(unsigned long s) + { + count++; + size += s; + } +}; + +struct slot_t +{ + struct slot_t *next; + void *ptr; + size_t size; + struct location_t loc; +}; + +static struct slot_t *freelist; +static struct slot_t *usedlist[HASHBASE]; +static int bypass = 1; +static unsigned long totalsize; +static unsigned long maxsize; + +void add_ptr(void *ptr, size_t size, const char *ret, int ln) +{ + struct slot_t *s = freelist; + if (s) + freelist = s->next; + else + s = (struct slot_t *)malloc(sizeof(struct slot_t)); + + unsigned n = ((unsigned long)ptr) % HASHBASE; + s->ptr = ptr; + s->size = size; + s->loc.ret = ret; + s->loc.ln = ln; + s->next = usedlist[n]; + usedlist[n] = s; + totalsize += size; + if (size >= 1 << 20) + log_debug("large memory allocated size=%ld", (long)size); + if (totalsize > maxsize) + maxsize = totalsize; +} + +void del_ptr(void *ptr) +{ + unsigned n = ((unsigned long)ptr) % HASHBASE; + struct slot_t **ps = &usedlist[n]; + while (*ps) + { + if ((*ps)->ptr == ptr) + { + struct slot_t *s = *ps; + totalsize -= s->size; + *ps = s->next; + s->next = freelist; + freelist = s; + return; + } + ps = &(*ps)->next; + } +} + +void enable_memchecker(void) +{ + bypass = 0; +} + +void dump_non_delete(void) +{ + bypass = 1; + + while (freelist) + { + struct slot_t *s = freelist; + freelist = freelist->next; + free(s); + } + + std::map m; + for (unsigned int n = 0; n < HASHBASE; n++) + { + while (usedlist[n]) + { + struct slot_t *s = usedlist[n]; + usedlist[n] = s->next; + m[s->loc].add(s->size); + free(s); + } + } + + for (std::map::iterator i = m.begin(); i != m.end(); i++) + { + const location_t &loc = i->first; + const count_t &val = i->second; + if (loc.ln == 0) + log_info("remain %ld %ld ret@%p", val.count, val.size, loc.ret); + else + log_info("remain %ld %ld %s(%d)", val.count, val.size, loc.ret, loc.ln); + } + log_info("Maximum Memory allocated: %lu\n", maxsize); +} + +void report_mallinfo(void) +{ + struct mallinfo mi = mallinfo(); + log_debug("mallinfo. arena: %d\n", mi.arena); + log_debug("mallinfo. ordblks: %d\n", mi.ordblks); + log_debug("mallinfo. smblks: %d\n", mi.smblks); + log_debug("mallinfo. hblks: %d\n", mi.hblks); + log_debug("mallinfo. usmblks: %d\n", mi.usmblks); + log_debug("mallinfo. fsmblks: %d\n", mi.fsmblks); + log_debug("mallinfo.uordblks: %d\n", mi.uordblks); + log_debug("mallinfo.fordblks: %d\n", mi.fordblks); + log_debug("mallinfo.keepcost: %d\n", mi.keepcost); +} + +unsigned long count_virtual_size(void) +{ + FILE *fp = fopen("/proc/self/maps", "r"); + char buf[256]; + buf[sizeof(buf) - 1] = '\0'; + unsigned long total = 0; + while (fgets(buf, sizeof(buf) - 1, fp) != NULL) + { + unsigned long start, end; + char r, w, x, p; + if (sscanf(buf, "%lx-%lx %c%c%c%c", &start, &end, &r, &w, &x, &p) != 6) + continue; + if (r == '-' && w == '-' && x == '-') + continue; + if (r != 'r') + continue; + if (p == 's') + continue; + total += end - start; + } + fclose(fp); + return total; +} + +unsigned long count_alloc_size(void) +{ + return totalsize; +} + +void *operator new(size_t size) +{ + const char *ret = (const char *)__builtin_return_address(0); + ScopedLock a(lock); + + void *p = malloc(size); + if (bypass == 0) + { + if (p == NULL) + log_error("ret@%p: operator new(%ld) failed", ret, (long)size); + else + add_ptr(p, size, ret, 0); + } + return p; +} + +void operator delete(void *p) +{ + ScopedLock a(lock); + + if (bypass == 0) + { + if (p) + del_ptr(p); + } + return free(p); +} + +void *operator new[](size_t size) +{ + const char *ret = (const char *)__builtin_return_address(0); + ScopedLock a(lock); + + void *p = malloc(size); + if (bypass == 0) + { + if (p == NULL) + log_error("ret@%p: operator new(%ld) failed", ret, (long)size); + else + add_ptr(p, size, ret, 0); + } + return p; +} + +void operator delete[](void *p) +{ + ScopedLock a(lock); + + if (bypass == 0) + { + if (p) + del_ptr(p); + } + return free(p); +} + +extern "C" void *malloc_debug(size_t size, const char *fn, int ln) +{ + ScopedLock a(lock); + + void *p = malloc(size); + if (bypass == 0) + { + if (p == NULL) + log_error("%s(%d): malloc(%ld) failed", fn, ln, (long)size); + else + add_ptr(p, size, fn, ln); + } + return p; +} + +extern "C" void *calloc_debug(size_t size, size_t nmem, const char *fn, int ln) +{ + ScopedLock a(lock); + + void *p = calloc(size, nmem); + if (bypass == 0) + { + if (p == NULL) + log_error("%s(%d): calloc(%ld, %ld) failed", fn, ln, (long)size, (long)nmem); + else + add_ptr(p, size * nmem, fn, ln); + } + return p; +} + +extern "C" void *realloc_debug(void *o, size_t size, const char *fn, int ln) +{ + ScopedLock a(lock); + + void *p = realloc(o, size); + if (bypass == 0) + { + if (p == NULL) + log_error("%s(%d): realloc(%p, %ld) failed", fn, ln, o, (long)size); + else + { + del_ptr(o); + add_ptr(p, size, fn, ln); + } + } + return p; +} + +extern "C" char *strdup_debug(const char *o, const char *fn, int ln) +{ + ScopedLock a(lock); + + char *p = strdup(o); + if (bypass == 0) + { + long size = strlen(o) + 1; + if (p == NULL) + log_error("%s(%d): strdup(%ld) failed", fn, ln, size); + else + add_ptr(p, size, fn, ln); + } + return p; +} + +extern "C" void free_debug(void *p, const char *fn, int lnp) +{ + ScopedLock a(lock); + + if (bypass == 0) + { + if (p) + del_ptr(p); + } + return free(p); +} + +#else +void *operator new(size_t size) +{ + return calloc(1, size); +} +void operator delete(void *p) +{ + return free(p); +} +void dump_non_delete(void) +{ +} +#endif diff --git a/src/search_local/index_storage/common/mem_check.h b/src/search_local/index_storage/common/mem_check.h new file mode 100644 index 0000000..ecdea72 --- /dev/null +++ b/src/search_local/index_storage/common/mem_check.h @@ -0,0 +1,161 @@ +/* + * ===================================================================================== + * + * Filename: mem_check.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_MEMCHEC____ +#define __H_MEMCHEC____ + +#include + +#if MEMCHECK +#define MALLOC(x) malloc_debug(x, __FILE__, __LINE__) +#define FREE(x) free_debug(x, __FILE__, __LINE__) +#define FREE_IF(x) \ + do \ + { \ + if ((x) != 0) \ + free_debug((void *)(x), __FILE__, __LINE__); \ + } while (0) +#define FREE_CLEAR(x) \ + do \ + { \ + if ((x) != 0) \ + { \ + free_debug((void *)(x), __FILE__, __LINE__); \ + (x) = 0; \ + } \ + } while (0) +#if ROCKSDB_COMPILER +#define REALLOC(p,sz) ({ void *a=realloc(p,sz); if(a) p = (decltype(p))a; a; }) +#else +#define REALLOC(p, sz) ({ void *a=realloc_debug(p,sz, __FILE__, __LINE__); if(a) p = (typeof(p))a; a; }) +#endif +#define CALLOC(x, y) calloc_debug(x, y, __FILE__, __LINE__) +#define STRDUP(x) strdup_debug(x, __FILE__, __LINE__) + +#if __cplusplus +extern "C" +{ +#endif + extern void *malloc_debug(size_t, const char *, int); + extern void free_debug(void *, const char *, int); + extern void *realloc_debug(void *, size_t, const char *, int); + extern void *calloc_debug(size_t, size_t, const char *, int); + extern char *strdup_debug(const char *, const char *, int); +#if __cplusplus +} +#endif +#else +#define MALLOC(x) malloc(x) +#define FREE(x) free(x) +#define FREE_IF(x) \ + do \ + { \ + if ((x) != 0) \ + free((void *)(x)); \ + } while (0) +#define FREE_CLEAR(x) \ + do \ + { \ + if ((x) != 0) \ + { \ + free((void *)(x)); \ + (x) = 0; \ + } \ + } while (0) +#if ROCKSDB_COMPILER +#define REALLOC(p, sz) ({ void *a=realloc(p,sz); if(a) p = (decltype(p))a; a; }) +#else +#define REALLOC(p, sz) ({ void *a=realloc(p,sz); if(a) p = (typeof(p))a; a; }) +#endif +#define CALLOC(x, y) calloc(x, y) +#define STRDUP(x) strdup(x) +#endif + +#if __cplusplus + +#if MEMCHECK +extern void enable_memchecker(void); +extern void dump_non_delete(void); +extern void report_mallinfo(void); +extern unsigned long count_virtual_size(void); +extern unsigned long count_alloc_size(void); +#else +static inline void enable_memchecker(void) +{ +} +static inline void dump_non_delete(void) {} +static inline void report_mallinfo(void) {} +#endif + +#define NEW(type, pointer) \ + do \ + { \ + try \ + { \ + pointer = 0; \ + pointer = new type; \ + } \ + catch (...) \ + { \ + pointer = 0; \ + } \ + } while (0) + +#define DELETE(pointer) \ + do \ + { \ + if (pointer) \ + { \ + delete pointer; \ + pointer = 0; \ + } \ + } while (0) + +#define DEC_DELETE(pointer) \ + do \ + { \ + if (pointer && pointer->DEC() == 0) \ + { \ + delete pointer; \ + pointer = 0; \ + } \ + } while (0) + +#define NEW_ARRAY(n, type, pointer) \ + do \ + { \ + try \ + { \ + pointer = 0; \ + pointer = new type[n]; \ + } \ + catch (...) \ + { \ + pointer = 0; \ + } \ + } while (0) + +#define DELETE_ARRAY(pointer) \ + do \ + { \ + delete[] pointer; \ + pointer = 0; \ + } while (0) + +#endif + +#endif diff --git a/src/search_local/index_storage/common/mtpqueue.h b/src/search_local/index_storage/common/mtpqueue.h new file mode 100644 index 0000000..f454819 --- /dev/null +++ b/src/search_local/index_storage/common/mtpqueue.h @@ -0,0 +1,122 @@ +#ifndef __PIPE_MTQUEUE_H__ +#define __PIPE_MTQUEUE_H__ + +#include +#include +#include "poller.h" +#include "lqueue.h" + +// typename T must be simple data type with 1,2,4,8,16 bytes +template +class ThreadingPipeQueue : PollerObject +{ +private: + typename LinkQueue::allocator alloc; + LinkQueue queue; + pthread_mutex_t lock; + int wakefd; + +private: + // lock management + inline void Lock(void) { pthread_mutex_lock(&lock); } + inline void Unlock(void) { pthread_mutex_unlock(&lock); } + + // pipe management + inline void Wake() + { + char c = 0; + write(wakefd, &c, 1); + } + inline void Discard() + { + char buf[256]; + int n; + do + { + n = read(netfd, buf, sizeof(buf)); + } while (n > 0); + } + + // reader implementation + virtual void hangup_notify(void) {} + virtual void input_notify(void) + { + T p; + int n = 0; + Lock(); + while (++n <= 64 && queue.Count() > 0) + { + p = queue.Pop(); + Unlock(); + // running task in unlocked mode + static_cast(this)->task_notify(p); + Lock(); + } + if (queue.Count() <= 0) + Discard(); + Unlock(); + } + +public: + ThreadingPipeQueue() : queue(&alloc), wakefd(-1) + { + pthread_mutex_init(&lock, NULL); + } + ~ThreadingPipeQueue() + { + pthread_mutex_destroy(&lock); + } + inline int attach_poller(PollerUnit *thread) + { + int fd[2]; + int ret = pipe(fd); + if (ret != 0) + return ret; + + wakefd = fd[1]; + netfd = fd[0]; + enable_input(); + return PollerObject::attach_poller(thread); + } + inline int Push(T p) + { + int qsz; + int ret; + + Lock(); + qsz = queue.Count(); + ret = queue.Push(p); + Unlock(); + if (qsz == 0) + Wake(); + return ret; + } + inline int Unshift(T p) + { + int qsz; + int ret; + + Lock(); + qsz = queue.Count(); + ret = queue.Unshift(p); + Unlock(); + if (qsz == 0) + Wake(); + return ret; + } + inline int Count(void) + { + int qsz; + + Lock(); + qsz = queue.Count(); + Unlock(); + return qsz; + } + inline int queue_empty(void) + { + return Count() == 0; + } +}; + +#endif diff --git a/src/search_local/index_storage/common/mtpqueue_nolock.h b/src/search_local/index_storage/common/mtpqueue_nolock.h new file mode 100644 index 0000000..d632922 --- /dev/null +++ b/src/search_local/index_storage/common/mtpqueue_nolock.h @@ -0,0 +1,189 @@ +/* + * ===================================================================================== + * + * Filename: mtpqueue_nolock.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __PIPE_MTQUEUE_NOLOCK_H__ +#define __PIPE_MTQUEUE_NOLOCK_H__ + +#include +#include +#include "poller.h" +#include "lqueue.h" +#include "LockFreeQueue.h" +#include "log.h" + +#define CAS(a_ptr, a_oldVal, a_newVal) __sync_bool_compare_and_swap(a_ptr, a_oldVal, a_newVal) + +/* + * 提供给业务类继承 + */ +class BaseTask +{ +private: + int syncflag; + int cmd; + +public: + BaseTask() : syncflag(0), cmd(0) + { + //do nothing + } + virtual ~BaseTask() + { + //do nothing + } + void set_sync_flag(int flag) + { + this->syncflag = flag; + } + int get_sync_flag() + { + return this->syncflag; + } + void set_cmd(int flag) + { + this->cmd = flag; + } + int get_cmd() + { + return this->cmd; + } +}; + +template +class ThreadingPipeNoLockQueue : PollerObject +{ +private: + LockFreeQueue queue; + int wakefd; + int resp_wakefd; + int resp_netfd; + volatile uint32_t queueSize; + +private: + // pipe management + inline void Wake() + { + char c = 0; + write(wakefd, &c, 1); + } + + inline void Response() + { + char c = 0; + write(resp_wakefd, &c, 1); + } + + inline void Discard() + { + char buf[1]; + int n; + n = read(netfd, buf, sizeof(buf)); + log_debug("the byte read from pipe is %d", n); + } + + // reader implementation + virtual void hangup_notify(void) + { + } + virtual void input_notify(void) + { + BaseTask *p; + int n = 0; + log_debug("in the input_notify the queue size is %d ", Count()); + while (++n <= 64 && queueSize > 0) + { + int ret = queue.de_queue(p); + if (true == ret) + { + __sync_fetch_and_sub(&queueSize, 1); + static_cast(this)->task_notify(p); + if (p->get_sync_flag() != 0) + { + Response(); + } + } + // running task in unlocked mode + } + log_debug("in the input_notify the queue size is %d ", Count()); + if (queueSize <= 0) + { + log_debug("Discard been called %d", Count()); + Discard(); + } + } + +public: + ThreadingPipeNoLockQueue() : wakefd(-1), resp_wakefd(-1), resp_netfd(-1) + { + } + ~ThreadingPipeNoLockQueue() + { + } + + inline int attach_poller(PollerUnit *thread) + { + int fd_response[2]; + int ret = pipe(fd_response); + if (ret != 0) + return ret; + resp_wakefd = fd_response[1]; + resp_netfd = fd_response[0]; + + int fd_send[2]; + ret = pipe(fd_send); + if (ret != 0) + return ret; + + wakefd = fd_send[1]; + netfd = fd_send[0]; + enable_input(); + return PollerObject::attach_poller(thread); + } + + inline int Push(BaseTask *p) + { + uint32_t qsz; + int ret; + ret = queue.en_queue(p); + if (true == ret) + { + qsz = __sync_fetch_and_add(&queueSize, 1); + if (qsz == 0) + Wake(); + } + if (p->get_sync_flag() != 0) + { + char buf[1]; + int n; + n = read(resp_netfd, buf, sizeof(buf)); + log_debug("resp from other thread!"); + } + return ret; + } + + inline int Count(void) + { + return queueSize; + } + + inline int queue_empty(void) + { + return Count() == 0; + } +}; + +#endif diff --git a/src/search_local/index_storage/common/multi_request.cc b/src/search_local/index_storage/common/multi_request.cc new file mode 100644 index 0000000..09ac95f --- /dev/null +++ b/src/search_local/index_storage/common/multi_request.cc @@ -0,0 +1,292 @@ +/* + * ===================================================================================== + * + * Filename: multi_request.cc + * + * Description: multi task requests. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "multi_request.h" +#include "task_request.h" +#include "task_multiplexer.h" +#include "key_list.h" +#include "mem_check.h" + +static MultiTaskReply multiTaskReply; + +MultiRequest::MultiRequest(TaskMultiplexer *o, TaskRequest *task) : owner(o), + wait(task), + keyList(NULL), + keyMask(NULL), + doneReq(0), + totalReq(0), + subReq(0), + firstPass(1), + keyFields(0), + internal(0) +{ +} + +MultiRequest::~MultiRequest() +{ + if (wait) + { + wait->reply_notify(); + wait = NULL; + } + if (internal == 0) + FREE_IF(keyList); + FREE_IF(keyMask); +} + +void MultiTaskReply::reply_notify(TaskRequest *cur) +{ + MultiRequest *req = cur->OwnerInfo(); + if (req == NULL) + delete cur; + else + req->complete_task(cur, cur->owner_index()); +} + +DTCValue *MultiRequest::get_key_value(int i) +{ + return &keyList[i * keyFields]; +} + +void MultiRequest::set_key_completed(int i) +{ + FD_SET(i, (fd_set *)keyMask); + doneReq++; +} + +int MultiRequest::is_key_completed(int i) +{ + return FD_ISSET(i, (fd_set *)keyMask); +} + +int MultiRequest::decode_key_list(void) +{ + if (!wait->flag_multi_key_val()) // single task + return 0; + + const DTCTableDefinition *tableDef = wait->table_definition(); + + keyFields = tableDef->key_fields(); + if (wait->internal_key_val_list()) + { + // embeded API + totalReq = wait->internal_key_val_list()->key_count(); + // Q&D discard const here + // this keyList member can't be const, + // but actually readonly after init + keyList = (DTCValue *)&wait->internal_key_val_list()->Value(0, 0); + internal = 1; + } + else + { + // from network + uint8_t fieldID[keyFields]; + Array keyNameList(*(wait->key_name_list())); + Array keyValList(*(wait->key_val_list())); + DTCBinary keyName; + for (int i = 0; i < keyFields; i++) + { + if (keyNameList.Get(keyName) != 0) + { + log_error("get key name[%d] error, key field count:%d", i, tableDef->key_fields()); + return -1; + } + fieldID[i] = tableDef->field_id(keyName.ptr); + } + if (keyNameList.Get(keyName) == 0) + { + log_error("bogus key name: %.*s", keyName.len, keyName.ptr); + return -1; + } + + totalReq = wait->versionInfo.get_tag(11)->u64; + keyList = (DTCValue *)MALLOC(totalReq * keyFields * sizeof(DTCValue)); + for (int i = 0; i < totalReq; i++) + { + DTCValue *keyVal = get_key_value(i); + for (int j = 0; j < keyFields; j++) + { + int fid = fieldID[j]; + switch (tableDef->field_type(fid)) + { + case DField::Signed: + case DField::Unsigned: + if (keyValList.Get(keyVal[fid].u64) != 0) + { + log_error("get key value[%d][%d] error", i, j); + return -2; + } + break; + + case DField::String: + case DField::Binary: + if (keyValList.Get(keyVal[fid].bin) != 0) + { + log_error("get key value[%d][%d] error", i, j); + return -2; + } + break; + + default: + log_error("invalid key type[%d][%d]", i, j); + return -3; + } + } + } + } + + // keyMask = (unsigned char *)CALLOC(1, (totalReq*keyFields+7)/8); + // 8 bytes aligned Awaste some memory. FD_SET operate memory by 8bytes + keyMask = (unsigned char *)CALLOC(8, (((totalReq * keyFields + 7) / 8) + 7) / 8); + return totalReq; +} + +int MultiRequest::split_task(void) +{ + log_debug("split_task begin, totalReq: %d", totalReq); + for (int i = 0; i < totalReq; i++) + { + if (is_key_completed(i)) + continue; + + DTCValue *keyVal = get_key_value(i); + TaskRequest *pTask = new TaskRequest; + if (pTask == NULL) + { + log_error("%s: %m", "new task error"); + return -1; + } + if (pTask->Copy(*wait, keyVal) < 0) + { + log_error("copy task error: %s", pTask->resultInfo.error_message()); + delete pTask; + return -1; + } + + pTask->set_owner_info(this, i, wait->OwnerAddress()); + pTask->push_reply_dispatcher(&multiTaskReply); + owner->push_task_queue(pTask); + subReq++; + } + + log_debug("split_task end, subReq: %d", subReq); + return 0; +} + +void MultiRequest::complete_task(TaskRequest *req, int index) +{ + log_debug("MultiRequest::complete_task start, index: %d", index); + if (wait) + { + if (wait->result_code() >= 0 && req->result_code() < 0) + { + wait->set_error_dup(req->resultInfo.result_code(), req->resultInfo.error_from(), req->resultInfo.error_message()); + } + + int ret; + if ((ret = wait->merge_result(*req)) != 0) + { + wait->set_error(ret, "multi_request", "merge result error"); + } + } + + delete req; + + set_key_completed(index); + subReq--; + + // 注意，如果将CTaskMultiplexer放到cache线程执行，则会导致每split一个task，都是直接到cache_process执行完到这里；然后再split出第二个task。这会导致这一个判断逻辑有问题。 + // 目前CTaskMultiplexer是跟incoming线程绑在一起的，因此没有问题 + if (firstPass == 0 && subReq == 0) + { + complete_waiter(); + delete this; + } + log_debug("MultiRequest::complete_task end, subReq: %d", subReq); +} + +void MultiRequest::complete_waiter(void) +{ + if (wait) + { + wait->reply_notify(); + wait = 0; + } +} + +void MultiRequest::second_pass(int err) +{ + firstPass = 0; + if (subReq == 0) + { + // no sub-request present, complete whole request + complete_waiter(); + delete this; + } + else if (err) + { + // mark all request is done except sub-requests + doneReq = totalReq - subReq; + complete_waiter(); + } + return; +} + +int TaskRequest::set_batch_cursor(int index) +{ + int err = 0; + + MultiRequest *mreq = get_batch_key_list(); + if (mreq == NULL) + return -1; + + if (index < 0 || index >= mreq->total_count()) + { + key = NULL; + multiKey = NULL; + return -1; + } + else + { + DTCValue *keyVal = mreq->get_key_value(index); + int kf = table_definition()->key_fields(); + + /* switch request_key() */ + key = &keyVal[0]; + if (kf > 1) + { + /* switch multi-fields key */ + multiKey = keyVal; + } + err = build_packed_key(); + if (err < 0) + { + log_error("build packed key error, error from: %s, error message: %s", resultInfo.error_from(), resultInfo.error_message()); + return -1; + } + } + return 0; +} + +void TaskRequest::done_batch_cursor(int index) +{ + MultiRequest *mreq = get_batch_key_list(); + if (mreq == NULL) + return; + + mreq->set_key_completed(index); +} diff --git a/src/search_local/index_storage/common/multi_request.h b/src/search_local/index_storage/common/multi_request.h new file mode 100644 index 0000000..23a7fdc --- /dev/null +++ b/src/search_local/index_storage/common/multi_request.h @@ -0,0 +1,67 @@ +/* + * ===================================================================================== + * + * Filename: multi_request.h + * + * Description: multi task requests. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_MULTI_REQUEST_H__ +#define __H_MULTI_REQUEST_H__ + +#include "request_base_all.h" + +class TaskMultiplexer; +class TaskRequest; +union DTCValue; + +class MultiRequest +{ +private: + TaskMultiplexer *owner; + TaskRequest *wait; + DTCValue *keyList; + unsigned char *keyMask; + int doneReq; + int totalReq; + int subReq; + int firstPass; + int keyFields; + int internal; + +public: + friend class TaskMultiplexer; + MultiRequest(TaskMultiplexer *o, TaskRequest *task); + ~MultiRequest(void); + + int decode_key_list(void); + int split_task(void); + int total_count(void) const { return totalReq; } + int remain_count(void) const { return totalReq - doneReq; } + void second_pass(int err); + void complete_task(TaskRequest *req, int index); + +public: + DTCValue *get_key_value(int i); + void set_key_completed(int i); + int is_key_completed(int i); + void complete_waiter(void); +}; + +class MultiTaskReply : public ReplyDispatcher +{ +public: + MultiTaskReply() {} + virtual void reply_notify(TaskRequest *cur); +}; + +#endif diff --git a/src/search_local/index_storage/common/myepoll.h b/src/search_local/index_storage/common/myepoll.h new file mode 100644 index 0000000..d363fe0 --- /dev/null +++ b/src/search_local/index_storage/common/myepoll.h @@ -0,0 +1,149 @@ +/* + * ===================================================================================== + * + * Filename: myepoll.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _SYS_EPOLL_H_ +#define _SYS_EPOLL_H_ + +#if __x86_64__ +#define _SYS_EPOLL_ +#endif + +#ifdef _SYS_EPOLL_ +#include +#else + +#include +#include +#include +#include +#include +#include + +/* Valid opcodes to issue to sys_epoll_ctl() */ +#ifndef EPOLL_CTL_ADD +#define EPOLL_CTL_ADD 1 +#endif +#ifndef EPOLL_CTL_DEL +#define EPOLL_CTL_DEL 2 +#endif +#ifndef EPOLL_CTL_MOD +#define EPOLL_CTL_MOD 3 +#endif + +/* These are specified by iBCS2 */ +#if defined(POLLIN) && !defined(EPOLLIN) +#define EPOLLIN POLLIN +#endif +#if defined(POLLPRI) && !defined(EPOLLPRI) +#define EPOLLPRI POLLPRI +#endif +#if defined(POLLOUT) && !defined(EPOLLOUT) +#define EPOLLOUT POLLOUT +#endif +#if defined(POLLERR) && !defined(EPOLLERR) +#define EPOLLERR POLLERR +#endif +#if defined(POLLHUP) && !defined(EPOLLHUP) +#define EPOLLHUP POLLHUP +#endif +#if defined(POLLNVAL) && !defined(EPOLLNVAL) +#define EPOLLNVAL POLLNVAL +#endif +#if defined(POLLRDNORM) && !defined(EPOLLRDNORM) +#define EPOLLRDNORM POLLRDNORM +#endif +#if defined(POLLRDBAND) && !defined(EPOLLRDBAND) +#define EPOLLRDBAND POLLRDBAND +#endif +#if defined(POLLWRNORM) && !defined(EPOLLWRNORM) +#define EPOLLWRNORM POLLWRNORM +#endif +#if defined(POLLWRBAND) && !defined(EPOLLWRBAND) +#define EPOLLWRBAND POLLWRBAND +#endif +#if defined(POLLMSG) && !defined(EPOLLMSG) +#define EPOLLMSG POLLMSG +#endif + +/* Set the One Shot behaviour for the target file descriptor */ +#ifndef EPOLLONESHOT +#define EPOLLONESHOT (1 << 30) +#endif + +/* Set the Edge Triggered behaviour for the target file descriptor */ +#ifndef EPOLLET +#define EPOLLET (1 << 31) +#endif + +/* + * On x86-64 make the 64bit structure have the same alignment as the + * 32bit structure. This makes 32bit emulation easier. + */ +#ifdef __x86_64__ +#define EPOLL_PACKED __attribute__((packed)) +#else +#define EPOLL_PACKED +#endif + +struct epoll_event +{ + uint32_t events; + union + { + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; + } data; +} EPOLL_PACKED; + +#ifndef __NR_epoll_create +#ifdef __x86_64__ +#define __NR_epoll_create 213 +#else +#define __NR_epoll_create 254 +#endif +#endif +#ifndef __NR_epoll_ctl +#define __NR_epoll_ctl (__NR_epoll_create + 1) +#endif +#ifndef __NR_epoll_wait +#define __NR_epoll_wait (__NR_epoll_create + 2) +#endif + +#ifdef __EPOLL_SYSCALLS__ +static inline _syscall1(int, epoll_create, int, maxfds); +static inline _syscall4(int, epoll_ctl, int, epfd, int, op, int, fd, struct epoll_event *, event); +static inline _syscall4(int, epoll_wait, int, epfd, struct epoll_event *, events, int, maxevents, int, timeout); +#else +static inline int epoll_create(int maxfds) +{ + return syscall(__NR_epoll_create, maxfds); +} + +static inline int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) +{ + return syscall(__NR_epoll_ctl, epfd, op, fd, event); +} +static inline int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) +{ + return syscall(__NR_epoll_wait, epfd, events, maxevents, timeout); +} +#endif + +#endif +#endif diff --git a/src/search_local/index_storage/common/namespace.h b/src/search_local/index_storage/common/namespace.h new file mode 100644 index 0000000..55c0c39 --- /dev/null +++ b/src/search_local/index_storage/common/namespace.h @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: namespace.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_VERSIONED_NAMESPACE_H +#define __DTC_VERSIONED_NAMESPACE_H + +#include "version.h" + +#if DTC_HAS_VERSIONED_NAMESPACE && DTC_HAS_VERSIONED_NAMESPACE == 1 + +#ifndef DTC_VERSIONED_NAMESPACE + +#define MAKE_DTC_VERSIONED_NAMESPACE_IMPL(MAJOR, MINOR, BETA) DTC_##MAJOR##_##MINOR##_##BETA +#define MAKE_DTC_VERSIONED_NAMESPACE(MAJOR, MINOR, BETA) MAKE_DTC_VERSIONED_NAMESPACE_IMPL(MAJOR, MINOR, BETA) +#define DTC_VERSIONED_NAMESPACE MAKE_DTC_VERSIONED_NAMESPACE(DTC_MAJOR_VERSION, DTC_MINOR_VERSION, DTC_BETA_VERSION) + +#endif //end DTC_VERSIONED_NAMESPACE + +#define DTC_BEGIN_NAMESPACE \ + namespace DTC_VERSIONED_NAMESPACE \ + { +#define DTC_END_NAMESPACE } +#define DTC_USING_NAMESPACE using namespace DTC_VERSIONED_NAMESPACE; + +#else + +#define DTC_BEGIN_NAMESPACE +#define DTC_END_NAMESPACE +#define DTC_USING_NAMESPACE + +#endif //end DTC_HAS_VERSIONED_NAMESPACE + +#endif diff --git a/src/search_local/index_storage/common/net_addr.cc b/src/search_local/index_storage/common/net_addr.cc new file mode 100644 index 0000000..043b811 --- /dev/null +++ b/src/search_local/index_storage/common/net_addr.cc @@ -0,0 +1,906 @@ +/* + * ===================================================================================== + * + * Filename: sock_addr.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "net_addr.h" + +static inline int has2colon(const char *name) +{ + char c; + while ((c = *name++) != '\0') + { + if (c == ':') + { + while ((c = *name++) != '\0') + { + if (c == ':') + return 1; + } + return 0; + } + } + return 0; +} + +static unsigned short name2port(const char *name, int type) +{ + const char *proto = type == SOCK_STREAM ? "tcp" : "udp"; + char buf[1024]; + struct servent result_buf; + struct servent *result = NULL; + const char *p = strchr(name, '/'); + if (p) + { + int len = p - name; + char localname[len + 1]; + memcpy(localname, name, len); + localname[len] = 0; + getservbyname_r(localname, proto, &result_buf, buf, sizeof(buf), &result); + } + else + { + getservbyname_r(name, proto, &result_buf, buf, sizeof(buf), &result); + } + if (result == NULL) + return 0; + return ntohs(result_buf.s_port); +} + +static unsigned int ip2addr(const char *name) +{ + if (name[0] == '*') + return 0; + + struct in_addr addr; + const char *p = strrchr(name, ':'); + if (p == NULL) + p = strchr(name, '/'); + if (p == NULL) + { + if (inet_pton(AF_INET, name, &addr) <= 0) + return INADDR_NONE; + } + else + { + int len = p - name; + char localname[len + 1]; + memcpy(localname, name, len); + localname[len] = 0; + if (inet_pton(AF_INET, localname, &addr) <= 0) + return INADDR_NONE; + } + + unsigned int v = ntohl(addr.s_addr); + if (v < 0x0100000) + return INADDR_NONE; + + return addr.s_addr; +} + +static pthread_mutex_t lifr = PTHREAD_MUTEX_INITIALIZER; +static int nifr = 0; +struct ifreq *volatile ifr = NULL; + +// clear interface information cache +// if module unloaded +__attribute__((__destructor__)) static void clean_ifr_cache(void) +{ + if (ifr != NULL) + { + free(ifr); + ifr = NULL; + } +} + +static int getintflist(void) +{ + if (ifr != NULL) + return 0; + pthread_mutex_lock(&lifr); + if (ifr == NULL) + { + int fd = socket(AF_INET, SOCK_DGRAM, 0); + if (fd >= 0) + { + struct ifconf ifc; + + ifc.ifc_len = 0; + ifc.ifc_req = NULL; + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + { + ifr = (struct ifreq *)malloc(ifc.ifc_len > 128 ? ifc.ifc_len : 128); + ifc.ifc_req = ifr; + if (ioctl(fd, SIOCGIFCONF, &ifc) == 0) + nifr = ifc.ifc_len / sizeof(struct ifreq); + } + close(fd); + } + } + pthread_mutex_unlock(&lifr); + return ifr == NULL ? -1 : 0; +} + +static int is_local_ipv4(uint32_t addr) +{ + if (addr == 0) + return 1; +#if __BYTE_ORDER == __BIG_ENDIAN + if ((addr >> 24) == 127) + return 1; +#elif __BYTE_ORDER == __BIG_ENDIAN + if ((addr & 0xFF) == 127) + return 1; +#endif + + if (getintflist() < 0) + return 0; + + int i; + for (i = 0; i < nifr; i++) + { + if (ifr[i].ifr_addr.sa_family == AF_INET) + { + if (addr == ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr) + return 1; + } + } + + return 0; +} + +static int intf2addr(const char *name, uint32_t addr[4]) +{ + if (getintflist() < 0) + return 0; + + int i; + for (i = 0; i < nifr; i++) + { + if (strncmp(ifr[i].ifr_name, name, sizeof(ifr[i].ifr_name)) != 0) + continue; + + if (ifr[i].ifr_addr.sa_family == AF_INET) + { + addr[0] = ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr; + return AF_INET; + } + // never happend, SIOCGIFCONF only applicable to IPv4 + if (ifr[i].ifr_addr.sa_family == AF_INET6) + { + memcpy(addr, ((struct sockaddr_in6 *)&ifr[i].ifr_addr)->sin6_addr.s6_addr, 16); + return AF_INET6; + } + } + + return 0; +} + +static int host2addr(const char *name, uint32_t addr[4]) +{ + char buf[1024]; + struct hostent ret; + struct hostent *result = NULL; + int err; + + gethostbyname_r(name, &ret, buf, sizeof(buf), &result, &err); + if (result != NULL) + { + if (ret.h_addrtype == AF_INET && ret.h_length == 4) + { + memcpy(addr, ret.h_addr, 4); + return AF_INET; + } + else if (ret.h_addrtype == AF_INET6 && ret.h_length == 16) + { + memcpy(addr, ret.h_addr, 16); + return AF_INET6; + } + } + return 0; +} + +static int name2addr(const char *name, uint32_t addr[4]) +{ + int family; + + family = intf2addr(name, addr); + if (family != 0) + return family; + + family = host2addr(name, addr); + if (family != 0) + return family; + + return family; +} + +void SocketAddress::build_name_none(void) +{ + sockname[0] = 0; +} + +void SocketAddress::build_name_unix(void) +{ + snprintf(sockname, sizeof(sockname), "%c%.*s", + un->sun_path[0] ?: '@', (int)sizeof(un->sun_path) - 1, un->sun_path + 1); +} + +void SocketAddress::build_name_ipv4(void) +{ + char ip[32]; + if (in4->sin_addr.s_addr == 0) + { + // wild address is * + ip[0] = '*'; + ip[1] = '\0'; + } + else + { + inet_ntop(AF_INET, &in4->sin_addr, ip, sizeof(ip)); + } + snprintf(sockname, sizeof(sockname), "%s:%d/%s", ip, ntohs(in4->sin_port), socktype == SOCK_STREAM ? "tcp" : "udp"); +} + +void SocketAddress::build_name_ipv6(void) +{ + char ip[64]; + if (IN6_IS_ADDR_UNSPECIFIED(this->in6->sin6_addr.s6_addr32)) + { + // wild address is :: + ip[0] = ':'; + ip[1] = ':'; + ip[2] = '\0'; + } + else + { + inet_ntop(AF_INET6, &in6->sin6_addr, ip, sizeof(ip)); + } + snprintf(sockname, sizeof(sockname), "%s:%d/%s", ip, ntohs(in6->sin6_port), socktype == SOCK_STREAM ? "tcp" : "udp"); +} + +void SocketAddress::build_name_bad(void) +{ + snprintf(sockname, sizeof(sockname), ""); +} + +void SocketAddress::build_name(void) +{ + switch (addr->sa_family) + { + case 0: + build_name_none(); + break; + + case AF_UNIX: + build_name_unix(); + break; + + case AF_INET: + build_name_ipv4(); + break; + + case AF_INET6: + build_name_ipv6(); + break; + + default: + build_name_bad(); + break; + } +} + +const char *SocketAddress::set_address(const char *name, int port, int type) +{ + if (is_unix_socket_path(name)) + { + if (port != 0) + return "UNIX socket path hasnot a port"; + return set_unix_address(name, type); + } + + if (has2colon(name)) + return set_ipv6_address(name, port, type); + + if (name[0] == '*' || isdigit(name[0])) + return set_ipv4_address(name, port, type); + + return set_host_address(name, port, type); +} + +const char *SocketAddress::set_address(const char *name, const char *portstr) +{ + if (is_unix_socket_path(name)) + { + if (portstr != NULL && portstr[0] != '\0') + return "UNIX socket path hasnot a port"; + return set_unix_address(name); + } + + int port = 0; + int type = SOCK_STREAM; + if (portstr != NULL && portstr[0] != '\0') + { + port = isdigit(portstr[0]) ? atoi(portstr) : name2port(portstr, SOCK_STREAM); + if (port == 0) + return "Invalid TCP/UDP port"; + const char *p = strrchr(portstr, '/'); + if (p != NULL) + { + if (!strcmp(p + 1, "udp")) + { + type = SOCK_DGRAM; + } + else if (!strcmp(p + 1, "tcp")) + { + type = SOCK_STREAM; + } + else + { + return "Invalid protocol name"; + } + } + } + + if (has2colon(name)) + return set_ipv6_address(name, port, type); + + if (name[0] == '*' || isdigit(name[0])) + return set_ipv4_address(name, port, type); + + return set_host_address(name, port, type); +} + +const char *SocketAddress::set_address(const struct sockaddr_un *addr, int len, int type) +{ + if (type != SOCK_STREAM && type != SOCK_DGRAM) + { + return "Invalid protocol type\n"; + } + socktype = type; + alen = len; + memcpy(un, addr, len); + build_name_unix(); + return NULL; +} + +const char *SocketAddress::set_address(const struct sockaddr_in *addr, int len, int type) +{ + if (type != SOCK_STREAM && type != SOCK_DGRAM) + { + return "Invalid protocol type\n"; + } + if (len < (int)sizeof(struct sockaddr_in)) + { + return "Invalid socklen\n"; + } + socktype = type; + alen = sizeof(struct sockaddr_in); + *in4 = *addr; + build_name_ipv4(); + return NULL; +} + +const char *SocketAddress::set_address(const struct sockaddr_in6 *addr, int len, int type) +{ + if (type != SOCK_STREAM && type != SOCK_DGRAM) + { + return "Invalid protocol type\n"; + } + if (len < (int)sizeof(struct sockaddr_in6)) + { + return "Invalid socklen\n"; + } + socktype = type; + alen = sizeof(struct sockaddr_in6); + *in6 = *addr; + build_name_ipv6(); + return NULL; +} + +const char *SocketAddress::set_address(const struct sockaddr *addr, int len, int type) +{ + switch (addr->sa_family) + { + case AF_UNIX: + return set_address((const sockaddr_un *)addr, len, type); + case AF_INET: + return set_address((const sockaddr_in *)addr, len, type); + case AF_INET6: + return set_address((const sockaddr_in6 *)addr, len, type); + } + return "Unsupported socket family"; +} + +const char *SocketAddress::set_unix_address(const char *name, int type) +{ + int namelen = strlen(name); + if (namelen >= (int)sizeof(this->un->sun_path)) + return "UNIX socket path name too long"; + this->alen = init_unix_socket_address(this->un, name); + if (namelen >= 4 && !strcmp(name + namelen - 4, "/udp")) + this->socktype = SOCK_DGRAM; + else if (namelen >= 4 && !strcmp(name + namelen - 4, "-udp")) + this->socktype = SOCK_DGRAM; + else if (namelen >= 6 && !strcmp(name + namelen - 6, "/dgram")) + this->socktype = SOCK_DGRAM; + else if (namelen >= 6 && !strcmp(name + namelen - 6, "-dgram")) + this->socktype = SOCK_DGRAM; + else + this->socktype = type; + build_name_unix(); + return NULL; // SUCC +} + +const char *SocketAddress::set_ipv4_address(const char *name, int port, int type) +{ + const char *p = NULL; + + if ((p = strchr(name, '/')) != NULL) + { + p++; + if (!strcmp(p, "udp")) + type = SOCK_DGRAM; + else if (!strcmp(p, "tcp")) + type = SOCK_STREAM; + else + return "Unknown IPv4 protocol"; + } + + if ((p = strrchr(name, ':')) != NULL) + { + p++; + if (strchr(p, '.')) + return "Invalid TCP/UDP port"; + port = isdigit(p[0]) ? atoi(p) : name2port(p, type); + if (port == 0) + return "Invalid TCP/UDP port"; + } + + if (port == 0) + port = 8888; + else if (port < 0 || port >= 65536) + return "TCP/UDP port must between 1-65535"; + + unsigned int ipv4 = ip2addr(name); + if (ipv4 == INADDR_NONE) + return "Invalid IPv4 address"; + + socktype = type; + alen = sizeof(struct sockaddr_in); + memset(in4, 0, alen); + in4->sin_family = AF_INET; + in4->sin_addr.s_addr = ipv4; + in4->sin_port = htons(port); + build_name_ipv4(); + return NULL; +} + +const char *SocketAddress::set_ipv6_address(const char *name, int port, int type) +{ + const char *p = NULL; + + if ((p = strchr(name, '/')) != NULL) + { + p++; + if (!strcmp(p, "udp")) + type = SOCK_DGRAM; + else if (!strcmp(p, "tcp")) + type = SOCK_STREAM; + else + return "Unknown IPv6 protocol"; + } + + if (port == 0) + { + // always has 2 colon, seek to right most colon + p = strrchr(name, ':') + 1; + if (strchr(p, '.')) + return "Invalid TCP/UDP port"; + port = isdigit(p[0]) ? atoi(p) : name2port(p, type); + if (port == 0) + return "Invalid TCP/UDP port"; + } + + if (port == 0) + port = 8888; + else if (port < 0 || port >= 65536) + return "TCP/UDP port must between 1-65535"; + + char addr[16]; + // IPv6 has not '*', use '::' instead + + if (p) + { + // p pointer to : or / + + // make a new copy of name, strip last colon + // colon is a valid seperator in IPv6 numeric address + int len = p - name; + char localname[len]; + memcpy(localname, name, len); + localname[len - 1] = 0; + if (inet_pton(AF_INET6, localname, &addr) <= 0) + return "Invalid IPv6 address"; + } + else + { + if (inet_pton(AF_INET6, name, &addr) <= 0) + return "Invalid IPv6 address"; + } + + socktype = type; + alen = sizeof(struct sockaddr_in6); + memset(in6, 0, alen); + in6->sin6_family = AF_INET6; + memcpy(in6->sin6_addr.s6_addr, addr, 16); + in6->sin6_port = htons(port); + build_name_ipv6(); + return NULL; +} + +const char *SocketAddress::set_host_address(const char *name, int port, int type) +{ + const char *p; + + if ((p = strchr(name, '/')) != NULL) + { + p++; + if (!strcmp(p, "udp")) + type = SOCK_DGRAM; + else if (!strcmp(p, "tcp")) + type = SOCK_STREAM; + else + return "Unknown IPv4/IPv6 protocol"; + } + + if ((p = strrchr(name, ':')) != NULL) + { + p++; + if (strchr(p, '.')) + return "Invalid TCP/UDP port"; + port = isdigit(p[0]) ? atoi(p) : name2port(p, type); + if (port == 0) + return "Invalid TCP/UDP port"; + } + + if (port == 0) + port = 8888; + else if (port < 0 || port >= 65536) + return "TCP/UDP port must between 1-65535"; + + // p pointer to start of port + if (p == NULL) + p = strchr(name, '/'); + + uint32_t addr[4]; + int family = 0; + // or if port missing, start of protocol + if (p == NULL) + { + family = name2addr(name, addr); + } + else + { + int len = p - name; + char localname[len]; + memcpy(localname, name, len); + localname[len - 1] = 0; + family = name2addr(localname, addr); + } + + switch (family) + { + default: + return "Invalid Hostname"; + + case AF_INET: + socktype = type; + alen = sizeof(struct sockaddr_in); + memset(in4, 0, alen); + in4->sin_family = AF_INET; + in4->sin_addr.s_addr = addr[0]; + in4->sin_port = htons(port); + build_name_ipv4(); + break; + + case AF_INET6: + socktype = type; + alen = sizeof(struct sockaddr_in6); + memset(in6, 0, alen); + in6->sin6_family = AF_INET6; + memcpy(in6->sin6_addr.s6_addr, addr, 16); + in6->sin6_port = htons(port); + build_name_ipv6(); + }; + return NULL; +} + +// Match().... return 1 if address is identical +int SocketAddress::Equal(const sockaddr *that, int alen, int type) const +{ + if (this->socktype != type) + return 0; + if (this->addr->sa_family != that->sa_family) + return 0; + + switch (this->addr->sa_family) + { + case AF_UNIX: + { + const struct sockaddr_un *un = (const struct sockaddr_un *)that; + if (this->un->sun_path[0] != un->sun_path[0]) + { + return 0; + } + if (this->un->sun_path[0] == '/' && un->sun_path[0] == '/') + { + if (!strncmp(this->un->sun_path, un->sun_path, sizeof(un->sun_path))) + { + return 1; + } + } + } + break; + + case AF_INET: + { + const struct sockaddr_in *in4 = (const struct sockaddr_in *)that; + return alen >= (int)sizeof(struct sockaddr_in) && + this->in4->sin_port == in4->sin_port && + this->in4->sin_addr.s_addr == in4->sin_addr.s_addr; + } + break; + + case AF_INET6: + if (alen < (int)sizeof(struct sockaddr_in6)) + return 0; + + { + const struct sockaddr_in6 *in6 = (const struct sockaddr_in6 *)that; + if (this->in6->sin6_port != in6->sin6_port) + return 0; + + if (!IN6_ARE_ADDR_EQUAL(this->in6->sin6_addr.s6_addr32, in6->sin6_addr.s6_addr32)) + return 0; + } + + return 1; + break; + }; + + if ((int)this->alen != alen) + return 0; + return !memcmp(&this->addr, addr, alen); +} + +int SocketAddress::Equal(const SocketAddress *that) const +{ + return Equal(that->addr, that->alen, that->socktype); +} + +int SocketAddress::Equal(const struct sockaddr_un *addr, int len, int type) const +{ + return Equal((const struct sockaddr *)addr, len, type); +} +int SocketAddress::Equal(const struct sockaddr_in *addr, int len, int type) const +{ + return Equal((const struct sockaddr *)addr, len, type); +} +int SocketAddress::Equal(const struct sockaddr_in6 *addr, int len, int type) const +{ + return Equal((const struct sockaddr *)addr, len, type); +} + +int SocketAddress::Equal(const char *path) const +{ + SocketAddress temp; + if (temp.set_address(path, 0) < 0) + return 0; + return Equal(&temp); +} + +int SocketAddress::Equal(const char *path, int port, int type) const +{ + SocketAddress temp; + if (temp.set_address(path, port, type) < 0) + return 0; + return Equal(&temp); +} +int SocketAddress::Equal(const char *path, const char *port) const +{ + SocketAddress temp; + if (temp.set_address(path, port) < 0) + return 0; + return Equal(&temp); +} + +// Match().... return wether this object accept the peer address +int SocketAddress::Match(const SocketAddress *that) const +{ + if (this->socktype != that->socktype) + return 0; + + int this_family = this->addr->sa_family; + uint32_t this_ipv4 = INADDR_NONE; + uint16_t this_port = 0; + uint32_t that_ipv4 = INADDR_NONE; + uint16_t that_port = 0; + + // cast down V4MAPPED to ipv4 + switch (this_family) + { + case AF_INET: + this_ipv4 = this->in4->sin_addr.s_addr; + this_port = this->in4->sin_port; + break; + case AF_INET6: + if (IN6_IS_ADDR_V4MAPPED(this->in6->sin6_addr.s6_addr32)) + { + this_family = AF_INET; + this_ipv4 = this->in6->sin6_addr.s6_addr[3]; + this_port = this->in6->sin6_port; + } + break; + } + + // cast down V4MAPPED to ipv4 + switch (that->addr->sa_family) + { + case AF_INET: + that_ipv4 = that->in4->sin_addr.s_addr; + that_port = that->in4->sin_port; + break; + case AF_INET6: + if (IN6_IS_ADDR_V4MAPPED(that->in6->sin6_addr.s6_addr32)) + { + that_ipv4 = that->in6->sin6_addr.s6_addr[3]; + that_port = that->in6->sin6_port; + } + if (IN6_IS_ADDR_UNSPECIFIED(that->in6->sin6_addr.s6_addr32)) + { + that_ipv4 = INADDR_ANY; + that_port = that->in6->sin6_port; + } + break; + } + + switch (this_family) + { + case AF_UNIX: + if (this->un->sun_path[0] != that->un->sun_path[0]) + { + return 0; + } + if (this->un->sun_path[0] == '/' && that->un->sun_path[0] == '/') + { + if (!strncmp(this->un->sun_path, that->un->sun_path, sizeof(that->un->sun_path))) + { + return 1; + } + } + break; + + case AF_INET: + if (this_port != that_port) + return 0; + + if (this_ipv4 == that_ipv4) + return 1; + + // * any address + if (this_ipv4 == INADDR_ANY) + { + if (is_local_ipv4(that_ipv4)) + return 1; + } + + return 0; + break; + + case AF_INET6: + // :: zero address + this_port = this->in6->sin6_port; + + if (IN6_IS_ADDR_UNSPECIFIED(this->in6->sin6_addr.s6_addr32)) + { + if (is_local_ipv4(that_ipv4) && this_port == that_port) + return 1; + + if (that->addr->sa_family == AF_INET6 && this_port == that->in6->sin6_port) + { + if (IN6_IS_ADDR_UNSPECIFIED(that->in6->sin6_addr.s6_addr32)) + return 1; + if (IN6_IS_ADDR_LOOPBACK(that->in6->sin6_addr.s6_addr32)) + return 1; + } + } + + if (that->addr->sa_family == AF_INET6) + { + if (this_port != that->in6->sin6_port) + return 0; + if (IN6_ARE_ADDR_EQUAL(this->in6->sin6_addr.s6_addr32, that->in6->sin6_addr.s6_addr32)) + return 1; + } + + return 0; + break; + }; + if (this->alen != that->alen) + return 0; + return !memcmp(&this->addr, &that->addr, this->alen); +} + +int SocketAddress::Match(const char *path, int port, int type) const +{ + SocketAddress temp; + if (temp.set_address(path, port, type) < 0) + return 0; + return Match(&temp); +} + +int SocketAddress::Match(const char *path) const +{ + SocketAddress temp; + if (temp.set_address(path, 0) < 0) + return 0; + return Match(&temp); +} + +int SocketAddress::Match(const char *path, const char *port) const +{ + SocketAddress temp; + if (temp.set_address(path, port) < 0) + return 0; + return Match(&temp); +} + +int SocketAddress::Match(const struct sockaddr_un *addr, int len, int type) const +{ + SocketAddress temp; + if (temp.set_address(addr, len, type) < 0) + return 0; + return Match(&temp); +} +int SocketAddress::Match(const struct sockaddr_in *addr, int len, int type) const +{ + SocketAddress temp; + if (temp.set_address(addr, len, type) < 0) + return 0; + return Match(&temp); +} + +int SocketAddress::Match(const struct sockaddr_in6 *addr, int len, int type) const +{ + SocketAddress temp; + if (temp.set_address(addr, len, type) < 0) + return 0; + return Match(&temp); +} diff --git a/src/search_local/index_storage/common/net_addr.h b/src/search_local/index_storage/common/net_addr.h new file mode 100644 index 0000000..d5d6864 --- /dev/null +++ b/src/search_local/index_storage/common/net_addr.h @@ -0,0 +1,153 @@ +/* + * ===================================================================================== + * + * Filename: net_addr.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_NETADDR_________ +#define __CH_NETADDR_________ + +#include +#include +#include +#include + +class SocketAddress +{ +public: // members + char sockname[128]; + int socktype; + socklen_t alen; + union + { + struct sockaddr_in in4[1]; + struct sockaddr_in6 in6[1]; + struct sockaddr_un un[1]; + struct sockaddr addr[1]; + }; + +public: // methods + SocketAddress(void) : socktype(0), alen(0) { this->addr->sa_family = 0; } + ~SocketAddress(void) {} + + // initialize to name + // success: return NULL + // failure: return error message + const char *set_address(const char *name, int port = 0, int type = SOCK_STREAM); + const char *set_address(const char *name, const char *portstr); + const char *set_address(const struct sockaddr *, int socklen, int type); + const char *set_address(const struct sockaddr_un *, int socklen, int type); + const char *set_address(const struct sockaddr_in *, int socklen, int type); + const char *set_address(const struct sockaddr_in6 *, int socklen, int type); + int socket_family(void) const { return this->addr->sa_family; } + int socket_type(void) const { return this->socktype; } + const char *socket_type_name(void) const + { + return this->socktype == SOCK_DGRAM ? "DGRAM" : this->socktype == SOCK_STREAM ? "STREAM" : this->socktype == SOCK_RAW ? "RAW" : this->socktype == SOCK_SEQPACKET ? "SEQPACKET" : ""; + } + int create_socket(void) const { return socket(this->addr->sa_family, this->socktype, 0); } + int connect_socket(int fd) const { return connect(fd, this->addr, this->alen); } + int bind_socket(int fd) const { return bind(fd, this->addr, this->alen); } + int send_to(int fd, const char *buf, int len, int flags = 0) const + { + return sendto(fd, buf, len, flags, this->addr, this->alen); + } + int recv_from(int fd, char *buf, int len, int flags = 0) + { + this->alen = sizeof(struct sockaddr); + int ret = recvfrom(fd, buf, len, flags, this->addr, &this->alen); + build_name(); + return ret; + } + int connect_socket(void) const + { + int fd = create_socket(); + if (fd >= 0) + { + if (connect_socket(fd) < 0) + close(fd); + fd = -1; + } + return fd; + } + int bind_socket(void) const + { + int fd = create_socket(); + if (fd >= 0) + { + if (bind_socket(fd) < 0) + close(fd); + fd = -1; + } + return fd; + } + + int Equal(const SocketAddress *) const; + int Equal(const char *) const; + int Equal(const char *, int, int = SOCK_STREAM) const; + int Equal(const char *, const char *) const; + int Equal(const struct sockaddr *, int, int = SOCK_STREAM) const; + int Equal(const struct sockaddr_un *, int, int = SOCK_STREAM) const; + int Equal(const struct sockaddr_in *, int, int = SOCK_STREAM) const; + int Equal(const struct sockaddr_in6 *, int, int = SOCK_STREAM) const; + int Match(const SocketAddress *) const; + int Match(const char *) const; + int Match(const char *, int, int = SOCK_STREAM) const; + int Match(const char *, const char *) const; + int Match(const struct sockaddr *, int, int = SOCK_STREAM) const; + int Match(const struct sockaddr_un *, int, int = SOCK_STREAM) const; + int Match(const struct sockaddr_in *, int, int = SOCK_STREAM) const; + int Match(const struct sockaddr_in6 *, int, int = SOCK_STREAM) const; + const char *Name(void) const { return this->addr->sa_family == 0 ? NULL : sockname; } + +private: + const char *set_unix_address(const char *name, int type = SOCK_STREAM); + const char *set_ipv4_address(const char *name, int port = 0, int type = SOCK_STREAM); + const char *set_ipv6_address(const char *name, int port = 0, int type = SOCK_STREAM); + const char *set_host_address(const char *name, int port = 0, int type = SOCK_STREAM); + void build_name(void); + void build_name_bad(void); + void build_name_none(void); + void build_name_ipv4(void); + void build_name_ipv6(void); + void build_name_unix(void); + +public: // static methods + static inline int is_unix_socket_path(const char *path) + { + return path[0] == '/' || path[0] == '@'; + } + + static inline int init_unix_socket_address(struct sockaddr_un *addr, const char *path) + { + + bzero(addr, sizeof(struct sockaddr_un)); + addr->sun_family = AF_LOCAL; + strncpy(addr->sun_path, path, sizeof(addr->sun_path) - 1); + socklen_t addrlen = SUN_LEN(addr); + if (path[0] == '@') + addr->sun_path[0] = '\0'; + return addrlen; + } + static inline void set_socket_buffer(int netfd, int rbufsz, int wbufsz) + { + if (rbufsz) + setsockopt(netfd, SOL_SOCKET, SO_RCVBUF, &rbufsz, sizeof(rbufsz)); + if (wbufsz) + setsockopt(netfd, SOL_SOCKET, SO_SNDBUF, &wbufsz, sizeof(wbufsz)); + } +}; + +extern int sock_bind(const SocketAddress *addr, int backlog = 0, int rbufsz = 0, int wbufsz = 0, int reuse = 0, int nodelay = 0, int defer = 0); +#endif diff --git a/src/search_local/index_storage/common/net_bind.cc b/src/search_local/index_storage/common/net_bind.cc new file mode 100644 index 0000000..8bd1825 --- /dev/null +++ b/src/search_local/index_storage/common/net_bind.cc @@ -0,0 +1,66 @@ +/* + * ===================================================================================== + * + * Filename: sock_bind.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "net_addr.h" +#include "log.h" + +int sock_bind(const SocketAddress *addr, int backlog, int rbufsz, int wbufsz, int reuse, int nodelay, int defer_accept) +{ + int netfd; + + if ((netfd = addr->create_socket()) == -1) + { + log_error("%s: make socket error, %m", addr->Name()); + return -1; + } + + int optval = 1; + if (reuse) + setsockopt(netfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + if (nodelay) + setsockopt(netfd, SOL_TCP, TCP_NODELAY, &optval, sizeof(optval)); + + /* 避免没有请求的空连接唤醒epoll浪费cpu资源 */ + if (defer_accept) + { + optval = 60; + setsockopt(netfd, SOL_TCP, TCP_DEFER_ACCEPT, &optval, sizeof(optval)); + } + + if (addr->bind_socket(netfd) == -1) + { + log_error("%s: bind failed, %m", addr->Name()); + close(netfd); + return -1; + } + + if (addr->socket_type() == SOCK_STREAM && listen(netfd, backlog) == -1) + { + log_error("%s: listen failed, %m", addr->Name()); + close(netfd); + return -1; + } + + if (rbufsz) + setsockopt(netfd, SOL_SOCKET, SO_RCVBUF, &rbufsz, sizeof(rbufsz)); + if (wbufsz) + setsockopt(netfd, SOL_SOCKET, SO_SNDBUF, &wbufsz, sizeof(wbufsz)); + + log_info("%s on %s", addr->socket_type() == SOCK_STREAM ? "listen" : "bind", addr->Name()); + return netfd; +} diff --git a/src/search_local/index_storage/common/new_hash.cc b/src/search_local/index_storage/common/new_hash.cc new file mode 100644 index 0000000..461136d --- /dev/null +++ b/src/search_local/index_storage/common/new_hash.cc @@ -0,0 +1,107 @@ +/* + * ===================================================================================== + * + * Filename: new_hash.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "new_hash.h" + +#define mix(a, b, c) \ + { \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 13); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 8); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 13); \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 12); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 16); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 5); \ + a = a - b; \ + a = a - c; \ + a = a ^ (c >> 3); \ + b = b - c; \ + b = b - a; \ + b = b ^ (a << 10); \ + c = c - a; \ + c = c - b; \ + c = c ^ (b >> 15); \ + } + +typedef unsigned int u4; + +unsigned int new_hash(const char *k, int length) +{ + unsigned int a, b, c; /* the internal state */ + u4 len; /* how many key bytes still need mixing */ + + /* Set up the internal state */ + len = length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + // 'HTTC' + c = 0x48545443; /* variable initialization of internal state */ + + /*---------------------------------------- handle most of the key */ + while (len >= 12) + { + a = a + (k[0] + ((u4)k[1] << 8) + ((u4)k[2] << 16) + ((u4)k[3] << 24)); + b = b + (k[4] + ((u4)k[5] << 8) + ((u4)k[6] << 16) + ((u4)k[7] << 24)); + c = c + (k[8] + ((u4)k[9] << 8) + ((u4)k[10] << 16) + ((u4)k[11] << 24)); + mix(a, b, c); + k = k + 12; + len = len - 12; + } + + /*------------------------------------- handle the last 11 bytes */ + c = c + length; + switch (len) /* all the case statements fall through */ + { + case 11: + c = c + ((u4)k[10] << 24); + case 10: + c = c + ((u4)k[9] << 16); + case 9: + c = c + ((u4)k[8] << 8); + /* the first byte of c is reserved for the length */ + case 8: + b = b + ((u4)k[7] << 24); + case 7: + b = b + ((u4)k[6] << 16); + case 6: + b = b + ((u4)k[5] << 8); + case 5: + b = b + k[4]; + case 4: + a = a + ((u4)k[3] << 24); + case 3: + a = a + ((u4)k[2] << 16); + case 2: + a = a + ((u4)k[1] << 8); + case 1: + a = a + k[0]; + /* case 0: nothing left to add */ + } + mix(a, b, c); + /*-------------------------------------------- report the result */ + return c; +} diff --git a/src/search_local/index_storage/common/new_hash.h b/src/search_local/index_storage/common/new_hash.h new file mode 100644 index 0000000..58c5809 --- /dev/null +++ b/src/search_local/index_storage/common/new_hash.h @@ -0,0 +1,25 @@ +/* + * ===================================================================================== + * + * Filename: new_hash.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef BITMAP_NEW_HASH_H__ +#define BITMAP_NEW_HASH_H__ + +#include + +uint32_t new_hash(const char *data, int len); + +#endif diff --git a/src/search_local/index_storage/common/non_copyable.h b/src/search_local/index_storage/common/non_copyable.h new file mode 100644 index 0000000..be0bc28 --- /dev/null +++ b/src/search_local/index_storage/common/non_copyable.h @@ -0,0 +1,38 @@ +/* + * ===================================================================================== + * + * Filename: non_copyable.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_NONCOPY_H__ +#define __DTC_NONCOPY_H__ + +#include "namespace.h" + +DTC_BEGIN_NAMESPACE + +class noncopyable +{ +protected: + noncopyable(void) {} + ~noncopyable(void) {} + +private: + noncopyable(const noncopyable &); + const noncopyable &operator=(const noncopyable &); +}; + +DTC_END_NAMESPACE + +#endif diff --git a/src/search_local/index_storage/common/packet.h b/src/search_local/index_storage/common/packet.h new file mode 100644 index 0000000..7063d72 --- /dev/null +++ b/src/search_local/index_storage/common/packet.h @@ -0,0 +1,123 @@ +/* + * ===================================================================================== + * + * Filename: packet.h + * + * Description: packet operation. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_PACKET_H__ +#define __CH_PACKET_H__ + +#include + +#include "protocol.h" +#include "section.h" +#include "net_addr.h" +#include "log.h" + +class NCRequest; +union DTCValue; +class DTCTask; +class TaskRequest; +class DTCTableDefinition; + +enum DTCSendResult +{ + SendResultError, + SendResultMoreData, + SendResultDone +}; + +/* just one buff malloced */ +class Packet +{ +private: + struct iovec *v; + int nv; + int bytes; + BufferChain *buf; + int sendedVecCount; + + Packet(const Packet &); + +public: + Packet() : v(NULL), nv(0), bytes(0), buf(NULL), sendedVecCount(0){}; + ~Packet() + { + /* free buffer chain buffer in several place, not freed all here */ + FREE_IF(buf); + } + + inline void Clean() + { + v = NULL; + nv = 0; + bytes = 0; + if (buf) + buf->Clean(); + } + int Send(int fd); + int send_to(int fd, void *name, int namelen); + int send_to(int fd, SocketAddress *addr) + { + return addr == NULL ? Send(fd) : send_to(fd, (void *)addr->addr, (int)addr->alen); + } + + /* for agent_sender */ + int Bytes(void); + void free_result_buff(); + const struct iovec *IOVec() { return v; } + int vec_count() { return nv; } + void send_done_one_vec() { sendedVecCount++; } + bool is_send_done() { return sendedVecCount == nv; } + + static int encode_header(PacketHeader &header); +#if __BYTE_ORDER == __LITTLE_ENDIAN + static int encode_header(const PacketHeader &header); +#endif + int encode_forward_request(TaskRequest &); + int encode_pass_thru(DTCTask &); + int encode_fetch_data(TaskRequest &); + + // encode result, for helper/server reply + // side effect: + // if error code set(result_code()<0), ignore DTCResultSet + // if no result/error code set, no result_code() to zero + // if no result key set, set result key to request key + int encode_result(DTCTask &, int mtu = 0, uint32_t ts = 0); + int encode_result(TaskRequest &, int mtu = 0); + int encode_detect(const DTCTableDefinition *tdef, int sn = 1); + int encode_request(NCRequest &r, const DTCValue *k = NULL); + static int encode_simple_request(NCRequest &rq, const DTCValue *kptr, char *&ptr, int &len); + char *allocate_simple(int size); + + int encode_result(DTCTask *task, int mtu = 0, uint32_t ts = 0) + { + return encode_result(*task, mtu, ts); + } + int encode_result(TaskRequest *task, int mtu = 0) + { + return encode_result(*task, mtu); + } + int encode_forward_request(TaskRequest *task) + { + return encode_forward_request(*task); + } + + /* for agent */ + int encode_agent_request(NCRequest &rq, const DTCValue *kptr); + + int encode_reload_config(const DTCTableDefinition *tdef, int sn = 1); +}; + +#endif diff --git a/src/search_local/index_storage/common/packet_base.cc b/src/search_local/index_storage/common/packet_base.cc new file mode 100644 index 0000000..6ad490f --- /dev/null +++ b/src/search_local/index_storage/common/packet_base.cc @@ -0,0 +1,131 @@ +/* + * ===================================================================================== + * + * Filename: packet_base.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "packet.h" +#include "protocol.h" + +#include "log.h" +int Packet::Send(int fd) +{ + if (nv <= 0) + return SendResultDone; + + struct msghdr msgh; + msgh.msg_name = NULL; + msgh.msg_namelen = 0; + msgh.msg_iov = v; + msgh.msg_iovlen = nv; + msgh.msg_control = NULL; + msgh.msg_controllen = 0; + msgh.msg_flags = 0; + + int rv = sendmsg(fd, &msgh, MSG_DONTWAIT | MSG_NOSIGNAL); + + if (rv < 0) + { + if (errno == EINTR || errno == EAGAIN || errno == EINPROGRESS) + return SendResultMoreData; + return SendResultError; + } + if (rv == 0) + return SendResultMoreData; + bytes -= rv; + if (bytes == 0) + { + nv = 0; + return SendResultDone; + } + while (nv > 0 && rv >= (int64_t)v->iov_len) + { + rv -= v->iov_len; + v++; + nv--; + } + if (rv > 0) + { + v->iov_base = (char *)v->iov_base + rv; + v->iov_len -= rv; + } + return nv == 0 ? SendResultDone : SendResultMoreData; +} + +int Packet::send_to(int fd, void *addr, int len) +{ + if (nv <= 0) + return SendResultDone; + struct msghdr msgh; + msgh.msg_name = addr; + msgh.msg_namelen = len; + msgh.msg_iov = v; + msgh.msg_iovlen = nv; + msgh.msg_control = NULL; + msgh.msg_controllen = 0; + msgh.msg_flags = 0; + + int rv = sendmsg(fd, &msgh, MSG_DONTWAIT | MSG_NOSIGNAL); + + if (rv < 0) + { +#if 0 + if(errno==EINTR || errno==EAGAIN || errno==EINPROGRESS) + return SendResultMoreData; +#endif + return SendResultError; + } + if (rv == 0) + return SendResultError; + + bytes -= rv; + if (bytes == 0) + { + nv = 0; + return SendResultDone; + } + return SendResultError; +} + +int Packet::encode_header(PacketHeader &header) +{ + int len = sizeof(header); + for (int i = 0; i < DRequest::Section::Total; i++) + { + len += header.len[i]; +#if __BYTE_ORDER == __BIG_ENDIAN + header.len[i] = bswap_32(header.len[i]); +#endif + } + return len; +} + +#if __BYTE_ORDER == __LITTLE_ENDIAN +int Packet::encode_header(const PacketHeader &header) +{ + int len = sizeof(header); + for (int i = 0; i < DRequest::Section::Total; i++) + { + len += header.len[i]; + } + return len; +} +#endif diff --git a/src/search_local/index_storage/common/packet_client.cc b/src/search_local/index_storage/common/packet_client.cc new file mode 100644 index 0000000..75dd301 --- /dev/null +++ b/src/search_local/index_storage/common/packet_client.cc @@ -0,0 +1,269 @@ +/* + * ===================================================================================== + * + * Filename: packet_client.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "value.h" +#include "section.h" +#include "protocol.h" +#include "version.h" +#include "packet.h" +#include "../api/c_api/dtc_int.h" +#include "table_def.h" +#include "decode.h" + +#include "log.h" + +template +int Templateencode_request(NCRequest &rq, const DTCValue *kptr, T *tgt) +{ + NCServer *sv = rq.server; + int key_type = rq.keytype; + const char *tab_name = rq.tablename; + + const char *accessKey = sv->accessToken.c_str(); + + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = rq.tdef ? DRequest::Flag::KeepAlive : DRequest::Flag::KeepAlive + DRequest::Flag::NeedTableDefinition; + header.flags |= (rq.flags & (DRequest::Flag::no_cache | DRequest::Flag::NoResult | DRequest::Flag::no_next_server | DRequest::Flag::MultiKeyValue)); + header.cmd = rq.cmd; + + DTCVersionInfo vi; + // tablename & hash + vi.set_table_name(tab_name); + if (rq.tdef) + vi.set_table_hash(rq.tdef->table_hash()); + vi.set_serial_nr(sv->NextSerialNr()); + // app version + if (sv->appname) + vi.set_tag(5, sv->appname); + // lib version + vi.set_tag(6, "ctlib-v" DTC_VERSION); + vi.set_tag(9, key_type); + + // hot backup id + vi.set_hot_backup_id(rq.hotBackupID); + + // hot backup timestamp + vi.set_master_hb_timestamp(rq.master_hb_timestamp); + vi.set_slave_hb_timestamp(rq.slave_hb_timestamp); + if (sv->tdef && rq.adminCode != 0) + vi.set_data_table_hash(sv->tdef->table_hash()); + + // accessKey + vi.set_access_key(accessKey); + + Array kt(0, NULL); + Array kn(0, NULL); + Array kv(0, NULL); + int isbatch = 0; + if (rq.flags & DRequest::Flag::MultiKeyValue) + { + if (sv->simple_batch_key() && rq.kvl.key_count() == 1) + { + /* single field single key batch, convert to normal */ + kptr = rq.kvl.val; + header.flags &= ~DRequest::Flag::MultiKeyValue; + } + else + { + isbatch = 1; + } + } + + if (isbatch) + { + int keyFieldCnt = rq.kvl.key_fields(); + int keyCount = rq.kvl.key_count(); + int i, j; + vi.set_tag(10, keyFieldCnt); + vi.set_tag(11, keyCount); + // key type + kt.ptr = (char *)MALLOC(sizeof(uint8_t) * keyFieldCnt); + if (kt.ptr == NULL) + throw std::bad_alloc(); + for (i = 0; i < keyFieldCnt; i++) + kt.Add((uint8_t)(rq.kvl.key_type(i))); + vi.set_tag(12, DTCValue::Make(kt.ptr, kt.len)); + // key name + kn.ptr = (char *)MALLOC((256 + sizeof(uint32_t)) * keyFieldCnt); + if (kn.ptr == NULL) + throw std::bad_alloc(); + for (i = 0; i < keyFieldCnt; i++) + kn.Add(rq.kvl.key_name(i)); + vi.set_tag(13, DTCValue::Make(kn.ptr, kn.len)); + // key value + unsigned int buf_size = 0; + for (j = 0; j < keyCount; j++) + { + for (i = 0; i < keyFieldCnt; i++) + { + DTCValue &v = rq.kvl(j, i); + switch (rq.kvl.key_type(i)) + { + case DField::Signed: + case DField::Unsigned: + if (buf_size < kv.len + sizeof(uint64_t)) + { + if (REALLOC(kv.ptr, buf_size + 256) == NULL) + throw std::bad_alloc(); + buf_size += 256; + } + kv.Add(v.u64); + break; + case DField::String: + case DField::Binary: + if (buf_size < (unsigned int)kv.len + sizeof(uint32_t) + v.bin.len) + { + if (REALLOC(kv.ptr, buf_size + sizeof(uint32_t) + v.bin.len) == NULL) + throw std::bad_alloc(); + buf_size += sizeof(uint32_t) + v.bin.len; + } + kv.Add(v.bin.ptr, v.bin.len); + break; + default: + break; + } + } + } + } + + DTCRequestInfo ri; + // key + if (isbatch) + ri.set_key(DTCValue::Make(kv.ptr, kv.len)); + else if (kptr) + ri.set_key(*kptr); + // cmd + if (sv->get_timeout()) + { + ri.set_timeout(sv->get_timeout()); + } + //limit + if (rq.limitCount) + { + ri.set_limit_start(rq.limitStart); + ri.set_limit_count(rq.limitCount); + } + if (rq.adminCode > 0) + { + ri.set_admin_code(rq.adminCode); + } + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(vi, DField::None); + + //log_info("header.len:%d, vi.access_key:%s, vi.access_key().len:%d", header.len[DRequest::Section::VersionInfo], vi.access_key().ptr, vi.access_key().len); + + /* no table definition */ + header.len[DRequest::Section::table_definition] = 0; + + /* calculate rq info */ + header.len[DRequest::Section::RequestInfo] = + encoded_bytes_simple_section(ri, isbatch ? DField::String : key_type); + + /* no result info */ + header.len[DRequest::Section::ResultInfo] = 0; + + /* copy update info */ + header.len[DRequest::Section::UpdateInfo] = + encoded_bytes_field_value(rq.ui); + + /* copy condition info */ + header.len[DRequest::Section::ConditionInfo] = + encoded_bytes_field_value(rq.ci); + + /* full set */ + header.len[DRequest::Section::FieldSet] = + encoded_bytes_field_set(rq.fs); + + /* no result set */ + header.len[DRequest::Section::DTCResultSet] = 0; + + const int len = Packet::encode_header(header); + char *p = tgt->allocate_simple(len); + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, vi, DField::None); + p = encode_simple_section(p, ri, isbatch ? DField::String : key_type); + p = encode_field_value(p, rq.ui); + p = encode_field_value(p, rq.ci); + p = encode_field_set(p, rq.fs); + + FREE(kt.ptr); + FREE(kn.ptr); + FREE(kv.ptr); + + return 0; +} + +char *Packet::allocate_simple(int len) +{ + buf = (BufferChain *)MALLOC(sizeof(BufferChain) + sizeof(struct iovec) + len); + if (buf == NULL) + throw std::bad_alloc(); + + buf->nextBuffer = NULL; + /* never use usedBytes here */ + buf->totalBytes = sizeof(struct iovec) + len; + v = (struct iovec *)buf->data; + nv = 1; + char *p = buf->data + sizeof(struct iovec); + v->iov_base = p; + v->iov_len = len; + bytes = len; + return p; +} + +int Packet::encode_request(NCRequest &rq, const DTCValue *kptr) +{ + return Templateencode_request(rq, kptr, this); +} + +class SimpleBuffer : public DTCBinary +{ +public: + char *allocate_simple(int size); +}; + +char *SimpleBuffer::allocate_simple(int size) +{ + len = size; + ptr = (char *)MALLOC(len); + if (ptr == NULL) + throw std::bad_alloc(); + return ptr; +} + +int Packet::encode_simple_request(NCRequest &rq, const DTCValue *kptr, char *&ptr, int &len) +{ + SimpleBuffer buf; + int ret = Templateencode_request(rq, kptr, &buf); + ptr = buf.ptr; + len = buf.len; + return ret; +} diff --git a/src/search_local/index_storage/common/packet_result.cc b/src/search_local/index_storage/common/packet_result.cc new file mode 100644 index 0000000..0f2d69f --- /dev/null +++ b/src/search_local/index_storage/common/packet_result.cc @@ -0,0 +1,198 @@ +/* + * ===================================================================================== + * + * Filename: packet_result.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "result.h" +#include "decode.h" + +ResultWriter::ResultWriter(const DTCFieldSet *fs, unsigned int st, unsigned int cnt) +{ + if (cnt == 0) + st = 0; + limitStart = st; + limitNext = st + cnt; + const int nf = fs == NULL ? 0 : fs->num_fields(); + fieldSet = nf ? fs : NULL; + totalRows = 0; + numRows = 0; +} + +int ResultWriter::set_rows(unsigned int rows) +{ + if (fieldSet == NULL /* count_only */ || + (numRows == 0 && rows <= limitStart /* Not in Range */)) + { + totalRows = rows; + numRows = limitNext == 0 ? rows : rows <= limitStart ? 0 : rows <= limitNext ? rows - limitStart : limitNext - limitStart; + } + else if (is_full() == 0) + return -1; + else + totalRows = rows; + + return 0; +} + +ResultPacket::ResultPacket(const DTCFieldSet *fs, unsigned int st, unsigned int cnt) : ResultWriter(fs, st, cnt) +{ + const int nf = fs == NULL ? 0 : fs->num_fields(); + int len = 5 + 1 + nf; + bc = (BufferChain *)MALLOC(1024); + if (bc == NULL) + throw(int) - ENOMEM; + bc->nextBuffer = NULL; + bc->usedBytes = len; + bc->totalBytes = 1024 - sizeof(BufferChain); + rowDataBegin = bc->usedBytes; + char *p = bc->data + 5; + *p++ = nf; + if (nf) + { + for (int i = 0; i < nf; i++) + *p++ = fs->field_id(i); + } +} + +void ResultPacket::Clean() +{ + ResultWriter::Clean(); + if (bc) + bc->Clean(); +} + +/* pool, bc should exist */ +int ResultPacket::Set(const DTCFieldSet *fs, unsigned int st, unsigned int ct) +{ + ResultWriter::Set(fs, st, ct); + const int nf = fs == NULL ? 0 : fs->num_fields(); + int len = 5 + 1 + nf; + if (bc == NULL) + return -1; + bc->nextBuffer = NULL; + bc->usedBytes = len; + rowDataBegin = bc->usedBytes; + char *p = bc->data + 5; + *p++ = nf; + if (nf) + { + for (int i = 0; i < nf; i++) + *p++ = fs->field_id(i); + } + + return 0; +} + +/* resultPacket is just a buff */ +ResultPacket::~ResultPacket(void) +{ + FREE_IF(bc); +} + +static int expand(BufferChain *&bc, int addsize) +{ + int expectsize = bc->usedBytes + addsize; + if (bc->totalBytes >= expectsize) + return 0; + + // add header and round to 8 byte aligned + expectsize += sizeof(BufferChain); + expectsize |= 7; + expectsize += 1; + + int sparsesize = expectsize; + if (sparsesize > addsize * 16) + sparsesize = addsize * 16; + + sparsesize += expectsize; + if (REALLOC(bc, sparsesize) != NULL) + { + bc->totalBytes = sparsesize - sizeof(BufferChain); + return 0; + } + if (REALLOC(bc, expectsize) != NULL) + { + bc->totalBytes = expectsize - sizeof(BufferChain); + return 0; + } + return -1; +} + +int ResultPacket::append_row(const RowValue &r) +{ + int ret = 0; + totalRows++; + if (limitNext > 0) + { + if (totalRows <= limitStart || totalRows > limitNext) + return 0; + } + if (fieldSet) + { + int len = 0; + for (int i = 0; i < fieldSet->num_fields(); i++) + { + const int id = fieldSet->field_id(i); + len += encoded_bytes_data_value(r.field_value(id), r.field_type(id)); + } + +#if 0 + BufferChain *c = (BufferChain *)REALLOC(bc, + sizeof(BufferChain)+ bc->usedBytes + len); + + if(c==NULL) return -ENOMEM; + bc = c; +#else + if (expand(bc, len) != 0) + return -ENOMEM; +#endif + + char *p = bc->data + bc->usedBytes; + bc->usedBytes += len; + + for (int i = 0; i < fieldSet->num_fields(); i++) + { + const int id = fieldSet->field_id(i); + p = encode_data_value(p, r.field_value(id), r.field_type(id)); + } + ret = 1; + } + numRows = totalRows - limitStart; + return ret; +} + +int ResultPacket::merge_no_limit(const ResultWriter *rp0) +{ + const ResultPacket &rp = *(const ResultPacket *)rp0; + + int expectedsize = bc->usedBytes + (rp.bc->usedBytes - rp.rowDataBegin); + if (bc->totalBytes < expectedsize) + { + BufferChain *c = (BufferChain *)REALLOC(bc, sizeof(BufferChain) + expectedsize + 1024); + if (c == NULL) + return -ENOMEM; + bc = c; + bc->totalBytes = expectedsize + 1024; + } + + char *p = bc->data + bc->usedBytes; + memcpy(p, rp.bc->data + rp.rowDataBegin, rp.bc->usedBytes - rp.rowDataBegin); + + bc->usedBytes += rp.bc->usedBytes - rp.rowDataBegin; + totalRows += rp.totalRows; + numRows += rp.numRows; + + return 0; +} diff --git a/src/search_local/index_storage/common/packet_server.cc b/src/search_local/index_storage/common/packet_server.cc new file mode 100644 index 0000000..6aedc23 --- /dev/null +++ b/src/search_local/index_storage/common/packet_server.cc @@ -0,0 +1,644 @@ +/* + * ===================================================================================== + * + * Filename: packet_erver.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "version.h" +#include "packet.h" +#include "table_def.h" +#include "decode.h" +#include "task_request.h" + +#include "log.h" + +/* not yet pollized*/ +int Packet::encode_detect(const DTCTableDefinition *tdef, int sn) +{ + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = DRequest::Flag::KeepAlive; + header.cmd = DRequest::Get; + + DTCVersionInfo vi; + // tablename & hash + vi.set_table_name(tdef->table_name()); + vi.set_table_hash(tdef->table_hash()); + vi.set_serial_nr(sn); + // app version + vi.set_tag(5, "dtcd"); + // lib version + vi.set_tag(6, "ctlib-v" DTC_VERSION); + vi.set_tag(9, tdef->field_type(0)); + + DTCRequestInfo ri; + // key + ri.set_key(DTCValue::Make(0)); + // ri.set_timeout(30); + + // field set + char fs[4] = {1, 0, 0, 0xFF}; + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(vi, DField::None); + + /* no table definition */ + header.len[DRequest::Section::table_definition] = 0; + + /* encode request info */ + header.len[DRequest::Section::RequestInfo] = + encoded_bytes_simple_section(ri, tdef->key_type()); + + /* no result info */ + header.len[DRequest::Section::ResultInfo] = 0; + + /* encode update info */ + header.len[DRequest::Section::UpdateInfo] = 0; + + /* encode condition info */ + header.len[DRequest::Section::ConditionInfo] = 0; + + /* full set */ + header.len[DRequest::Section::FieldSet] = 4; + + /* no result set */ + header.len[DRequest::Section::DTCResultSet] = 0; + + bytes = encode_header(header); + const int len = bytes; + + /* exist and large enough, use. else free and malloc */ + int total_len = sizeof(BufferChain) + sizeof(struct iovec) + len; + if (buf == NULL) + { + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + else if (buf && buf->totalBytes < (int)(total_len - sizeof(BufferChain))) + { + FREE_IF(buf); + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + + /* usedBtytes never used for Packet's buf */ + buf->nextBuffer = NULL; + v = (struct iovec *)buf->data; + nv = 1; + char *p = buf->data + sizeof(struct iovec); + v->iov_base = p; + v->iov_len = len; + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, vi, DField::None); + p = encode_simple_section(p, ri, tdef->key_type()); + + // encode field set + memcpy(p, fs, 4); + p += 4; + + if (p - (char *)v->iov_base != len) + fprintf(stderr, "%s(%d): BAD ENCODER len=%ld must=%d\n", + __FILE__, __LINE__, (long)(p - (char *)v->iov_base), len); + + return 0; +} + +int Packet::encode_reload_config(const DTCTableDefinition *tdef, int sn) +{ + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = DRequest::Flag::KeepAlive; + header.cmd = DRequest::ReloadConfig; + + DTCVersionInfo vi; + // tablename & hash + vi.set_table_name(tdef->table_name()); + vi.set_table_hash(tdef->table_hash()); + vi.set_serial_nr(sn); + // app version + vi.set_tag(5, "dtcd"); + // lib version + vi.set_tag(6, "ctlib-v" DTC_VERSION); + vi.set_tag(9, tdef->field_type(0)); + + DTCRequestInfo ri; + // key + ri.set_key(DTCValue::Make(0)); + // ri.set_timeout(30); + + // field set + char fs[4] = {1, 0, 0, 0xFF}; + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(vi, DField::None); + + /* no table definition */ + header.len[DRequest::Section::table_definition] = 0; + + /* encode request info */ + header.len[DRequest::Section::RequestInfo] = + encoded_bytes_simple_section(ri, tdef->key_type()); + + /* no result info */ + header.len[DRequest::Section::ResultInfo] = 0; + + /* encode update info */ + header.len[DRequest::Section::UpdateInfo] = 0; + + /* encode condition info */ + header.len[DRequest::Section::ConditionInfo] = 0; + + /* full set */ + header.len[DRequest::Section::FieldSet] = 4; + + /* no result set */ + header.len[DRequest::Section::DTCResultSet] = 0; + + bytes = encode_header(header); + const int len = bytes; + + /* pool, exist and large enough, use. else free and malloc */ + int total_len = sizeof(BufferChain) + sizeof(struct iovec) + len; + if (buf == NULL) + { + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + else if (buf && buf->totalBytes < (int)(total_len - sizeof(BufferChain))) + { + FREE_IF(buf); + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + + /* usedBtytes never used for Packet's buf */ + buf->nextBuffer = NULL; + v = (struct iovec *)buf->data; + nv = 1; + char *p = buf->data + sizeof(struct iovec); + v->iov_base = p; + v->iov_len = len; + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, vi, DField::None); + p = encode_simple_section(p, ri, tdef->key_type()); + + // encode field set + memcpy(p, fs, 4); + p += 4; + + if (p - (char *)v->iov_base != len) + fprintf(stderr, "%s(%d): BAD ENCODER len=%ld must=%d\n", + __FILE__, __LINE__, (long)(p - (char *)v->iov_base), len); + + return 0; +} + +static char *EncodeBinary(char *p, const char *src, int len) +{ + if (len) + memcpy(p, src, len); + return p + len; +} + +static char *EncodeBinary(char *p, const DTCBinary &b) +{ + return EncodeBinary(p, b.ptr, b.len); +} + +int Packet::encode_fetch_data(TaskRequest &task) +{ + const DTCTableDefinition *tdef = task.table_definition(); + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = DRequest::Flag::KeepAlive; + header.cmd = task.flag_fetch_data() ? DRequest::Get : task.request_code(); + + // save & remove limit information + uint32_t limitStart = task.requestInfo.limit_start(); + uint32_t limitCount = task.requestInfo.limit_count(); + if (task.request_code() != DRequest::Replicate) + { + task.requestInfo.set_limit_start(0); + task.requestInfo.set_limit_count(0); + } + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(task.versionInfo, DField::None); + + /* no table definition */ + header.len[DRequest::Section::table_definition] = 0; + + /* encode request info */ + header.len[DRequest::Section::RequestInfo] = + encoded_bytes_simple_section(task.requestInfo, tdef->key_type()); + + /* no result info */ + header.len[DRequest::Section::ResultInfo] = 0; + + /* no update info */ + header.len[DRequest::Section::UpdateInfo] = 0; + + /* encode condition info */ + header.len[DRequest::Section::ConditionInfo] = + encoded_bytes_multi_key(task.multi_key_array(), task.table_definition()); + + /* full set */ + header.len[DRequest::Section::FieldSet] = + tdef->packed_field_set(task.flag_field_set_with_key()).len; + + /* no result set */ + header.len[DRequest::Section::DTCResultSet] = 0; + + bytes = encode_header(header); + const int len = bytes; + + /* pool, exist and large enough, use. else free and malloc */ + int total_len = sizeof(BufferChain) + sizeof(struct iovec) + len; + if (buf == NULL) + { + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + else if (buf && buf->totalBytes < total_len - (int)sizeof(BufferChain)) + { + FREE_IF(buf); + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + + buf->nextBuffer = NULL; + v = (struct iovec *)buf->data; + nv = 1; + char *p = buf->data + sizeof(struct iovec); + v->iov_base = p; + v->iov_len = len; + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, task.versionInfo, DField::None); + p = encode_simple_section(p, task.requestInfo, tdef->key_type()); + // restore limit info + task.requestInfo.set_limit_start(limitStart); + task.requestInfo.set_limit_count(limitCount); + p = encode_multi_key(p, task.multi_key_array(), task.table_definition()); + p = EncodeBinary(p, tdef->packed_field_set(task.flag_field_set_with_key())); + + if (p - (char *)v->iov_base != len) + fprintf(stderr, "%s(%d): BAD ENCODER len=%ld must=%d\n", + __FILE__, __LINE__, (long)(p - (char *)v->iov_base), len); + + return 0; +} + +int Packet::encode_pass_thru(DTCTask &task) +{ + const DTCTableDefinition *tdef = task.table_definition(); + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = DRequest::Flag::KeepAlive; + header.cmd = task.request_code(); + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(task.versionInfo, DField::None); + + /* no table definition */ + header.len[DRequest::Section::table_definition] = 0; + + /* encode request info */ + header.len[DRequest::Section::RequestInfo] = + encoded_bytes_simple_section(task.requestInfo, tdef->key_type()); + + /* no result info */ + header.len[DRequest::Section::ResultInfo] = 0; + + /* encode update info */ + header.len[DRequest::Section::UpdateInfo] = + task.request_operation() ? encoded_bytes_field_value(*task.request_operation()) : 0; + + /* encode condition info */ + header.len[DRequest::Section::ConditionInfo] = + task.request_condition() ? encoded_bytes_field_value(*task.request_condition()) : 0; + + /* full set */ + header.len[DRequest::Section::FieldSet] = + task.request_fields() ? encoded_bytes_field_set(*task.request_fields()) : 0; + + /* no result set */ + header.len[DRequest::Section::DTCResultSet] = 0; + + bytes = encode_header(header); + const int len = bytes; + + /* pool, exist and large enough, use. else free and malloc */ + int total_len = sizeof(BufferChain) + sizeof(struct iovec) + len; + if (buf == NULL) + { + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + else if (buf && buf->totalBytes < total_len - (int)sizeof(BufferChain)) + { + FREE_IF(buf); + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + + buf->nextBuffer = NULL; + v = (struct iovec *)buf->data; + nv = 1; + char *p = buf->data + sizeof(struct iovec); + v->iov_base = p; + v->iov_len = len; + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, task.versionInfo, DField::None); + p = encode_simple_section(p, task.requestInfo, tdef->key_type()); + if (task.request_operation()) + p = encode_field_value(p, *task.request_operation()); + if (task.request_condition()) + p = encode_field_value(p, *task.request_condition()); + if (task.request_fields()) + p = encode_field_set(p, *task.request_fields()); + + if (p - (char *)v->iov_base != len) + fprintf(stderr, "%s(%d): BAD ENCODER len=%ld must=%d\n", + __FILE__, __LINE__, (long)(p - (char *)v->iov_base), len); + + return 0; +} + +int Packet::encode_forward_request(TaskRequest &task) +{ + if (task.flag_pass_thru()) + return encode_pass_thru(task); + if (task.flag_fetch_data()) + return encode_fetch_data(task); + if (task.request_code() == DRequest::Get || task.request_code() == DRequest::Replicate) + return encode_fetch_data(task); + return encode_pass_thru(task); +} + +int Packet::encode_result(DTCTask &task, int mtu, uint32_t ts) +{ + const DTCTableDefinition *tdef = task.table_definition(); + // rp指向返回数据集 + ResultPacket *rp = task.result_code() >= 0 ? task.get_result_packet() : NULL; + BufferChain *rb = NULL; + int nrp = 0, lrp = 0, off = 0; + + if (mtu <= 0) + { + mtu = MAXPACKETSIZE; + } + + /* rp may exist but no result */ + if (rp && (rp->numRows || rp->totalRows)) + { + //rb指向数据结果集缓冲区起始位置 + rb = rp->bc; + if (rb) + rb->Count(nrp, lrp); + off = 5 - encoded_bytes_length(rp->numRows); + encode_length(rb->data + off, rp->numRows); + lrp -= off; + task.resultInfo.set_total_rows(rp->totalRows); + } + else + { + if (rp && rp->totalRows == 0 && rp->bc) + { + FREE(rp->bc); + rp->bc = NULL; + } + task.resultInfo.set_total_rows(0); + if (task.result_code() == 0) + { + task.set_error(0, NULL, NULL); + } + //任务出现错误的时候，可能结果集里面还有值，此时需要将结果集的buffer释放掉 + else if (task.result_code() < 0) + { + ResultPacket *resultPacket = task.get_result_packet(); + if (resultPacket) + { + if (resultPacket->bc) + { + FREE(resultPacket->bc); + resultPacket->bc = NULL; + } + } + } + } + if (ts) + { + task.resultInfo.set_time_info(ts); + } + task.versionInfo.set_serial_nr(task.request_serial()); + task.versionInfo.set_tag(6, "ctlib-v" DTC_VERSION); + if (task.result_key() == NULL && task.request_key() != NULL) + task.set_result_key(*task.request_key()); + + PacketHeader header; + + header.version = 1; + header.scts = 8; + header.flags = DRequest::Flag::KeepAlive | task.flag_multi_key_val(); + /* rp may exist but no result */ + header.cmd = (rp && (rp->numRows || rp->totalRows)) ? DRequest::DTCResultSet : DRequest::result_code; + + /* calculate version info */ + header.len[DRequest::Section::VersionInfo] = + encoded_bytes_simple_section(task.versionInfo, DField::None); + + /* copy table definition */ + header.len[DRequest::Section::table_definition] = + task.flag_table_definition() ? tdef->packed_definition().len : 0; + + /* no request info */ + header.len[DRequest::Section::RequestInfo] = 0; + + /* calculate result info */ + header.len[DRequest::Section::ResultInfo] = + encoded_bytes_simple_section(task.resultInfo, + tdef->field_type(0)); + + /* no update info */ + header.len[DRequest::Section::UpdateInfo] = 0; + + /* no condition info */ + header.len[DRequest::Section::ConditionInfo] = 0; + + /* no field set */ + header.len[DRequest::Section::FieldSet] = 0; + + /* copy result set */ + header.len[DRequest::Section::DTCResultSet] = lrp; + + bytes = encode_header(header); + if (bytes > mtu) + { + /* clear result set */ + nrp = 0; + lrp = 0; + rb = NULL; + rp = NULL; + /* set message size error */ + task.set_error(-EMSGSIZE, "encode_result", "encoded result exceed the maximum network packet size"); + /* re-encode resultinfo */ + header.len[DRequest::Section::ResultInfo] = + encoded_bytes_simple_section(task.resultInfo, + tdef->field_type(0)); + header.cmd = DRequest::result_code; + header.len[DRequest::Section::DTCResultSet] = 0; + /* FIXME: only work in LITTLE ENDIAN machine */ + bytes = encode_header(header); + } + + //non-result packet len + const int len = bytes - lrp; + + /* pool, exist and large enough, use. else free and malloc */ + int total_len = sizeof(BufferChain) + sizeof(struct iovec) * (nrp + 1) + len; + if (buf == NULL) + { + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + else if (buf && buf->totalBytes < total_len - (int)sizeof(BufferChain)) + { + FREE_IF(buf); + buf = (BufferChain *)MALLOC(total_len); + if (buf == NULL) + { + return -ENOMEM; + } + buf->totalBytes = total_len - sizeof(BufferChain); + } + + //发送实际数据集 + buf->nextBuffer = nrp ? rb : NULL; + v = (struct iovec *)buf->data; + char *p = buf->data + sizeof(struct iovec) * (nrp + 1); + v->iov_base = p; + v->iov_len = len; + nv = nrp + 1; + + for (int i = 1; i <= nrp; i++, rb = rb->nextBuffer) + { + v[i].iov_base = rb->data + off; + v[i].iov_len = rb->usedBytes - off; + off = 0; + } + + memcpy(p, &header, sizeof(header)); + p += sizeof(header); + p = encode_simple_section(p, task.versionInfo, DField::None); + if (task.flag_table_definition()) + p = EncodeBinary(p, tdef->packed_definition()); + p = encode_simple_section(p, task.resultInfo, tdef->field_type(0)); + + if (p - (char *)v->iov_base != len) + fprintf(stderr, "%s(%d): BAD ENCODER len=%ld must=%d\n", + __FILE__, __LINE__, (long)(p - (char *)v->iov_base), len); + + return 0; +} + +int Packet::encode_result(TaskRequest &task, int mtu) +{ + return encode_result((DTCTask &)task, mtu, task.Timestamp()); +} + +void Packet::free_result_buff() +{ + BufferChain *resbuff = buf->nextBuffer; + buf->nextBuffer = NULL; + + while (resbuff) + { + char *p = (char *)resbuff; + resbuff = resbuff->nextBuffer; + FREE(p); + } +} + +int Packet::Bytes(void) +{ + int sendbytes = 0; + for (int i = 0; i < nv; i++) + { + sendbytes += v[i].iov_len; + } + return sendbytes; +} diff --git a/src/search_local/index_storage/common/parse_cluster_config.cc b/src/search_local/index_storage/common/parse_cluster_config.cc new file mode 100644 index 0000000..a4e44a0 --- /dev/null +++ b/src/search_local/index_storage/common/parse_cluster_config.cc @@ -0,0 +1,295 @@ +/* + * ===================================================================================== + * + * Filename: parse_cluster_config.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include "config.h" +#include + +#include "parse_cluster_config.h" +#include "markup_stl.h" +#include "log.h" +#include "net_addr.h" +#include + +using namespace std; + +extern int errno; +extern DTCConfig *gConfig; + +namespace ClusterConfig +{ + + bool parse_cluster_config(std::string &strSelfName, std::vector *result, const char *buf, int len) + { + log_debug("%.*s", len, buf); + //配置中不允许有相同servername的节点出现，本set用于检查重复servername + std::set filter; + pair::iterator, bool> ret; + + string xmldoc(buf, len); + MarkupSTL xml; + if (!xml.set_doc(xmldoc.c_str())) + { + log_error("parse config file error"); + return false; + } + xml.reset_main_pos(); + + if (!xml.find_elem("serverlist")) + { + log_error("no serverlist info"); + return -1; + } + strSelfName = xml.get_attrib("selfname"); + log_debug("strSelfName is %s", strSelfName.c_str()); + + while (xml.find_child_elem("server")) + { + if (xml.into_elem()) + { + struct ClusterNode node; + node.name = xml.get_attrib("name"); + node.addr = xml.get_attrib("address"); + if (node.name.empty() || node.addr.empty()) + { + log_error("name:%s addr:%s may error\n", node.name.c_str(), node.addr.c_str()); + return false; + } + + if (strSelfName == node.name) + { + node.self = true; + } + else + { + node.self = false; + } + + ret = filter.insert(node.addr); + if (ret.second == false) + { + log_error("server address duplicate. name:%s addr:%s \n", node.name.c_str(), node.addr.c_str()); + return false; + } + result->push_back(node); + xml.out_of_elem(); + } + } + if (result->empty()) + log_notice("ClusterConfig is empty,dtc runing in normal mode"); + return true; + } + + //读取配置文件 + bool parse_cluster_config(std::vector *result) + { + bool bResult = false; + FILE *fp = fopen(CLUSTER_CONFIG_FILE, "rb"); + if (fp == NULL) + { + log_error("load %s failed:%m", CLUSTER_CONFIG_FILE); + return false; + } + + // Determine file length + fseek(fp, 0L, SEEK_END); + int nFileLen = ftell(fp); + fseek(fp, 0L, SEEK_SET); + + // Load string + allocator mem; + std::string strSelfName; + allocator::pointer pBuffer = mem.allocate(nFileLen + 1, NULL); + if (fread(pBuffer, nFileLen, 1, fp) == 1) + { + pBuffer[nFileLen] = '\0'; + bResult = parse_cluster_config(strSelfName, result, pBuffer, nFileLen); + } + fclose(fp); + mem.deallocate(pBuffer, 1); + return bResult; + } + + bool parse_cluster_config(std::string &strSelfName, std::map &dtcClusterMap, const char *buf, int len) + { + log_debug("%.*s", len, buf); + + string xmldoc(buf, len); + MarkupSTL xml; + if (!xml.set_doc(xmldoc.c_str())) + { + log_error("parse config file error"); + return false; + } + xml.reset_main_pos(); + + if (!xml.find_elem("serverlist")) + { + log_error("no serverlist info"); + return -1; + } + strSelfName = xml.get_attrib("selfname"); + log_debug("strSelfName is %s", strSelfName.c_str()); + + while (xml.find_child_elem("server")) + { + if (xml.into_elem()) + { + + std::string name = xml.get_attrib("name"); + std::string addr = xml.get_attrib("address"); + if (name.empty() || addr.empty()) + { + log_error("name:%s addr:%s may error\n", name.c_str(), addr.c_str()); + return false; + } + + std::pair::iterator, bool> ret; + ret = dtcClusterMap.insert(std::pair(name, addr)); + if (ret.second) + { + log_debug("insert dtcClusterMap success: name[%s]addr[%s]", name.c_str(), addr.c_str()); + } + else + { + log_error("insert dtcClusterMap fail,maybe dtc name duplicated: name[%s]addr[%s]", name.c_str(), addr.c_str()); + } + + xml.out_of_elem(); + } + } + return true; + } + + //保存vector到配置文件 + bool save_cluster_config(std::vector *result, std::string &strSelfName) + { + MarkupSTL xml; + xml.set_doc("\r\n"); + xml.add_elem("serverlist"); + xml.add_attrib("selfname", strSelfName.c_str()); + xml.into_elem(); + + std::vector::iterator it; + for (it = result->begin(); it < result->end(); it++) + { + xml.add_elem("server"); + xml.add_attrib("name", it->name.c_str()); + xml.add_attrib("address", it->addr.c_str()); + } + return xml.Save(CLUSTER_CONFIG_FILE); + } + + //检查是否有配置文件，如果没有则生成配置文件 + bool check_and_create(const char *filename) + { + if (filename == NULL) + filename = CLUSTER_CONFIG_FILE; + + if (access(filename, F_OK)) + { + //配置文件不存在,创建配置文件 + if (errno == ENOENT) + { + log_notice("%s didn't exist,create it.", filename); + MarkupSTL xml; + xml.set_doc("\r\n"); + xml.add_elem("serverlist"); + xml.add_attrib("selfname", ""); + xml.into_elem(); + return xml.Save(filename); + } + log_error("access %s error:%m\n", filename); + return false; + } + else + return true; + } + + bool change_node_address(std::string servername, std::string newaddress) + { + MarkupSTL xml; + if (!xml.Load(CLUSTER_CONFIG_FILE)) + { + log_error("load config file: %s error", CLUSTER_CONFIG_FILE); + return false; + } + xml.reset_main_pos(); + while (xml.find_child_elem("server")) + { + if (!xml.into_elem()) + { + log_error("xml.into_elem() error"); + return false; + } + + string name = xml.get_attrib("name"); + string address = xml.get_attrib("address"); + if (address == newaddress) + { + log_error("new address:%s is exist in Cluster Config", newaddress.c_str()); + return false; + } + + if (!xml.out_of_elem()) + { + log_error("xml.OutElem() error"); + return false; + } + } + + xml.reset_main_pos(); + while (xml.find_child_elem("server")) + { + if (!xml.into_elem()) + { + log_error("xml.into_elem() error"); + return false; + } + + string name = xml.get_attrib("name"); + if (name != servername) + { + if (!xml.out_of_elem()) + { + log_error("xml.OutElem() error"); + return false; + } + continue; + } + + string oldaddress = xml.get_attrib("address"); + if (oldaddress == newaddress) + { + log_error("server name:%s ip address:%s not change", + servername.c_str(), newaddress.c_str()); + return false; + } + + if (!xml.set_attrib("address", newaddress.c_str())) + { + log_error("change address error.server name: %s old address: %s new address: %s", + servername.c_str(), oldaddress.c_str(), newaddress.c_str()); + return false; + } + return xml.Save(CLUSTER_CONFIG_FILE); + } + log_error("servername: \"%s\" not found", servername.c_str()); + return false; + } +} // namespace ClusterConfig diff --git a/src/search_local/index_storage/common/parse_cluster_config.h b/src/search_local/index_storage/common/parse_cluster_config.h new file mode 100644 index 0000000..a990bb2 --- /dev/null +++ b/src/search_local/index_storage/common/parse_cluster_config.h @@ -0,0 +1,44 @@ +/* + * ===================================================================================== + * + * Filename: parse_cluster_config.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef PARSE_CLUSTER_CONFIG_H__ +#define PARSE_CLUSTER_CONFIG_H__ + +#include +#include +#include +namespace ClusterConfig +{ +#define CLUSTER_CONFIG_FILE "../conf/ClusterConfig.xml" + + struct ClusterNode + { + std::string name; + std::string addr; + bool self; + }; + + bool parse_cluster_config(std::string &strSelfName, std::vector *result, const char *buf, int len); + bool parse_cluster_config(std::vector *result); + bool save_cluster_config(std::vector *result, std::string &strSelfName); + bool check_and_create(const char *filename = NULL); + bool change_node_address(std::string servername, std::string newaddress); + bool get_local_ip_by_ip_file(); + bool parse_cluster_config(std::string &strSelfName, std::map &dtcClusterMap, const char *buf, int len); +} // namespace ClusterConfig + +#endif diff --git a/src/search_local/index_storage/common/pipequeue.h b/src/search_local/index_storage/common/pipequeue.h new file mode 100644 index 0000000..371db9c --- /dev/null +++ b/src/search_local/index_storage/common/pipequeue.h @@ -0,0 +1,170 @@ +/* + * ===================================================================================== + * + * Filename: pipequeue.h + * + * Description: + * + * Version: 1.0 + * Created: 10/05/2014 17:40:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#include +#include "mem_check.h" +#include "poll_thread.h" + +// typename T must be simple data type with 1,2,4,8,16 bytes +template +class PipeQueueReader : PollerObject +{ +private: + T buf[32]; + int i; // first pending + int n; // last pending + +private: + virtual void hangup_notify(void) {} + virtual void input_notify(void) + { + T p; + int n = 0; + while (++n <= 64 && (p = Pop()) != NULL) + static_cast(this)->task_notify(p); + } + inline T Pop(void) + { + if (i >= n) + { + i = 0; + n = read(netfd, buf, sizeof(buf)); + if (n <= 0) + return NULL; + n /= sizeof(T); + } + return buf[i++]; + } + +public: + inline PipeQueueReader() : i(0), n(0) {} + virtual ~PipeQueueReader() {} + inline int attach_poller(PollerUnit *thread, int fd) + { + netfd = fd; + enable_input(); + return PollerObject::attach_poller(thread); + } +}; + +template +class PipeQueueWriter : PollerObject, TimerObject +{ +private: + PollThread *owner; + T *buf; + int i; // first unsent + int n; // last unsent + int m; // buf size + +private: + virtual void hangup_notify(void) {} + virtual void output_notify(void) + { + if (n > 0) + { + // i always less than n + int r = write(netfd, buf + i, (n - i) * sizeof(T)); + if (r > 0) + { + i += r / sizeof(T); + if (i >= n) + i = n = 0; + } + } + if (n == 0) + { + disable_output(); + disable_timer(); + } + else + enable_output(); + delay_apply_events(); + } + virtual void timer_notify(void) + { + output_notify(); + } + +public: + inline PipeQueueWriter() : i(0), n(0), m(16) { buf = (T *)MALLOC(m * sizeof(T)); } + virtual ~PipeQueueWriter() { FREE_IF(buf); } + inline int attach_poller(PollThread *thread, int fd) + { + owner = thread; + netfd = fd; + return PollerObject::attach_poller(thread); + } + inline int Push(T p) + { + if (n >= m) + { + if (i > 0) + { + memmove(buf, buf + i, (n - i) * sizeof(T)); + n -= i; + } + else + { + T *newbuf = (T *)REALLOC(buf, 2 * m * sizeof(T)); + if (newbuf == NULL) + return -1; + buf = newbuf; + m *= 2; + } + i = 0; + } + buf[n++] = p; + + // force flush every 16 pending task + if (n - i >= 16 && n % 16 == 0) + output_notify(); + // some task pending, trigger ready timer + if (n > 0) + attach_ready_timer(owner); + return 0; + } +}; + +template +class PipeQueue : public PipeQueueReader +{ +private: + PipeQueueWriter writer; + +public: + inline PipeQueue() {} + inline ~PipeQueue() {} + inline int attach_poller(PollThread *fr, PollThread *to, int fd[2]) + { + PipeQueueReader::attach_poller(to, fd[0]); + writer.attach_poller(fr, fd[1]); + return 0; + } + inline int attach_poller(PollThread *fr, PollThread *to) + { + int fd[2]; + int ret = pipe(fd); + if (ret != 0) + return ret; + + PipeQueueReader::attach_poller(to, fd[0]); + writer.attach_poller(fr, fd[1]); + return 0; + } + inline int Push(T p) { return writer.Push(p); } +}; diff --git a/src/search_local/index_storage/common/pipetask.h b/src/search_local/index_storage/common/pipetask.h new file mode 100644 index 0000000..b1a40a6 --- /dev/null +++ b/src/search_local/index_storage/common/pipetask.h @@ -0,0 +1,113 @@ +/* + * ===================================================================================== + * + * Filename: pipetask.h + * + * Description: + * + * Version: 1.0 + * Created: 10/05/2014 17:40:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#ifndef __H_DTC_PIPETASK_TEMP_H__ +#define __H_DTC_PIPETASK_TEMP_H__ + +#include "log.h" +#include "pipequeue.h" +#include "compiler.h" + +template +class TaskDispatcher; +template +class ReplyDispatcher; +template +class RequestOutput; + +template +class TaskIncomingPipe : public PipeQueue > +{ +public: + TaskIncomingPipe(void) {} + virtual ~TaskIncomingPipe() {} + inline void task_notify(T p) + { + proc->task_notify(p); + } + +public: + TaskOneWayDispatcher *proc; +}; + +template +class TaskReturnPipe : public PipeQueue > +{ +public: + TaskReturnPipe(){}; + virtual ~TaskReturnPipe(){}; + inline void task_notify(T *p) + { + p->reply_notify(); + } +}; + +template +class TaskPipe : public TaskDispatcher, + public ReplyDispatcher +{ +private: + static LinkQueue *> pipelist; + +public: + inline TaskPipe() { pipelist.Push(this); } + ~TaskPipe() {} + + virtual void task_notify(T *p) + { + p->push_reply_dispatcher(this); + incQueue.Push(p); + } + virtual void reply_notify(T *p) + { + retQueue.Push(p); + } + inline int bind_dispatcher(RequestOutput *fr, TaskDispatcher *to) + { +#if 0 + log_debug("Bind taskpipe from thread %s to thread %s", + fr->owner_thread()->Name(), to->owner_thread()->Name()); +#endif + TaskDispatcher::owner = fr->owner_thread(); + + incQueue.attach_poller(fr->owner_thread(), to->owner_thread()); + retQueue.attach_poller(to->owner_thread(), fr->owner_thread()); + + fr->bind_dispatcher(this); + incQueue.proc = to; + fr->disable_queue(); + return 0; + } + + static inline void destroy_all(void) + { + TaskPipe *p; + while ((p = pipelist.Pop()) != NULL) + { + delete p; + } + } + +private: + TaskIncomingPipe incQueue; + TaskReturnPipe retQueue; +}; + +template +LinkQueue *> TaskPipe::pipelist; + +#endif diff --git a/src/search_local/index_storage/common/plugin_agent_mgr.h b/src/search_local/index_storage/common/plugin_agent_mgr.h new file mode 100644 index 0000000..888a53a --- /dev/null +++ b/src/search_local/index_storage/common/plugin_agent_mgr.h @@ -0,0 +1,87 @@ +/* + * ===================================================================================== + * + * Filename: plugin_mgr.h + * + * Description: plugin manager + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PLUGIN_MANAGER_H__ +#define __DTC_PLUGIN_MANAGER_H__ + +#include +#include +#include + +#include "non_copyable.h" +#include "plugin_request.h" +#include "plugin_timer.h" + +class DTCConfig; +class PluginAgentListenerPool; +class PluginWorkerThread; + +class PluginAgentManager : private noncopyable +{ +public: + PluginAgentManager(void); + ~PluginAgentManager(void); + + static PluginAgentManager *Instance(void); + static void Destroy(void); + + int open(void); + int close(void); + + inline dll_func_t *get_dll(void) + { + return _dll; + } + + inline worker_notify_t *get_worker_notifier(void) + { + return _worker_notifier; + } + +public: + char *plugin_log_path; + char *plugin_log_name; + int plugin_log_level; + int plugin_log_size; + int plugin_worker_number; + int plugin_timer_notify_interval; + + inline void set_plugin_name(const char *plugin_name) { _plugin_name = strdup(plugin_name); } + inline void set_plugin_conf_file(const char *plugin_conf_file) { PluginGlobal::_plugin_conf_file = strdup(plugin_conf_file); } + +protected: +protected: +private: + int register_plugin(const char *file_name); + void unregister_plugin(void); + int create_worker_pool(int worker_number); + int load_plugin_depend(); + int create_plugin_timer(const int interval); + +private: + dll_func_t *_dll; + DTCConfig *_config; + char *_plugin_name; + PluginAgentListenerPool *_plugin_listener_pool; + Thread **_plugin_worker_pool; + worker_notify_t *_worker_notifier; + int _worker_number; + plugin_log_init_t _sb_if_handle; + Thread *_plugin_timer; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_decoder.cc b/src/search_local/index_storage/common/plugin_decoder.cc new file mode 100644 index 0000000..0f242d2 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_decoder.cc @@ -0,0 +1,313 @@ +/* + * ===================================================================================== + * + * Filename: plugin_net.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "plugin_decoder.h" +#include "mem_check.h" +#include "plugin_request.h" +#include "log.h" +#include "plugin_mgr.h" + +int PluginReceiver::parse_protocol(PluginStream *request) +{ + + log_debug("enter PluginReceiver::parse_protocol!!!!!!!!"); + const int max_recv_len = PluginGlobal::_max_plugin_recv_len; + int &remain_len = request->_recv_remain_len; + char *&recv_buf = request->_recv_buf; + int &recv_len = request->_recv_len; + int &real_len = request->_real_len; + int &all_len = request->_all_len; + + //get real length + if (0 == real_len) + { + real_len = _dll->handle_input(recv_buf, recv_len, &(request->_skinfo)); + } + + if (real_len < 0) + { + _stage = NET_FATAL_ERROR; + log_error("invoke handle_input failed, fd[%d], length[%d], net stage[%d]", _fd, recv_len, _stage); + return _stage; + } + + if (0 == real_len) + { + if (recv_len < max_recv_len) + { + remain_len = max_recv_len - recv_len; + } + else + { + all_len = max_recv_len + recv_len; + _all_len_changed = 1; + } + + _stage = NET_RECVING; + goto parse_out; + } + + if (real_len > recv_len) + { + remain_len = real_len - recv_len; + + if (all_len < real_len) + { + all_len = real_len; + _all_len_changed = 1; + } + + _stage = NET_RECVING; + goto parse_out; + } + + if (real_len < recv_len) + { + request->mark_multi_packet(); + } + else + { + request->mark_single_packet(); + } + + if (all_len < recv_len) + { + all_len = recv_len; + _all_len_changed = 1; + } + + remain_len = 0; + _stage = NET_RECV_DONE; + +parse_out: + if (_all_len_changed) + { + _all_len_changed = 0; + if (REALLOC(recv_buf, all_len) == NULL) + { + _stage = NET_FATAL_ERROR; + log_error("recv error, fd[%d], length[%d], net stage[%d], msg:%s", _fd, recv_len, _stage, strerror(errno)); + } + } + + return _stage; +} + +int PluginReceiver::recv(PluginStream *request) +{ + const int max_recv_len = PluginGlobal::_max_plugin_recv_len; + int &remain_len = request->_recv_remain_len; + char *&recv_buf = request->_recv_buf; + int &recv_len = request->_recv_len; + int &all_len = request->_all_len; + int curr_len = 0; + + //init changed flag + _all_len_changed = 0; + + if ((NET_IDLE == _stage) && (NULL == recv_buf)) + { + recv_buf = (char *)MALLOC(max_recv_len); + if (NULL == recv_buf) + { + _stage = NET_FATAL_ERROR; + log_error("malloc memory failed, size[%d], msg:%s", max_recv_len, strerror(errno)); + return _stage; + } + + all_len = max_recv_len; + remain_len = max_recv_len; + _all_len_changed = 1; + } + + curr_len = ::read(_fd, recv_buf + recv_len, remain_len); + switch (curr_len) + { + case -1: + //read error + if (errno != EAGAIN && errno != EINTR && errno != EINPROGRESS) + { + _stage = NET_FATAL_ERROR; + log_error("recv failed from fd[%d], net stage[%d], msg[%s]", _fd, _stage, strerror(errno)); + return _stage; + } + return _stage; + + case 0: + //disconnect by user + _stage = NET_DISCONNECT; + return NET_DISCONNECT; + + default: + recv_len += curr_len; + parse_protocol(request); + break; + } + + return _stage; +} + +int PluginReceiver::recvfrom(PluginDatagram *request, int mtu) +{ + char *&recv_buf = request->_recv_buf; + int &recv_len = request->_recv_len; + int &real_len = request->_real_len; + + int max_len = -1; + ioctl(_fd, SIOCINQ, &max_len); + if (max_len < 0) + { + log_error("%s", "next packet size unknown"); + return -1; + } + + /* treat 0 as max packet, because we can't ditiguish the no-packet & zero-packet */ + if (0 == max_len) + { + max_len = mtu; + } + + recv_buf = (char *)MALLOC(max_len); + if (NULL == recv_buf) + { + log_error("allocate packet buffer[%d] failed, msg[no enough memory]", recv_len); + return -1; + } + + void *&req_addr = request->_addr; + socklen_t &req_addr_len = request->_addr_len; + recv_len = ::recvfrom(_fd, recv_buf, max_len, MSG_DONTWAIT | MSG_TRUNC, (struct sockaddr *)req_addr, &req_addr_len); + + if (recv_len <= 0) + { + if (errno != EAGAIN) + { + log_notice("recvfrom error: size=%d errno=%d", recv_len, errno); + } + + return -1; + } + + request->_skinfo.remote_ip = ((struct sockaddr_in *)req_addr)->sin_addr.s_addr; + request->_skinfo.remote_port = ((struct sockaddr_in *)req_addr)->sin_port; + + real_len = _dll->handle_input(recv_buf, recv_len, &(request->_skinfo)); + + if ((real_len <= 0) || (real_len > recv_len)) + { + log_notice("the recv buffer is invalid."); + return -1; + } + + return 0; +} + +int PluginSender::send(PluginStream *request) +{ + const int send_len = request->_send_len; + int &sent_len = request->_sent_len; + const char *send_buf = request->_send_buf; + + if (0 == send_len || NULL == send_buf) + { + _stage = NET_FATAL_ERROR; + log_error("send length[%d] and send buffer[%p] is invalid.", send_len, send_buf); + return _stage; + } + + int ret = ::write(_fd, send_buf + sent_len, send_len - sent_len); + if (ret < 0) + { + if (errno == EAGAIN || errno == EINTR || errno == EINPROGRESS) + { + _stage = NET_SENDING; + log_debug("*STEP: send data pending, _fd[%d], net stage[%d], sendlen[%d], sentlen[%d]", + _fd, _stage, send_len, sent_len); + + return _stage; + } + + _stage = NET_FATAL_ERROR; + log_error("*STEP: send data error, _fd[%d], net stage[%d], msg[%m]", _fd, _stage); + return _stage; + } + + if (sent_len + ret == send_len) + { + sent_len += ret; + _stage = NET_SEND_DONE; + log_debug("*STEP: send data done, fd[%d], net stage[%d], sent len[%d], send len[%d]", _fd, _stage, sent_len, send_len); + return _stage; + } + + if (sent_len + ret < send_len) + { + sent_len += ret; + _stage = NET_SENDING; + log_debug("*STEP: sending data, fd[%d], net stage[%d], sent len[%d], send len[%d]", _fd, _stage, sent_len, send_len); + return _stage; + } + + _stage = NET_FATAL_ERROR; + log_error("*STEP: send data exception, fd[%d], net stage[%d], sent len[%d], send len[%d]", _fd, _stage, sent_len + ret, send_len); + + return _stage; +} + +int PluginSender::sendto(PluginDatagram *request) +{ + char *&send_buf = request->_send_buf; + int send_len = request->_send_len; + int &sent_len = request->_sent_len; + struct iovec iov; + struct msghdr msgh; + + /* init send buffer & send length*/ + iov.iov_base = send_buf; + iov.iov_len = send_len; + + /* init msghdr */ + msgh.msg_name = (struct sockaddr_in *)(request->_addr); + msgh.msg_namelen = request->_addr_len; + msgh.msg_iov = &iov; + msgh.msg_iovlen = 1; + msgh.msg_control = NULL; + msgh.msg_controllen = 0; + msgh.msg_flags = 0; + + sent_len = sendmsg(_fd, &msgh, MSG_DONTWAIT | MSG_NOSIGNAL); + + if ((sent_len <= 0) || (sent_len < send_len)) + { + log_error("sendto data to client failed, sent length:%d, send length:%d, msg:%s", sent_len, send_len, strerror(errno)); + return -1; + } + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_decoder.h b/src/search_local/index_storage/common/plugin_decoder.h new file mode 100644 index 0000000..007a2ba --- /dev/null +++ b/src/search_local/index_storage/common/plugin_decoder.h @@ -0,0 +1,102 @@ +/* + * ===================================================================================== + * + * Filename: plugin_decoder.h + * + * Description: plugin network state machine management + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PLUGIN_DECODER_H__ +#define __DTC_PLUGIN_DECODER_H__ + +#include "non_copyable.h" +#include "plugin_global.h" +#include "plugin_request.h" + +typedef enum +{ + NET_IDLE, + NET_FATAL_ERROR, + NET_DISCONNECT, + NET_RECVING, + NET_SENDING, + NET_SEND_DONE, + NET_RECV_DONE, + +} net_state_t; + +class PluginReceiver : private noncopyable +{ +public: + PluginReceiver(int fd, dll_func_t *dll) : _fd(fd), _stage(NET_IDLE), _dll(dll), _all_len_changed(0) + { + } + ~PluginReceiver(void) + { + } + + int recv(PluginStream *request); + int recvfrom(PluginDatagram *request, int mtu); + inline int proc_remain_packet(PluginStream *request) + { + request->recalc_multipacket(); + return parse_protocol(request); + } + + inline void set_stage(net_state_t stage) + { + _stage = stage; + } + +public: +protected: +protected: +private: + int parse_protocol(PluginStream *request); + +private: + int _fd; + net_state_t _stage; + dll_func_t *_dll; + int _all_len_changed; +}; + +class PluginSender : private noncopyable +{ +public: + PluginSender(int fd, dll_func_t *dll) : _fd(fd), _stage(NET_IDLE), _dll(dll) + { + } + + ~PluginSender(void) + { + } + + int send(PluginStream *request); + int sendto(PluginDatagram *request); + + inline void set_stage(net_state_t stage) + { + _stage = stage; + } + +public: +protected: +protected: +private: +private: + int _fd; + net_state_t _stage; + dll_func_t *_dll; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_dgram.cc b/src/search_local/index_storage/common/plugin_dgram.cc new file mode 100644 index 0000000..8dbb81d --- /dev/null +++ b/src/search_local/index_storage/common/plugin_dgram.cc @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "plugin_agent_mgr.h" +#include "plugin_dgram.h" +#include "plugin_unit.h" +#include "poll_thread.h" +#include "mem_check.h" +#include "log.h" + +extern "C" +{ + extern unsigned int get_local_ip(); +} + +static int GetSocketFamily(int fd) +{ + struct sockaddr addr; + bzero(&addr, sizeof(addr)); + socklen_t alen = sizeof(addr); + getsockname(fd, &addr, &alen); + return addr.sa_family; +} + +PluginDgram::PluginDgram(PluginDecoderUnit *plugin_decoder, int fd) : PollerObject(plugin_decoder->owner_thread(), fd), + mtu(0), + _addr_len(0), + _owner(plugin_decoder), + _worker_notifier(NULL), + _plugin_receiver(fd, PluginAgentManager::Instance()->get_dll()), + _plugin_sender(fd, PluginAgentManager::Instance()->get_dll()), + _local_ip(0) +{ +} + +PluginDgram::~PluginDgram() +{ +} + +int PluginDgram::Attach() +{ + /* init local ip */ + _local_ip = get_local_ip(); + + switch (GetSocketFamily(netfd)) + { + default: + case AF_UNIX: + mtu = 16 << 20; + _addr_len = sizeof(struct sockaddr_un); + break; + case AF_INET: + mtu = 65535; + _addr_len = sizeof(struct sockaddr_in); + break; + case AF_INET6: + mtu = 65535; + _addr_len = sizeof(struct sockaddr_in6); + break; + } + + //get worker notifier + _worker_notifier = PluginAgentManager::Instance()->get_worker_notifier(); + if (NULL == _worker_notifier) + { + log_error("worker notifier is invalid."); + return -1; + } + + enable_input(); + + return attach_poller(); +} + +//server peer +int PluginDgram::recv_request(void) +{ + //create dgram request + PluginDatagram *dgram_request = NULL; + NEW(PluginDatagram(this, PluginAgentManager::Instance()->get_dll()), dgram_request); + if (NULL == dgram_request) + { + log_error("create PluginRequest for dgram failed, msg:%s", strerror(errno)); + return -1; + } + + //set request info + dgram_request->_skinfo.sockfd = netfd; + dgram_request->_skinfo.type = SOCK_DGRAM; + dgram_request->_skinfo.local_ip = _local_ip; + dgram_request->_skinfo.local_port = 0; + dgram_request->_incoming_notifier = _owner->get_incoming_notifier(); + dgram_request->_addr_len = _addr_len; + dgram_request->_addr = MALLOC(_addr_len); + if (NULL == dgram_request->_addr) + { + log_error("malloc failed, msg:%m"); + DELETE(dgram_request); + return -1; + } + + if (_plugin_receiver.recvfrom(dgram_request, mtu) != 0) + { + DELETE(dgram_request); + return -1; + } + + dgram_request->set_time_info(); + if (_worker_notifier->Push(dgram_request) != 0) + { + log_error("push plugin request failed, fd[%d]", netfd); + DELETE(dgram_request); + return -1; + } + + return 0; +} + +void PluginDgram::input_notify(void) +{ + if (recv_request() < 0) + /* */; +} diff --git a/src/search_local/index_storage/common/plugin_dgram.h b/src/search_local/index_storage/common/plugin_dgram.h new file mode 100644 index 0000000..bf6d49f --- /dev/null +++ b/src/search_local/index_storage/common/plugin_dgram.h @@ -0,0 +1,54 @@ +#ifndef __PLUGIN__DGRAM_H__ +#define __PLUGIN__DGRAM_H__ + +#include + +#include "poller.h" +#include "timer_list.h" +#include "plugin_decoder.h" +#include "plugin_unit.h" + +class PluginDatagram; + +class PluginDgram : public PollerObject +{ +public: + PluginDgram(PluginDecoderUnit *, int fd); + virtual ~PluginDgram(); + + virtual int Attach(void); + inline int send_response(PluginDatagram *plugin_request) + { + int ret = 0; + _owner->record_request_time(plugin_request->_response_timer.live()); + + if (!plugin_request->recv_only()) + { + ret = _plugin_sender.sendto(plugin_request); + } + + DELETE(plugin_request); + return ret; + } + +private: + virtual void input_notify(void); + +protected: + int recv_request(void); + int init_request(void); + +private: + int mtu; + int _addr_len; + PluginDecoderUnit *_owner; + worker_notify_t *_worker_notifier; + PluginReceiver _plugin_receiver; + PluginSender _plugin_sender; + uint32_t _local_ip; + +private: + int init_socket_info(void); +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_global.cc b/src/search_local/index_storage/common/plugin_global.cc new file mode 100644 index 0000000..e39ccd8 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_global.cc @@ -0,0 +1,64 @@ +/* + * ===================================================================================== + * + * Filename: plugin_global.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "mem_check.h" +#include "plugin_global.h" + +const char *PluginGlobal::_internal_name[2] = {(const char *)"dtc_internal", NULL}; +const int PluginGlobal::_max_worker_number = 1024; +const int PluginGlobal::_default_worker_number = 1; +const int PluginGlobal::_max_plugin_recv_len = 512; +int PluginGlobal::saved_argc = 0; +char **PluginGlobal::saved_argv = NULL; +char *PluginGlobal::_plugin_conf_file = NULL; +int PluginGlobal::_idle_timeout = 10; +int PluginGlobal::_single_incoming_thread = 0; +int PluginGlobal::_max_listen_count = 256; +int PluginGlobal::_max_request_window = 0; +char *PluginGlobal::_bind_address = NULL; +int PluginGlobal::_udp_recv_buffer_size = 0; +int PluginGlobal::_udp_send_buffer_size = 0; + +void PluginGlobal::dup_argv(int argc, char **argv) +{ + saved_argc = argc; + saved_argv = (char **)MALLOC(sizeof(char *) * (argc + 1)); + + if (!saved_argv) + return; + + saved_argv[argc] = NULL; + + while (--argc >= 0) + { + saved_argv[argc] = STRDUP(argv[argc]); + } +} + +void PluginGlobal::free_argv(void) +{ + char **argv; + + for (argv = saved_argv; *argv; argv++) + free(*argv); + + FREE_IF(saved_argv); + saved_argv = NULL; +} diff --git a/src/search_local/index_storage/common/plugin_global.h b/src/search_local/index_storage/common/plugin_global.h new file mode 100644 index 0000000..92f3ede --- /dev/null +++ b/src/search_local/index_storage/common/plugin_global.h @@ -0,0 +1,139 @@ +/* + * ===================================================================================== + * + * Filename: plugin_global.h + * + * Description: plugin global var + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PLUGIN_GLOBAL_H__ +#define __DTC_PLUGIN_GLOBAL_H__ + +#include +#include + +#include "non_copyable.h" + +#define PLUGIN_STOP_REQUEST (plugin_request_t *)0xFFFFFFFF +#define SERVER_BENCH_SO_FILE "../lib/sb_interface.so" +#define DTC_API_SO_FILE "../bin/libdtc.so" + +#define DLFUNC_NO_ERROR(h, v, type, name) \ + do \ + { \ + v = (type)dlsym(h, name); \ + dlerror(); \ + } while (0) + +#define DLFUNC(h, v, type, name) \ + do \ + { \ + v = (type)dlsym(h, name); \ + if ((error = dlerror()) != NULL) \ + { \ + log_error("dlsym error, %s", error); \ + dlclose(h); \ + h = NULL; \ + goto out; \ + } \ + } while (0) + +typedef enum plugin_thread_type +{ + PROC_MAIN = 0, + PROC_CONN, + PROC_WORK +} plugin_thread_type_t; + +typedef enum plugin_client_type +{ + PLUGIN_STREAM, + PLUGIN_DGRAM, + PLUGIN_UNKNOW +} plugin_client_type_t; + +enum +{ + PLUGIN_RECV_ONLY = 1, + PLUGIN_DISCONNECT = 2, +}; + +typedef struct skinfo_struct +{ + int sockfd; //fd + int type; //类型 + int64_t recvtm; //接收时间 + int64_t sendtm; //发送时间 + + time_t tasktm; //任务开始时间 + + u_int local_ip; //本地ip + u_short local_port; //本地port + u_int remote_ip; //对端ip + u_short remote_port; //对端port + uint64_t flags; //预留标志位，以bit操作，0x01:不发送回包，0x00:需要发送回包 +} skinfo_t; + +typedef int (*handle_init_t)(int, char **, int); +typedef int (*handle_input_t)(const char *, int, const skinfo_t *); +typedef int (*handle_process_t)(char *, int, char **, int *, const skinfo_t *); +typedef int (*handle_open_t)(char **, int *, const skinfo_t *); +typedef int (*handle_close_t)(const skinfo_t *); +typedef void (*handle_fini_t)(int); +typedef int (*handle_timer_notify_t)(int, void **); +typedef int (*plugin_log_init_t)(const char *, int, u_int, const char *); + +typedef struct dll_func_struct +{ + void *handle; + handle_init_t handle_init; + handle_input_t handle_input; + handle_process_t handle_process; + handle_open_t handle_open; + handle_close_t handle_close; + handle_fini_t handle_fini; + handle_timer_notify_t handle_timer_notify; +} dll_func_t; + +class PluginGlobal : private noncopyable +{ +public: + PluginGlobal(void); + ~PluginGlobal(void); + + void dup_argv(int argc, char **argv); + void free_argv(void); + +public: + static const char *_internal_name[2]; + static const int _max_worker_number; + static const int _default_worker_number; + static const int _max_plugin_recv_len; + static int saved_argc; + static char **saved_argv; + static char *_plugin_conf_file; + // for agent_plugin + static int _idle_timeout; + static int _single_incoming_thread; + static int _max_listen_count; + static int _max_request_window; + static char *_bind_address; + static int _udp_recv_buffer_size; + static int _udp_send_buffer_size; + +protected: +protected: +private: +private: +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_listener_pool.cc b/src/search_local/index_storage/common/plugin_listener_pool.cc new file mode 100644 index 0000000..3c0e2ec --- /dev/null +++ b/src/search_local/index_storage/common/plugin_listener_pool.cc @@ -0,0 +1,218 @@ +#include + +#include "plugin_listener_pool.h" +#include "listener.h" +#include "poll_thread.h" +#include "plugin_unit.h" +#include "poll_thread.h" +#include "unix_socket.h" +#include "config.h" +#include "log.h" +#include "mem_check.h" + +PluginListenerPool::PluginListenerPool(void) +{ + memset(_listener, 0, sizeof(_listener)); + memset(_thread, 0, sizeof(_thread)); + memset(_decoder, 0, sizeof(_decoder)); + memset(_udpfd, 0xff, sizeof(_udpfd)); +} + +PluginListenerPool::~PluginListenerPool(void) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_thread[i]) + { + _thread[i]->interrupt(); + } + + DELETE(_listener[i]); + DELETE(_decoder[i]); + + if (_udpfd[i] >= 0) + { + close(_udpfd[i]); + } + } +} + +int PluginListenerPool::init_decoder(int n, int idle) +{ + if (_thread[n] == NULL) + { + char name[16]; + snprintf(name, sizeof(name) - 1, "plugin_inc%d", n); + + _thread[n] = NULL; + NEW(PollThread(name), _thread[n]); + if (NULL == _thread[n]) + { + log_error("create PollThread object failed, %m"); + return -1; + } + if (_thread[n]->initialize_thread() == -1) + { + return -1; + } + + _decoder[n] = NULL; + NEW(PluginDecoderUnit(_thread[n], idle), _decoder[n]); + if (NULL == _decoder[n]) + { + log_error("create PluginDecoderUnit object failed, %m"); + return -1; + } + + if (_decoder[n]->attach_incoming_notifier() != 0) + { + log_error("attach incoming notifier failed."); + return -1; + } + } + + return 0; +} + +int PluginListenerPool::Bind(DTCConfig *gc) +{ + int succ_count = 0; + + int idle = gc->get_int_val("cache", "idle_timeout", 100); + if (idle < 0) + { + log_notice("idle_timeout invalid, use default value: 100"); + idle = 100; + } + + int single = gc->get_int_val("cache", "SingleIncomingThread", 0); + int backlog = gc->get_int_val("cache", "MaxListenCount", 256); + int win = gc->get_int_val("cache", "MaxRequestWindow", 0); + + for (int i = 0; i < MAXLISTENERS; i++) + { + const char *errmsg = NULL; + char bindStr[32]; + char bindPort[32]; + int rbufsz = 0; + int wbufsz = 0; + + if (i == 0) + { + snprintf(bindStr, sizeof(bindStr) - 1, "PluginAddr"); + snprintf(bindPort, sizeof(bindPort) - 1, "PluginPort"); + } + else + { + snprintf(bindStr, sizeof(bindStr) - 1, "PluginAddr%d", i); + snprintf(bindPort, sizeof(bindPort) - 1, "PluginPort%d", i); + } + + const char *addrStr = gc->get_str_val("cache", bindStr); + if (addrStr == NULL) + { + continue; + } + + errmsg = _sockaddr[i].set_address(addrStr, gc->get_str_val("cache", bindPort)); + if (errmsg) + { + log_error("bad BindAddr%d/BindPort%d: %s\n", i, i, errmsg); + continue; + } + + int n = single ? 0 : i; + if (_sockaddr[i].socket_type() == SOCK_DGRAM) + { // DGRAM listener + rbufsz = gc->get_int_val("cache", "UdpRecvBufferSize", 0); + wbufsz = gc->get_int_val("cache", "UdpSendBufferSize", 0); + } + else + { + // STREAM socket listener + rbufsz = wbufsz = 0; + } + + _listener[i] = new DTCListener(&_sockaddr[i]); + _listener[i]->set_request_window(win); + if (_listener[i]->Bind(backlog, rbufsz, wbufsz) != 0) + { + if (i == 0) + { + log_crit("Error bind unix-socket"); + return -1; + } + else + { + continue; + } + } + + if (init_decoder(n, idle) != 0) + { + return -1; + } + + if (_listener[i]->Attach(_decoder[n], backlog) < 0) + { + return -1; + } + + //inc succ count + succ_count++; + } + + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_thread[i] == NULL) + { + continue; + } + + _thread[i]->running_thread(); + } + + if (0 == succ_count) + { + log_error("all plugin bind address & port invalid."); + return -1; + } + + return 0; +} + +int PluginListenerPool::Match(const char *name, int port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_listener[i] == NULL) + { + continue; + } + + if (_sockaddr[i].Match(name, port)) + { + return 1; + } + } + + return 0; +} + +int PluginListenerPool::Match(const char *name, const char *port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_listener[i] == NULL) + { + continue; + } + + if (_sockaddr[i].Match(name, port)) + { + return 1; + } + } + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_listener_pool.h b/src/search_local/index_storage/common/plugin_listener_pool.h new file mode 100644 index 0000000..ed7383e --- /dev/null +++ b/src/search_local/index_storage/common/plugin_listener_pool.h @@ -0,0 +1,31 @@ +#ifndef __PLUGIN_LISTENER_POOL_H__ +#define __PLUGIN_LISTENER_POOL_H__ + +#include "daemon.h" +#include "plugin_unit.h" +#include "net_addr.h" + +class DTCListener; +class PollThread; + +class PluginListenerPool +{ +public: + PluginListenerPool(); + ~PluginListenerPool(); + int Bind(DTCConfig *gc); + + int Match(const char *, int = 0); + int Match(const char *, const char *); + +private: + SocketAddress _sockaddr[MAXLISTENERS]; + DTCListener *_listener[MAXLISTENERS]; + PollThread *_thread[MAXLISTENERS]; + PluginDecoderUnit *_decoder[MAXLISTENERS]; + int _udpfd[MAXLISTENERS]; + + int init_decoder(int n, int idle); +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_mgr.cc b/src/search_local/index_storage/common/plugin_mgr.cc new file mode 100644 index 0000000..2ed61e3 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_mgr.cc @@ -0,0 +1,381 @@ +/* + * ===================================================================================== + * + * Filename: plugin_mgr.cc + * + * Description: plugin management. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +#include "config.h" +#include "plugin_listener_pool.h" +#include "plugin_worker.h" +#include "plugin_mgr.h" +#include "singleton.h" +#include "mem_check.h" +#include "log.h" + +extern DTCConfig *gConfig; +typedef void (*so_set_network_mode_t)(void); +typedef void (*so_set_strings_t)(const char *); + +PluginManager::PluginManager(void) : _dll(NULL), + _config(NULL), + _plugin_name(NULL), + _plugin_listener_pool(NULL), + _worker_notifier(NULL), + _worker_number(1), + _sb_if_handle(NULL), + _plugin_timer(NULL) +{ +} + +PluginManager::~PluginManager(void) +{ +} + +PluginManager *PluginManager::Instance(void) +{ + return Singleton::Instance(); +} + +void PluginManager::Destroy(void) +{ + return Singleton::Destroy(); +} + +int PluginManager::open(int mode) +{ + //create worker notifier + NEW(worker_notify_t, _worker_notifier); + if (NULL == _worker_notifier) + { + log_error("create worker notifier failed, msg:%s", strerror(errno)); + return -1; + } + + if (load_plugin_depend(mode) != 0) + { + return -1; + } + + //create dll info + _dll = (dll_func_t *)MALLOC(sizeof(dll_func_t)); + if (NULL == _dll) + { + log_error("malloc dll_func_t object failed, msg:%s", strerror(errno)); + return -1; + } + memset(_dll, 0x00, sizeof(dll_func_t)); + + _plugin_name = gConfig->get_str_val("cache", "PluginName"); + if (NULL == _plugin_name) + { + log_error("PluginName[%p] is invalid", _plugin_name); + return -1; + } + + //get plugin config file + PluginGlobal::_plugin_conf_file = gConfig->get_str_val("cache", "PluginConfigFile"); + if (NULL == PluginGlobal::_plugin_conf_file) + { + log_info("PluginConfigFile[%p] is invalid", PluginGlobal::_plugin_conf_file); + } + + if (register_plugin(_plugin_name) != 0) + { + log_error("register plugin[%s] failed", _plugin_name); + return -1; + } + + //invoke handle_init + if (NULL != _dll->handle_init) + { + const char *plugin_config_file[2] = {(const char *)PluginGlobal::_plugin_conf_file, NULL}; + if (_dll->handle_init(1, (char **)plugin_config_file, PROC_MAIN) != 0) + { + log_error("invoke plugin[%s]::handle_init() failed", _plugin_name); + return -1; + } + } + + //crate PluginListenerPool object + NEW(PluginListenerPool, _plugin_listener_pool); + if (NULL == _plugin_listener_pool) + { + log_error("create PluginListenerPool object failed, msg:%s", strerror(errno)); + return -1; + } + if (_plugin_listener_pool->Bind(gConfig) != 0) + { + return -1; + } + + //create worker pool + if (create_worker_pool(gConfig->get_int_val("cache", "PluginWorkerNumber")) != 0) + { + log_error("create plugin worker pool failed."); + return -1; + } + + //create plugin timer notify + if (_dll->handle_timer_notify) + { + if (create_plugin_timer(gConfig->get_int_val("cache", "PluginTimerNotifyInterval", 10)) != 0) + { + log_error("create usr timer notify failed."); + return -1; + } + } + + return 0; +} + +int PluginManager::close(void) +{ + //stop plugin timer + if (_plugin_timer) + { + _plugin_timer->interrupt(); + DELETE(_plugin_timer); + _plugin_timer = NULL; + } + + //stop worker + _worker_notifier->Stop(PLUGIN_STOP_REQUEST); + + //destroy worker + for (int i = 0; i < _worker_number; ++i) + { + if (NULL != _plugin_worker_pool[i]) + { + _plugin_worker_pool[i]->interrupt(); + } + + DELETE(_plugin_worker_pool[i]); + } + FREE(_plugin_worker_pool); + _plugin_worker_pool = NULL; + + //delete plugin listener pool + DELETE(_plugin_listener_pool); + + //unregister plugin + if (NULL != _dll->handle_fini) + { + _dll->handle_fini(PROC_MAIN); + } + unregister_plugin(); + FREE_CLEAR(_dll); + + //destroy work notifier + DELETE(_worker_notifier); + + return 0; +} + +int PluginManager::load_plugin_depend(int mode) +{ + char *error = NULL; + so_set_network_mode_t so_set_network_mode = NULL; + so_set_strings_t so_set_server_address = NULL; + so_set_strings_t so_set_server_tablename = NULL; + + //load server bench so + void *sb_if_handle = dlopen(SERVER_BENCH_SO_FILE, RTLD_NOW | RTLD_GLOBAL); + if ((error = dlerror()) != NULL) + { + log_error("so file[%s] dlopen error, %s", SERVER_BENCH_SO_FILE, error); + return -1; + } + + //load dtc api so + void *dtcapi_handle = dlopen(DTC_API_SO_FILE, RTLD_NOW | RTLD_GLOBAL); + if ((error = dlerror()) != NULL) + { + log_error("so file[%s] dlopen error, %s", DTC_API_SO_FILE, error); + return -1; + } + + //init plugin logger + const char *plugin_log_path = gConfig->get_str_val("cache", "PluginLogPath"); + if (NULL == plugin_log_path) + { + plugin_log_path = (const char *)"../log"; + } + const char *plugin_log_name = gConfig->get_str_val("cache", "PluginLogName"); + if (NULL == plugin_log_name) + { + plugin_log_name = (const char *)"plugin_"; + } + int plugin_log_level = gConfig->get_idx_val("cache", "LogLevel", ((const char *const[]){"emerg", "alert", "crit", "error", "warning", "notice", "info", "debug", NULL}), 6); + int plugin_log_size = gConfig->get_int_val("cache", "PluginLogSize", 1 << 28); + + DLFUNC(sb_if_handle, _sb_if_handle, plugin_log_init_t, "log_init"); + if (_sb_if_handle(plugin_log_path, plugin_log_level, plugin_log_size, plugin_log_name) != 0) + { + log_error("init plugin logger failed."); + return -1; + } + + //set_network_mode + DLFUNC(dtcapi_handle, so_set_network_mode, so_set_network_mode_t, "set_network_mode"); + if (so_set_network_mode && mode) + { + so_set_network_mode(); + } + + //set_server_address + DLFUNC(dtcapi_handle, so_set_server_address, so_set_strings_t, "set_server_address"); + if (so_set_server_address) + { + log_debug("set server address:%s", gConfig->get_str_val("cache", "BindAddr")); + so_set_server_address(gConfig->get_str_val("cache", "BindAddr")); + } + //set_server_tablename + DLFUNC(dtcapi_handle, so_set_server_tablename, so_set_strings_t, "set_server_tablename"); + if (so_set_server_tablename) + { + if (!gConfig->get_str_val("TABLE_DEFINE", "table_name")) + { + log_error("can't find tablename in table.conf"); + return 0; + } + log_debug("set server tablename:%s", gConfig->get_str_val("TABLE_DEFINE", "table_name")); + so_set_server_tablename(gConfig->get_str_val("TABLE_DEFINE", "table_name")); + } + + return 0; + +out: + return -1; +} + +int PluginManager::register_plugin(const char *file_name) +{ + char *error = NULL; + int ret_code = -1; + + _dll->handle = dlopen(file_name, RTLD_NOW); + if ((error = dlerror()) != NULL) + { + log_error("dlopen error, %s", error); + goto out; + } + + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_init, handle_init_t, "handle_init"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_fini, handle_fini_t, "handle_fini"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_open, handle_open_t, "handle_open"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_close, handle_close_t, "handle_close"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_timer_notify, handle_timer_notify_t, "handle_timer_notify"); + DLFUNC(_dll->handle, _dll->handle_input, handle_input_t, "handle_input"); + DLFUNC(_dll->handle, _dll->handle_process, handle_process_t, "handle_process"); + + ret_code = 0; + +out: + if (0 == ret_code) + { + log_info("open plugin:%s successful.", file_name); + } + else + { + log_info("open plugin:%s failed.", file_name); + } + + return ret_code; +} + +void PluginManager::unregister_plugin(void) +{ + if (NULL != _dll) + { + if (NULL != _dll->handle) + { + dlclose(_dll->handle); + } + + memset(_dll, 0x00, sizeof(dll_func_t)); + } + + return; +} + +int PluginManager::create_worker_pool(int worker_number) +{ + _worker_number = worker_number; + + if (_worker_number < 1 || _worker_number > PluginGlobal::_max_worker_number) + { + log_warning("worker number[%d] is invalid, default[%d]", _worker_number, PluginGlobal::_default_worker_number); + _worker_number = PluginGlobal::_default_worker_number; + } + + _plugin_worker_pool = (Thread **)calloc(_worker_number, sizeof(Thread *)); + if (NULL == _plugin_worker_pool) + { + log_error("calloc worker pool failed, msg:%s", strerror(errno)); + return -1; + } + + for (int i = 0; i < _worker_number; ++i) + { + char name[32]; + snprintf(name, sizeof(name) - 1, "plugin_worker_%d", i); + + NEW(PluginWorkerThread(name, _worker_notifier, i), _plugin_worker_pool[i]); + if (NULL == _plugin_worker_pool[i]) + { + log_error("create %s failed, msg:%s", name, strerror(errno)); + return -1; + } + + if (_plugin_worker_pool[i]->initialize_thread() != 0) + { + log_error("%s initialize failed.", name); + return -1; + } + + _plugin_worker_pool[i]->running_thread(); + } + + return 0; +} + +int PluginManager::create_plugin_timer(const int interval) +{ + char name[32] = {'\0'}; + int plugin_timer_interval = 0; + + plugin_timer_interval = (interval < 1) ? 1 : interval; + snprintf(name, sizeof(name) - 1, "%s", "plugin_timer"); + + NEW(PluginTimer(name, plugin_timer_interval), _plugin_timer); + if (NULL == _plugin_timer) + { + log_error("create %s failed, msg:%s", name, strerror(errno)); + return -1; + } + + if (_plugin_timer->initialize_thread() != 0) + { + log_error("%s initialize failed.", name); + return -1; + } + + _plugin_timer->running_thread(); + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_mgr.h b/src/search_local/index_storage/common/plugin_mgr.h new file mode 100644 index 0000000..97269b5 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_mgr.h @@ -0,0 +1,77 @@ +/* + * ===================================================================================== + * + * Filename: plugin_mgr.h + * + * Description: plugin manager + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PLUGIN_MANAGER_H__ +#define __DTC_PLUGIN_MANAGER_H__ + +#include +#include +#include + +#include "non_copyable.h" +#include "plugin_request.h" +#include "plugin_timer.h" + +class DTCConfig; +class PluginListenerPool; +class PluginWorkerThread; + +class PluginManager : private noncopyable +{ +public: + PluginManager(void); + ~PluginManager(void); + + static PluginManager *Instance(void); + static void Destroy(void); + + int open(int mode); + int close(void); + + inline dll_func_t *get_dll(void) + { + return _dll; + } + + inline worker_notify_t *get_worker_notifier(void) + { + return _worker_notifier; + } + +public: +protected: +protected: +private: + int register_plugin(const char *file_name); + void unregister_plugin(void); + int create_worker_pool(int worker_number); + int load_plugin_depend(int mode); + int create_plugin_timer(const int interval); + +private: + dll_func_t *_dll; + DTCConfig *_config; + const char *_plugin_name; + PluginListenerPool *_plugin_listener_pool; + Thread **_plugin_worker_pool; + worker_notify_t *_worker_notifier; + int _worker_number; + plugin_log_init_t _sb_if_handle; + Thread *_plugin_timer; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_proxy_listener_pool.cc b/src/search_local/index_storage/common/plugin_proxy_listener_pool.cc new file mode 100644 index 0000000..2981f02 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_proxy_listener_pool.cc @@ -0,0 +1,208 @@ +#include + +#include "plugin_proxy_listener_pool.h" +#include "plugin_global.h" +#include "listener.h" +#include "poll_thread.h" +#include "plugin_unit.h" +#include "poll_thread.h" +#include "unix_socket.h" +#include "config.h" +#include "log.h" +#include "mem_check.h" + +PluginAgentListenerPool::PluginAgentListenerPool(void) +{ + memset(_listener, 0, sizeof(_listener)); + memset(_thread, 0, sizeof(_thread)); + memset(_decoder, 0, sizeof(_decoder)); + memset(_udpfd, 0xff, sizeof(_udpfd)); +} + +PluginAgentListenerPool::~PluginAgentListenerPool(void) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_thread[i]) + { + _thread[i]->interrupt(); + } + + DELETE(_listener[i]); + DELETE(_decoder[i]); + + if (_udpfd[i] >= 0) + { + close(_udpfd[i]); + } + } +} + +int PluginAgentListenerPool::init_decoder(int n, int idle) +{ + if (_thread[n] == NULL) + { + char name[16]; + snprintf(name, sizeof(name) - 1, "plugin_inc%d", n); + + _thread[n] = NULL; + NEW(PollThread(name), _thread[n]); + if (NULL == _thread[n]) + { + log_error("create PollThread object failed, %m"); + return -1; + } + if (_thread[n]->initialize_thread() == -1) + { + return -1; + } + + _decoder[n] = NULL; + NEW(PluginDecoderUnit(_thread[n], idle), _decoder[n]); + if (NULL == _decoder[n]) + { + log_error("create PluginDecoderUnit object failed, %m"); + return -1; + } + + if (_decoder[n]->attach_incoming_notifier() != 0) + { + log_error("attach incoming notifier failed."); + return -1; + } + } + + return 0; +} + +int PluginAgentListenerPool::Bind() +{ + int succ_count = 0; + + int idle = PluginGlobal::_idle_timeout; + if (idle < 0) + { + log_notice("idle_timeout invalid, use default value: 100"); + idle = 100; + } + idle = 100; + + int single = PluginGlobal::_single_incoming_thread; + int backlog = PluginGlobal::_max_listen_count; + int win = PluginGlobal::_max_request_window; + + for (int i = 0; i < 1; i++) // Only One + { + const char *errmsg = NULL; + int rbufsz = 0; + int wbufsz = 0; + + const char *addrStr = PluginGlobal::_bind_address; + if (addrStr == NULL) + { + continue; + } + + const char *port = NULL; + errmsg = _sockaddr[i].set_address(addrStr, port); + if (errmsg) + { + log_error("bad BindAddr%d/BindPort%d: %s\n", i, i, errmsg); + continue; + } + + int n = single ? 0 : i; + if (_sockaddr[i].socket_type() == SOCK_DGRAM) + { // DGRAM listener + rbufsz = PluginGlobal::_udp_recv_buffer_size; + wbufsz = PluginGlobal::_udp_send_buffer_size; + } + else + { + // STREAM socket listener + rbufsz = wbufsz = 0; + } + + _listener[i] = new DTCListener(&_sockaddr[i]); + _listener[i]->set_request_window(win); + if (_listener[i]->Bind(backlog, rbufsz, wbufsz) != 0) + { + if (i == 0) + { + log_crit("Error bind unix-socket"); + return -1; + } + else + { + continue; + } + } + + if (init_decoder(n, idle) != 0) + { + return -1; + } + + if (_listener[i]->Attach(_decoder[n], backlog) < 0) + { + return -1; + } + + //inc succ count + succ_count++; + } + + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_thread[i] == NULL) + { + continue; + } + + _thread[i]->running_thread(); + } + + if (0 == succ_count) + { + log_error("all plugin bind address & port invalid."); + return -1; + } + + return 0; +} + +int PluginAgentListenerPool::Match(const char *name, int port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_listener[i] == NULL) + { + continue; + } + + if (_sockaddr[i].Match(name, port)) + { + return 1; + } + } + + return 0; +} + +int PluginAgentListenerPool::Match(const char *name, const char *port) +{ + for (int i = 0; i < MAXLISTENERS; i++) + { + if (_listener[i] == NULL) + { + continue; + } + + if (_sockaddr[i].Match(name, port)) + { + return 1; + } + } + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_proxy_listener_pool.h b/src/search_local/index_storage/common/plugin_proxy_listener_pool.h new file mode 100644 index 0000000..b4b2e5e --- /dev/null +++ b/src/search_local/index_storage/common/plugin_proxy_listener_pool.h @@ -0,0 +1,31 @@ +#ifndef __PLUGIN_LISTENER_POOL_H__ +#define __PLUGIN_LISTENER_POOL_H__ + +#include "daemon.h" +#include "plugin_unit.h" +#include "net_addr.h" + +class DTCListener; +class PollThread; + +class PluginAgentListenerPool +{ +public: + PluginAgentListenerPool(); + ~PluginAgentListenerPool(); + int Bind(); + + int Match(const char *, int = 0); + int Match(const char *, const char *); + +private: + SocketAddress _sockaddr[MAXLISTENERS]; + DTCListener *_listener[MAXLISTENERS]; + PollThread *_thread[MAXLISTENERS]; + PluginDecoderUnit *_decoder[MAXLISTENERS]; + int _udpfd[MAXLISTENERS]; + + int init_decoder(int n, int idle); +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_proxy_mgr.cc b/src/search_local/index_storage/common/plugin_proxy_mgr.cc new file mode 100644 index 0000000..d719fa3 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_proxy_mgr.cc @@ -0,0 +1,335 @@ +/* + * ===================================================================================== + * + * Filename: plugin_mgr.cc + * + * Description: dtc plugin management. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +//#include "config.h" +#include "plugin_proxy_listener_pool.h" +#include "plugin_worker.h" +#include "plugin_agent_mgr.h" +#include "singleton.h" +#include "mem_check.h" +#include "log.h" +#include "plugin_agent_mgr.h" + +//extern DTCConfig *gConfig; +typedef void (*so_set_network_mode_t)(void); +typedef void (*so_set_strings_t)(const char *); + +PluginAgentManager::PluginAgentManager(void) : _dll(NULL), + _config(NULL), + _plugin_name(NULL), + _plugin_listener_pool(NULL), + _worker_notifier(NULL), + _worker_number(1), + _sb_if_handle(NULL), + _plugin_timer(NULL), + plugin_log_path(NULL), + plugin_log_name(NULL), + plugin_log_level(0), + plugin_log_size(0), + plugin_worker_number(0), + plugin_timer_notify_interval(0) +{ +} + +PluginAgentManager::~PluginAgentManager(void) +{ +} + +PluginAgentManager *PluginAgentManager::Instance(void) +{ + return Singleton::Instance(); +} + +void PluginAgentManager::Destroy(void) +{ + return Singleton::Destroy(); +} + +int PluginAgentManager::open() +{ + //create worker notifier + NEW(worker_notify_t, _worker_notifier); + if (NULL == _worker_notifier) + { + log_error("create worker notifier failed, msg:%s", strerror(errno)); + return -1; + } + + if (load_plugin_depend() != 0) + { + return -1; + } + + //create dll info + _dll = (dll_func_t *)MALLOC(sizeof(dll_func_t)); + if (NULL == _dll) + { + log_error("malloc dll_func_t object failed, msg:%s", strerror(errno)); + return -1; + } + memset(_dll, 0x00, sizeof(dll_func_t)); + + if (NULL == _plugin_name) + { + log_error("PluginName[%p] is invalid", _plugin_name); + return -1; + } + + //get plugin config file + if (NULL == PluginGlobal::_plugin_conf_file) + { + log_info("PluginConfigFile[%p] is invalid", PluginGlobal::_plugin_conf_file); + } + + if (register_plugin(_plugin_name) != 0) + { + log_error("register plugin[%s] failed", _plugin_name); + return -1; + } + + //invoke handle_init + if (NULL != _dll->handle_init) + { + const char *plugin_config_file[2] = {(const char *)PluginGlobal::_plugin_conf_file, NULL}; + if (_dll->handle_init(1, (char **)plugin_config_file, PROC_MAIN) != 0) + { + log_error("invoke plugin[%s]::handle_init() failed", _plugin_name); + return -1; + } + log_debug("plugin::handle_init _plugin_conf_file=%s", (const char *)PluginGlobal::_plugin_conf_file); + } + + //crate PluginAgentListenerPool object + NEW(PluginAgentListenerPool, _plugin_listener_pool); + if (NULL == _plugin_listener_pool) + { + log_error("create PluginAgentListenerPool object failed, msg:%s", strerror(errno)); + return -1; + } + if (_plugin_listener_pool->Bind() != 0) + { + return -1; + } + + //create worker pool + if (create_worker_pool(plugin_worker_number) != 0) + { + log_error("create plugin worker pool failed."); + return -1; + } + + //create plugin timer notify + if (_dll->handle_timer_notify) + { + if (create_plugin_timer(plugin_timer_notify_interval) != 0) + { + log_error("create usr timer notify failed."); + return -1; + } + } + + return 0; +} + +int PluginAgentManager::close(void) +{ + //stop plugin timer + if (_plugin_timer) + { + _plugin_timer->interrupt(); + DELETE(_plugin_timer); + _plugin_timer = NULL; + } + + //stop worker + _worker_notifier->Stop(PLUGIN_STOP_REQUEST); + + //destroy worker + for (int i = 0; i < _worker_number; ++i) + { + if (NULL != _plugin_worker_pool[i]) + { + _plugin_worker_pool[i]->interrupt(); + } + + DELETE(_plugin_worker_pool[i]); + } + FREE(_plugin_worker_pool); + _plugin_worker_pool = NULL; + + //delete plugin listener pool + DELETE(_plugin_listener_pool); + + //unregister plugin + if (NULL != _dll->handle_fini) + { + _dll->handle_fini(PROC_MAIN); + } + unregister_plugin(); + FREE_CLEAR(_dll); + + //destroy work notifier + DELETE(_worker_notifier); + + return 0; +} + +int PluginAgentManager::load_plugin_depend() +{ + char *error = NULL; + + //load server bench so + void *sb_if_handle = dlopen(SERVER_BENCH_SO_FILE, RTLD_NOW | RTLD_GLOBAL); + if ((error = dlerror()) != NULL) + { + log_error("so file[%s] dlopen error, %s", SERVER_BENCH_SO_FILE, error); + return -1; + } + + DLFUNC(sb_if_handle, _sb_if_handle, plugin_log_init_t, "log_init"); + if (_sb_if_handle(plugin_log_path, plugin_log_level, plugin_log_size, plugin_log_name) != 0) + { + log_error("init plugin logger failed."); + return -1; + } + + return 0; + +out: + return -1; +} + +int PluginAgentManager::register_plugin(const char *file_name) +{ + char *error = NULL; + int ret_code = -1; + + _dll->handle = dlopen(file_name, RTLD_NOW | RTLD_NODELETE); + if ((error = dlerror()) != NULL) + { + log_error("dlopen error, %s", error); + goto out; + } + + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_init, handle_init_t, "handle_init"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_fini, handle_fini_t, "handle_fini"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_open, handle_open_t, "handle_open"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_close, handle_close_t, "handle_close"); + DLFUNC_NO_ERROR(_dll->handle, _dll->handle_timer_notify, handle_timer_notify_t, "handle_timer_notify"); + DLFUNC(_dll->handle, _dll->handle_input, handle_input_t, "handle_input"); + DLFUNC(_dll->handle, _dll->handle_process, handle_process_t, "handle_process"); + + ret_code = 0; + +out: + if (0 == ret_code) + { + log_info("open plugin:%s successful.", file_name); + } + else + { + log_info("open plugin:%s failed.", file_name); + } + + return ret_code; +} + +void PluginAgentManager::unregister_plugin(void) +{ + if (NULL != _dll) + { + if (NULL != _dll->handle) + { + dlclose(_dll->handle); + } + + memset(_dll, 0x00, sizeof(dll_func_t)); + } + + return; +} + +int PluginAgentManager::create_worker_pool(int worker_number) +{ + _worker_number = worker_number; + + if (_worker_number < 1 || _worker_number > PluginGlobal::_max_worker_number) + { + log_warning("worker number[%d] is invalid, default[%d]", _worker_number, PluginGlobal::_default_worker_number); + _worker_number = PluginGlobal::_default_worker_number; + } + + _plugin_worker_pool = (Thread **)calloc(_worker_number, sizeof(Thread *)); + if (NULL == _plugin_worker_pool) + { + log_error("calloc worker pool failed, msg:%s", strerror(errno)); + return -1; + } + + for (int i = 0; i < _worker_number; ++i) + { + char name[32]; + snprintf(name, sizeof(name) - 1, "plugin_worker_%d", i); + + NEW(PluginWorkerThread(name, _worker_notifier, i), _plugin_worker_pool[i]); + if (NULL == _plugin_worker_pool[i]) + { + log_error("create %s failed, msg:%s", name, strerror(errno)); + return -1; + } + + if (_plugin_worker_pool[i]->initialize_thread() != 0) + { + log_error("%s initialize failed.", name); + return -1; + } + + _plugin_worker_pool[i]->running_thread(); + } + + return 0; +} + +int PluginAgentManager::create_plugin_timer(const int interval) +{ + char name[32] = {'\0'}; + int plugin_timer_interval = 0; + + plugin_timer_interval = (interval < 1) ? 1 : interval; + snprintf(name, sizeof(name) - 1, "%s", "plugin_timer"); + + NEW(PluginTimer(name, plugin_timer_interval), _plugin_timer); + if (NULL == _plugin_timer) + { + log_error("create %s failed, msg:%s", name, strerror(errno)); + return -1; + } + + if (_plugin_timer->initialize_thread() != 0) + { + log_error("%s initialize failed.", name); + return -1; + } + + _plugin_timer->running_thread(); + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_request.cc b/src/search_local/index_storage/common/plugin_request.cc new file mode 100644 index 0000000..1144017 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_request.cc @@ -0,0 +1,114 @@ +/* + * ===================================================================================== + * + * Filename: plugin_request.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include "plugin_request.h" +#include "plugin_sync.h" +#include "plugin_dgram.h" +#include "log.h" +#include "plugin_mgr.h" + +int PluginStream::handle_process(void) +{ + if (0 != _dll->handle_process(_recv_buf, _real_len, &_send_buf, &_send_len, &_skinfo)) + { + mark_handle_fail(); + log_error("invoke handle_process failed, worker[%d]", _gettid_()); + } + else + { + mark_handle_succ(); + } + + if (_incoming_notifier->Push(this) != 0) + { + log_error("push plugin request to incoming failed, worker[%d]", _gettid_()); + delete this; + return -1; + } + + return 0; +} + +int PluginDatagram::handle_process(void) +{ + if (0 != _dll->handle_process(_recv_buf, _real_len, &_send_buf, &_send_len, &_skinfo)) + { + mark_handle_fail(); + log_error("invoke handle_process failed, worker[%d]", _gettid_()); + } + else + { + mark_handle_succ(); + } + + if (_incoming_notifier->Push(this) != 0) + { + log_error("push plugin request to incoming failed, worker[%d]", _gettid_()); + delete this; + return -1; + } + + return 0; +} + +int PluginStream::task_notify(void) +{ + if (disconnect()) + { + DELETE(_plugin_sync); + delete this; + return -1; + } + + if (!handle_succ()) + { + DELETE(_plugin_sync); + delete this; + return -1; + } + + if (NULL == _plugin_sync) + { + delete this; + return -1; + } + + _plugin_sync->set_stage(PLUGIN_SEND); + _plugin_sync->send_response(); + + return 0; +} + +int PluginDatagram::task_notify(void) +{ + if (!handle_succ()) + { + delete this; + return -1; + } + + if (NULL == _plugin_dgram) + { + delete this; + return -1; + } + + _plugin_dgram->send_response(this); + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_request.h b/src/search_local/index_storage/common/plugin_request.h new file mode 100644 index 0000000..9f6e3f1 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_request.h @@ -0,0 +1,261 @@ +/* + * ===================================================================================== + * + * Filename: plugin_request.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PLUGIN_REQUESET_H__ +#define __DTC_PLUGIN_REQUESET_H__ + +#include "value.h" +#include "plugin_global.h" +#include "mtpqueue.h" +#include "wait_queue.h" +#include "mem_check.h" +#include "stop_watch.h" + +template +class PluginIncomingNotify : public ThreadingPipeQueue > +{ +public: + PluginIncomingNotify(void) + { + } + + virtual ~PluginIncomingNotify(void) + { + } + + void task_notify(T *p) + { + if (p) + { + p->task_notify(); + } + + return; + } +}; + +enum +{ + PLUGIN_REQ_SUCC = 1, + PLUGIN_REQ_MULTI = 2, +}; + +class PluginSync; +class PluginDgram; +class PluginRequest; + +typedef PluginRequest plugin_request_t; +typedef PluginIncomingNotify incoming_notify_t; +typedef threading_wait_queue worker_notify_t; + +class PluginRequest +{ +public: + PluginRequest(void) : _recv_buf(NULL), + _recv_len(0), + _send_buf(0), + _send_len(0), + _sent_len(0), + _real_len(0), + _incoming_notifier(NULL), + _flags(0) + { + } + + virtual ~PluginRequest(void) + { + } + + inline void set_time_info(void) + { + struct timeval now; + gettimeofday(&now, NULL); + + _response_timer = (int)(now.tv_sec * 1000000ULL + now.tv_usec); + _skinfo.recvtm = now.tv_sec; + _skinfo.tasktm = now.tv_sec; + } + + inline void mark_handle_succ(void) + { + _flags |= PLUGIN_REQ_SUCC; + } + + inline void mark_handle_fail(void) + { + _flags &= ~PLUGIN_REQ_SUCC; + } + + inline int handle_succ(void) + { + return (_flags & PLUGIN_REQ_SUCC); + } + + inline int recv_only(void) + { + return (_skinfo.flags & PLUGIN_RECV_ONLY); + } + + inline int disconnect(void) + { + return (_skinfo.flags & PLUGIN_DISCONNECT); + } + + virtual int handle_process(void) = 0; + virtual int task_notify(void) = 0; + +public: + char *_recv_buf; + int _recv_len; + + char *_send_buf; + int _send_len; + int _sent_len; + + int _real_len; + + incoming_notify_t *_incoming_notifier; + stopwatch_usec_t _response_timer; + skinfo_t _skinfo; + +protected: + /* 预留标志,按照bit操作 */ + /* 0: handle_process 执行失败 */ + /* 1: handle_process 执行成功 */ + /* 2: 粘包请求 */ + uint64_t _flags; +}; + +class PluginStream : public PluginRequest, private noncopyable +{ +public: //methods + PluginStream(PluginSync *sync, dll_func_t *dll) : _plugin_sync(sync), + _dll(dll), + _all_len(0), + _recv_remain_len(0) + { + } + + virtual ~PluginStream(void) + { + release_buffer(); + _incoming_notifier = NULL; + _plugin_sync = NULL; + _dll = NULL; + } + + inline void recalc_multipacket(void) + { + const int max_recv_len = PluginGlobal::_max_plugin_recv_len; + + FREE_CLEAR(_send_buf); + _send_len = 0; + _sent_len = 0; + + _all_len = max_recv_len; + _recv_remain_len = max_recv_len; + _recv_len -= _real_len; + memmove(_recv_buf, _recv_buf + _real_len, _recv_len); + _real_len = 0; + mark_single_packet(); + + return; + } + + inline void release_buffer(void) + { + FREE_CLEAR(_recv_buf); + FREE_CLEAR(_send_buf); + + _recv_len = 0; + _send_len = 0; + _sent_len = 0; + _real_len = 0; + _all_len = 0; + _recv_remain_len = 0; + } + + inline void mark_multi_packet(void) + { + _flags |= PLUGIN_REQ_MULTI; + } + + inline void mark_single_packet(void) + { + _flags &= ~PLUGIN_REQ_MULTI; + } + + inline int multi_packet(void) + { + return (_flags & PLUGIN_REQ_MULTI); + } + + virtual int handle_process(void); + virtual int task_notify(void); + +public: //property + PluginSync *_plugin_sync; + dll_func_t *_dll; + + int _all_len; + int _recv_remain_len; +}; + +class PluginDatagram : public PluginRequest, private noncopyable +{ +public: //methods + PluginDatagram(PluginDgram *dgram, dll_func_t *dll) : _addr(NULL), + _addr_len(0), + _plugin_dgram(dgram), + _dll(dll) + { + } + + virtual ~PluginDatagram(void) + { + release_buffer(); + _incoming_notifier = NULL; + _plugin_dgram = NULL; + _dll = NULL; + } + + inline void release_buffer(void) + { + FREE_CLEAR(_recv_buf); + _recv_len = 0; + + FREE_CLEAR(_send_buf); + _send_len = 0; + _sent_len = 0; + + _real_len = 0; + + FREE_CLEAR(_addr); + _addr_len = 0; + } + + virtual int handle_process(void); + virtual int task_notify(void); + +public: //property + void *_addr; + socklen_t _addr_len; + PluginDgram *_plugin_dgram; + dll_func_t *_dll; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_sync.cc b/src/search_local/index_storage/common/plugin_sync.cc new file mode 100644 index 0000000..2ea275f --- /dev/null +++ b/src/search_local/index_storage/common/plugin_sync.cc @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "plugin_agent_mgr.h" +#include "plugin_sync.h" +#include "plugin_unit.h" +#include "poll_thread.h" +#include "log.h" +#include "stat_dtc.h" +#include "mem_check.h" + +//static int statEnable=0; +static StatItemU32 statPluginAcceptCount; +static StatItemU32 statPluginCurConnCount; +extern "C" +{ + extern unsigned int get_local_ip(); +} + +PluginSync::PluginSync(PluginDecoderUnit *plugin_decoder, int fd, void *peer, int peer_size) : PollerObject(plugin_decoder->owner_thread(), fd), + _plugin_stage(PLUGIN_IDLE), + owner(plugin_decoder), + _plugin_request(NULL), + _worker_notifier(NULL), + _plugin_receiver(fd, PluginAgentManager::Instance()->get_dll()), + _plugin_sender(fd, PluginAgentManager::Instance()->get_dll()) +{ + _addr_len = peer_size; + _addr = MALLOC(peer_size); + memcpy((char *)_addr, (char *)peer, peer_size); + + ++statPluginAcceptCount; + ++statPluginCurConnCount; +} + +PluginSync::~PluginSync(void) +{ + --statPluginCurConnCount; + + if (PLUGIN_PROC == _plugin_stage) + { + _plugin_request->_plugin_sync = NULL; + } + else + { + DELETE(_plugin_request); + } + + FREE_CLEAR(_addr); +} + +int PluginSync::create_request(void) +{ + if (NULL != _plugin_request) + { + DELETE(_plugin_request); + _plugin_request = NULL; + } + + NEW(PluginStream(this, PluginAgentManager::Instance()->get_dll()), _plugin_request); + if (NULL == _plugin_request) + { + log_error("create plugin request object failed, msg:%s", strerror(errno)); + return -1; + } + + //set plugin request info + _plugin_request->_skinfo.sockfd = netfd; + _plugin_request->_skinfo.type = SOCK_STREAM; + _plugin_request->_skinfo.local_ip = get_local_ip(); + _plugin_request->_skinfo.local_port = 0; + _plugin_request->_skinfo.remote_ip = ((struct sockaddr_in *)_addr)->sin_addr.s_addr; + _plugin_request->_skinfo.remote_port = ((struct sockaddr_in *)_addr)->sin_port; + _plugin_request->_incoming_notifier = owner->get_incoming_notifier(); + + return 0; +} + +int PluginSync::Attach() +{ + if (create_request() != 0) + { + log_error("create request object failed"); + return -1; + } + + //get worker notifier + _worker_notifier = PluginAgentManager::Instance()->get_worker_notifier(); + if (NULL == _worker_notifier) + { + log_error("get worker notifier failed."); + return -1; + } + + enable_input(); + + if (attach_poller() == -1) + { + return -1; + } + + attach_timer(owner->idle_list()); + _plugin_stage = PLUGIN_IDLE; + + return 0; +} + +int PluginSync::recv_request() +{ + disable_timer(); + + int ret_stage = _plugin_receiver.recv(_plugin_request); + + switch (ret_stage) + { + default: + case NET_FATAL_ERROR: + return -1; + + case NET_IDLE: + attach_timer(owner->idle_list()); + _plugin_stage = PLUGIN_IDLE; + break; + + case NET_RECVING: + //如果收到部分包，则需要加入idle list, 防止该连接挂死 + attach_timer(owner->idle_list()); + _plugin_stage = PLUGIN_RECV; + break; + + case NET_RECV_DONE: + _plugin_request->set_time_info(); + if (_worker_notifier->Push(_plugin_request) != 0) + { + log_error("push plugin request failed, fd[%d]", netfd); + return -1; + } + _plugin_stage = PLUGIN_PROC; + break; + } + + return 0; +} + +int PluginSync::proc_multi_request(void) +{ + _plugin_receiver.set_stage(NET_IDLE); + switch (_plugin_receiver.proc_remain_packet(_plugin_request)) + { + default: + case NET_FATAL_ERROR: + return -1; + + case NET_RECVING: + _plugin_stage = PLUGIN_RECV; + _plugin_sender.set_stage(NET_IDLE); + _plugin_receiver.set_stage(NET_RECVING); + break; + + case NET_RECV_DONE: + _plugin_request->set_time_info(); + if (_worker_notifier->Push(_plugin_request) != 0) + { + log_error("push plugin request failed, fd[%d]", netfd); + return -1; + } + _plugin_stage = PLUGIN_PROC; + break; + } + + return 0; +} + +int PluginSync::Response(void) +{ + if (_plugin_request->recv_only()) + { + goto proc_multi; + } + + switch (_plugin_sender.send(_plugin_request)) + { + default: + case NET_FATAL_ERROR: + return -1; + + case NET_SENDING: + _plugin_stage = PLUGIN_SEND; + enable_output(); + return 0; + + case NET_SEND_DONE: + break; + } + +proc_multi: + //multi request process logic + if (_plugin_request->multi_packet()) + { + if (proc_multi_request() != 0) + { + return -1; + } + } + else + { + _plugin_request->release_buffer(); + _plugin_sender.set_stage(NET_IDLE); + _plugin_receiver.set_stage(NET_IDLE); + _plugin_stage = PLUGIN_IDLE; + } + + //防止多一次output事件触发 + disable_output(); + enable_input(); + attach_timer(owner->idle_list()); + + return 0; +} + +void PluginSync::input_notify(void) +{ + log_debug("enter input_notify!"); + if (_plugin_stage == PLUGIN_IDLE || _plugin_stage == PLUGIN_RECV) + { + if (recv_request() < 0) + { + delete this; + return; + } + } + else /* receive input events again. */ + { + /* check whether client close connection. */ + if (check_link_status()) + { + log_debug("client close connection, delete PluginSync obj, plugin stage=%d", _plugin_stage); + delete this; + return; + } + else + { + DisableInput(); + } + } + + apply_events(); + + return; +} + +void PluginSync::output_notify(void) +{ + if (_plugin_stage == PLUGIN_SEND) + { + if (Response() < 0) + { + delete this; + } + } + else + { + disable_output(); + log_info("Spurious PluginSync::output_notify, plugin stage=%d", _plugin_stage); + } + + return; +} diff --git a/src/search_local/index_storage/common/plugin_sync.h b/src/search_local/index_storage/common/plugin_sync.h new file mode 100644 index 0000000..e41df3a --- /dev/null +++ b/src/search_local/index_storage/common/plugin_sync.h @@ -0,0 +1,71 @@ +#ifndef __PLUGIN_SYNC_H__ +#define __PLUGIN_SYNC_H__ + +#include "poller.h" +#include "timer_list.h" +#include "plugin_unit.h" +#include "plugin_decoder.h" + +class PluginDecoderUnit; + +typedef enum +{ + PLUGIN_IDLE, + PLUGIN_RECV, //wait for recv request, server side + PLUGIN_SEND, //wait for send response, server side + PLUGIN_PROC //IN processing +} plugin_state_t; + +class PluginSync : public PollerObject, private TimerObject +{ +public: + PluginSync(PluginDecoderUnit *plugin_decoder, int fd, void *peer, int peer_size); + virtual ~PluginSync(); + + int Attach(void); + virtual void input_notify(void); + + inline int send_response(void) + { + owner->record_request_time(_plugin_request->_response_timer.live()); + if (Response() < 0) + { + delete this; + } + else + { + delay_apply_events(); + } + + return 0; + } + + inline void set_stage(plugin_state_t stage) + { + _plugin_stage = stage; + } + +private: + virtual void output_notify(void); + +protected: + plugin_state_t _plugin_stage; + + int recv_request(void); + int Response(void); + +private: + int create_request(void); + int proc_multi_request(void); + +private: + PluginDecoderUnit *owner; + PluginStream *_plugin_request; + worker_notify_t *_worker_notifier; + void *_addr; + int _addr_len; + PluginReceiver _plugin_receiver; + PluginSender _plugin_sender; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_timer.cc b/src/search_local/index_storage/common/plugin_timer.cc new file mode 100644 index 0000000..62f55bd --- /dev/null +++ b/src/search_local/index_storage/common/plugin_timer.cc @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include +#include + +#include "plugin_agent_mgr.h" +#include "plugin_timer.h" +#include "plugin_global.h" +#include "log.h" + +PluginTimer::PluginTimer(const char *name, const int interval) : Thread(name, Thread::ThreadTypeSync), _interval(interval) +{ + pthread_mutex_init(&_wake_lock, NULL); +} + +PluginTimer::~PluginTimer() +{ +} + +int PluginTimer::Initialize(void) +{ + _dll = PluginAgentManager::Instance()->get_dll(); + if ((NULL == _dll) || (NULL == _dll->handle_timer_notify)) + { + log_error("get server bench handle failed."); + return -1; + } + + return pthread_mutex_trylock(&_wake_lock) == 0 ? 0 : -1; +} + +void PluginTimer::interrupt(void) +{ + pthread_mutex_unlock(&_wake_lock); + + return Thread::interrupt(); +} + +static void BlockAllSignals(void) +{ + sigset_t sset; + sigfillset(&sset); + sigdelset(&sset, SIGSEGV); + sigdelset(&sset, SIGBUS); + sigdelset(&sset, SIGABRT); + sigdelset(&sset, SIGILL); + sigdelset(&sset, SIGCHLD); + sigdelset(&sset, SIGFPE); + pthread_sigmask(SIG_BLOCK, &sset, &sset); +} + +void *PluginTimer::Process(void) +{ + BlockAllSignals(); + + struct timeval tv; + struct timespec ts; + time_t nextsec = 0; + unsigned long long nextnsec = 0; + int ret_value = 0; + +#define ONESEC_NSEC 1000000000ULL +#define ONESEC_MSEC 1000ULL + gettimeofday(&tv, NULL); + nextsec = tv.tv_sec; + nextnsec = (tv.tv_usec + _interval * ONESEC_MSEC) * ONESEC_MSEC; + while (nextnsec >= ONESEC_NSEC) + { + nextsec++; + nextnsec -= ONESEC_NSEC; + } + + ts.tv_sec = nextsec; + ts.tv_nsec = nextnsec; + + while (!Stopping()) + { + if (pthread_mutex_timedlock(&_wake_lock, &ts) == 0) + { + break; + } + + ret_value = _dll->handle_timer_notify(0, NULL); + if (0 != ret_value) + { + log_error("invoke handle_timer_notify failed, return value:%d timer notify[%d]", ret_value, _gettid_()); + } + + gettimeofday(&tv, NULL); + + nextsec = tv.tv_sec; + nextnsec = (tv.tv_usec + _interval * ONESEC_MSEC) * ONESEC_MSEC; + while (nextnsec >= ONESEC_NSEC) + { + nextsec++; + nextnsec -= ONESEC_NSEC; + } + + ts.tv_sec = nextsec; + ts.tv_nsec = nextnsec; + } + + pthread_mutex_unlock(&_wake_lock); + + return NULL; +} diff --git a/src/search_local/index_storage/common/plugin_timer.h b/src/search_local/index_storage/common/plugin_timer.h new file mode 100644 index 0000000..2292bd0 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_timer.h @@ -0,0 +1,29 @@ +#ifndef __PLUGIN_TIMER_H__ +#define __PLUGIN_TIMER_H__ + +#include +#include +#include +#include +#include + +#include "thread.h" + +class PluginTimer : public Thread +{ +public: + PluginTimer(const char *name, const int timeout); + virtual ~PluginTimer(); + +private: + virtual void *Process(void); + virtual int Initialize(void); + virtual void interrupt(void); + +private: + dll_func_t *_dll; + const int _interval; + pthread_mutex_t _wake_lock; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_unit.cc b/src/search_local/index_storage/common/plugin_unit.cc new file mode 100644 index 0000000..ca45d23 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_unit.cc @@ -0,0 +1,61 @@ +#include +#include + +#include "plugin_unit.h" +#include "plugin_sync.h" +#include "plugin_dgram.h" +#include "log.h" +#include "mem_check.h" + +PluginDecoderUnit::PluginDecoderUnit(PollThread *o, int it) : DecoderUnit(o, it) +{ +} + +PluginDecoderUnit::~PluginDecoderUnit() +{ +} + +int PluginDecoderUnit::process_stream(int newfd, int req, void *peer, int peerSize) +{ + PluginSync *plugin_client = NULL; + NEW(PluginSync(this, newfd, peer, peerSize), plugin_client); + + if (0 == plugin_client) + { + log_error("create PluginSync object failed, errno[%d], msg[%m]", errno); + return -1; + } + + if (plugin_client->Attach() == -1) + { + log_error("Invoke PluginSync::Attach() failed"); + delete plugin_client; + return -1; + } + + /* accept唤醒后立即recv */ + plugin_client->input_notify(); + + return 0; +} + +int PluginDecoderUnit::process_dgram(int newfd) +{ + PluginDgram *plugin_dgram = NULL; + NEW(PluginDgram(this, newfd), plugin_dgram); + + if (0 == plugin_dgram) + { + log_error("create PluginDgram object failed, errno[%d], msg[%m]", errno); + return -1; + } + + if (plugin_dgram->Attach() == -1) + { + log_error("Invoke PluginDgram::Attach() failed"); + delete plugin_dgram; + return -1; + } + + return 0; +} diff --git a/src/search_local/index_storage/common/plugin_unit.h b/src/search_local/index_storage/common/plugin_unit.h new file mode 100644 index 0000000..657b3cc --- /dev/null +++ b/src/search_local/index_storage/common/plugin_unit.h @@ -0,0 +1,36 @@ +#ifndef __H_DTC_PLUGIN_UNIT_H__ +#define __H_DTC_PLUGIN_UNIT_H__ + +#include "stat_dtc.h" +#include "decoder_base.h" +#include "plugin_request.h" +#include "poll_thread.h" + +class PluginDecoderUnit : public DecoderUnit +{ +public: + PluginDecoderUnit(PollThread *owner, int idletimeout); + virtual ~PluginDecoderUnit(); + + virtual int process_stream(int fd, int req, void *peer, int peerSize); + virtual int process_dgram(int fd); + + inline void record_request_time(unsigned int msec) + { + } + + inline incoming_notify_t *get_incoming_notifier(void) + { + return &_incoming_notify; + } + + inline int attach_incoming_notifier(void) + { + return _incoming_notify.attach_poller(owner); + } + +private: + incoming_notify_t _incoming_notify; +}; + +#endif diff --git a/src/search_local/index_storage/common/plugin_worker.cc b/src/search_local/index_storage/common/plugin_worker.cc new file mode 100644 index 0000000..fe0b1d9 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_worker.cc @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include + +#include "plugin_agent_mgr.h" +#include "plugin_worker.h" +#include "plugin_global.h" +#include "mem_check.h" +#include "log.h" + +PluginWorkerThread::PluginWorkerThread(const char *name, worker_notify_t *worker_notify, int seq_no) : Thread(name, Thread::ThreadTypeSync), _worker_notify(worker_notify), _seq_no(seq_no) +{ +} + +PluginWorkerThread::~PluginWorkerThread() +{ +} + +int PluginWorkerThread::Initialize(void) +{ + _dll = PluginAgentManager::Instance()->get_dll(); + if ((NULL == _dll) || (NULL == _dll->handle_init) || (NULL == _dll->handle_process)) + { + log_error("get server bench handle failed."); + return -1; + } + + return 0; +} + +void PluginWorkerThread::Prepare(void) +{ + const char *plugin_config_file[2] = {(const char *)PluginGlobal::_plugin_conf_file, NULL}; + if (_dll->handle_init(1, (char **)plugin_config_file, _seq_no + PROC_WORK) != 0) + { + log_error("invoke handle_init() failed, worker[%d]", _seq_no); + return; + } + + return; +} + +void *PluginWorkerThread::Process(void) +{ + plugin_request_t *plugin_request = NULL; + + while (!Stopping()) + { + plugin_request = _worker_notify->Pop(); + + if (PLUGIN_STOP_REQUEST == plugin_request) + { + break; + } + + if (NULL == plugin_request) + { + log_error("the worker_notify::Pop() invalid, plugin_request=%p, worker[%d]", plugin_request, _gettid_()); + continue; + } + + if (Stopping()) + { + break; + } + + plugin_request->handle_process(); + } + + if (NULL != _dll->handle_fini) + { + _dll->handle_fini(_seq_no + PROC_WORK); + } + + return NULL; +} diff --git a/src/search_local/index_storage/common/plugin_worker.h b/src/search_local/index_storage/common/plugin_worker.h new file mode 100644 index 0000000..44fca66 --- /dev/null +++ b/src/search_local/index_storage/common/plugin_worker.h @@ -0,0 +1,36 @@ +#ifndef __PLUGIN_WORKER_THREAD_H__ +#define __PLUGIN_WORKER_THREAD_H__ + +#include +#include +#include +#include +#include + +#include "timestamp.h" +#include "thread.h" +#include "plugin_request.h" + +class PluginWorkerThread : public Thread +{ +public: + PluginWorkerThread(const char *name, worker_notify_t *worker_notify, int seq_no); + virtual ~PluginWorkerThread(); + + inline int get_seq_no(void) + { + return _seq_no; + } + +protected: + virtual void *Process(void); + virtual int Initialize(void); + virtual void Prepare(void); + +private: + worker_notify_t *_worker_notify; + dll_func_t *_dll; + int _seq_no; +}; + +#endif diff --git a/src/search_local/index_storage/common/poll_thread.cc b/src/search_local/index_storage/common/poll_thread.cc new file mode 100644 index 0000000..7337092 --- /dev/null +++ b/src/search_local/index_storage/common/poll_thread.cc @@ -0,0 +1,79 @@ +/* + * ===================================================================================== + * + * Filename: pool_thread.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "poll_thread.h" +#include "mem_check.h" +#include "poller.h" +#include "myepoll.h" +#include "log.h" +#include + +volatile extern int stop; + +PollThread::PollThread(const char *name) : Thread(name, Thread::ThreadTypeAsync), PollerUnit(1000), pollTimeout(2000) +{ +} + +PollThread::~PollThread() +{ +} + +int PollThread::Initialize(void) +{ + if (Thread::g_autoconf != NULL) + { + int mp0 = get_max_pollers(); + int mp1; + mp1 = g_autoconf->get_int_val("MaxIncomingPollers", Name(), mp0); + log_debug("autoconf thread %s MaxIncomingPollers %d", taskname, mp1); + if (mp1 > mp0) + { + set_max_pollers(mp1); + } + } + if (initialize_poller_unit() < 0) + return -1; + return 0; +} + +void *PollThread::Process(void) +{ + while (!Stopping()) + { + // if previous event loop has no events, + // don't allow zero epoll wait time + int timeout = expire_micro_seconds(pollTimeout, nrEvents == 0); + int interrupted = wait_poller_events(timeout); + update_now_time(timeout, interrupted); + + if (Stopping()) + break; + + process_poller_events(); + check_expired(get_now_time()); + TimerUnit::check_ready(); + ReadyUnit::check_ready(get_now_time()); + delay_apply_events(); + } + return 0; +} diff --git a/src/search_local/index_storage/common/poll_thread.h b/src/search_local/index_storage/common/poll_thread.h new file mode 100644 index 0000000..b49eefa --- /dev/null +++ b/src/search_local/index_storage/common/poll_thread.h @@ -0,0 +1,46 @@ +/* + * ===================================================================================== + * + * Filename: pool_thread.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __POLLTHREAD_H__ +#define __POLLTHREAD_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "timestamp.h" +#include "poller.h" +#include "timer_list.h" +#include "thread.h" + +class PollThread : public Thread, public PollerUnit, public TimerUnit, public ReadyUnit +{ +public: + PollThread(const char *name); + virtual ~PollThread(); + +protected: + int pollTimeout; + virtual void *Process(void); + virtual int Initialize(void); +}; + +#endif diff --git a/src/search_local/index_storage/common/poll_thread_group.cc b/src/search_local/index_storage/common/poll_thread_group.cc new file mode 100644 index 0000000..ee17045 --- /dev/null +++ b/src/search_local/index_storage/common/poll_thread_group.cc @@ -0,0 +1,112 @@ +/* + * ===================================================================================== + * + * Filename: poll_thread_group.cc + * + * Description: + * + * Version: 1.0 + * Created: 10/05/2014 17:40:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#include "poll_thread_group.h" + +PollThreadGroup::PollThreadGroup(const std::string groupName) : threadIndex(0), pollThreads(NULL) +{ + this->groupName = groupName; +} + +PollThreadGroup::PollThreadGroup(const std::string groupName, int numThreads, int mp) : threadIndex(0), pollThreads(NULL) +{ + this->groupName = groupName; + Start(numThreads, mp); +} + +PollThreadGroup::~PollThreadGroup() +{ + if (pollThreads != NULL) + { + for (int i = 0; i < this->numThreads; i++) + { + if (pollThreads[i]) + { + pollThreads[i]->interrupt(); + delete pollThreads[i]; + } + } + + delete pollThreads; + + pollThreads = NULL; + } +} + +PollThread *PollThreadGroup::get_poll_thread() +{ + if (pollThreads != NULL) + return pollThreads[threadIndex++ % numThreads]; + + return NULL; +} + +PollThread *PollThreadGroup::get_poll_thread(int threadIdx) +{ + if (threadIdx > numThreads) + { + return NULL; + } + else + { + return pollThreads[threadIdx]; + } +} + +void PollThreadGroup::Start(int numThreads, int mp) +{ + char threadName[256]; + this->numThreads = numThreads; + + pollThreads = new PollThread *[this->numThreads]; + + for (int i = 0; i < this->numThreads; i++) + { + snprintf(threadName, sizeof(threadName), "%s@%d", groupName.c_str(), i); + + pollThreads[i] = new PollThread(threadName); + //set_max_pollers一定要再InitializeThread前调用，否则不生效 + pollThreads[i]->set_max_pollers(mp); + pollThreads[i]->initialize_thread(); + } +} + +void PollThreadGroup::running_threads() +{ + if (pollThreads == NULL) + return; + + for (int i = 0; i < this->numThreads; i++) + { + pollThreads[i]->running_thread(); + } +} + +int PollThreadGroup::get_poll_threadIndex(PollThread *thread) +{ + for (int i = 0; i < this->numThreads; i++) + { + if (thread == pollThreads[i]) + return i; + } + return -1; +} + +int PollThreadGroup::get_poll_threadSize() +{ + return numThreads; +} diff --git a/src/search_local/index_storage/common/poll_thread_group.h b/src/search_local/index_storage/common/poll_thread_group.h new file mode 100644 index 0000000..a36eef3 --- /dev/null +++ b/src/search_local/index_storage/common/poll_thread_group.h @@ -0,0 +1,44 @@ +/* + * ===================================================================================== + * + * Filename: poll_thread_group.h + * + * Description: + * + * Version: 1.0 + * Created: 10/05/2014 17:40:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#ifndef __POLLTHREADGROUP_H__ +#define __POLLTHREADGROUP_H__ + +#include "poll_thread.h" + +class PollThreadGroup +{ +public: + PollThreadGroup(const std::string groupName); + PollThreadGroup(const std::string groupName, int numThreads, int mp); + virtual ~PollThreadGroup(); + + PollThread *get_poll_thread(); + void Start(int numThreads, int mp); + void running_threads(); + int get_poll_threadIndex(PollThread *thread); + int get_poll_threadSize(); + PollThread *get_poll_thread(int threadIdx); + +protected: + int threadIndex; + int numThreads; + std::string groupName; + PollThread **pollThreads; +}; + +#endif diff --git a/src/search_local/index_storage/common/poller.cc b/src/search_local/index_storage/common/poller.cc new file mode 100644 index 0000000..66b6036 --- /dev/null +++ b/src/search_local/index_storage/common/poller.cc @@ -0,0 +1,388 @@ +/* + * ===================================================================================== + * + * Filename: pooler.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "mem_check.h" + +#include "poll_thread.h" +#include "poller.h" +#include "log.h" + +PollerObject::~PollerObject() +{ + if (ownerUnit && epslot) + ownerUnit->free_epoll_slot(epslot); + + if (netfd > 0) + { + log_debug("%d fd been closed!", netfd); + close(netfd); + netfd = 0; + } + if (eventSlot) + { + eventSlot->poller = NULL; + eventSlot = NULL; + } +} + +int PollerObject::attach_poller(PollerUnit *unit) +{ + if (unit) + { + if (ownerUnit == NULL) + ownerUnit = unit; + else + return -1; + } + if (netfd < 0) + return -1; + + if (epslot <= 0) + { + if (!(epslot = ownerUnit->alloc_epoll_slot())) + return -1; + struct EpollSlot *slot = ownerUnit->get_slot(epslot); + slot->poller = this; + + int flag = fcntl(netfd, F_GETFL); + fcntl(netfd, F_SETFL, O_NONBLOCK | flag); + struct epoll_event ev; + memset(&ev, 0x0, sizeof(ev)); + ev.events = newEvents; + slot->seq++; + ev.data.u64 = ((unsigned long long)slot->seq << 32) + epslot; + if (ownerUnit->Epctl(EPOLL_CTL_ADD, netfd, &ev) == 0) + oldEvents = newEvents; + else + { + ownerUnit->free_epoll_slot(epslot); + log_warning("Epctl: %m"); + return -1; + } + return 0; + } + return apply_events(); +} + +int PollerObject::detach_poller() +{ + if (epslot) + { + struct epoll_event ev; + memset(&ev, 0x0, sizeof(ev)); + if (ownerUnit->Epctl(EPOLL_CTL_DEL, netfd, &ev) == 0) + oldEvents = newEvents; + else + { + log_warning("Epctl: %m"); + return -1; + } + ownerUnit->free_epoll_slot(epslot); + epslot = 0; + } + return 0; +} + +int PollerObject::apply_events() +{ + if (epslot <= 0 || oldEvents == newEvents) + return 0; + + struct epoll_event ev; + memset(&ev, 0x0, sizeof(ev)); + + ev.events = newEvents; + struct EpollSlot *slot = ownerUnit->get_slot(epslot); + slot->seq++; + ev.data.u64 = ((unsigned long long)slot->seq << 32) + epslot; + if (ownerUnit->Epctl(EPOLL_CTL_MOD, netfd, &ev) == 0) + oldEvents = newEvents; + else + { + log_warning("Epctl: %m"); + return -1; + } + + return 0; +} + +int PollerObject::delay_apply_events() +{ + if (epslot <= 0 || oldEvents == newEvents) + return 0; + + if (eventSlot) + return 0; + + eventSlot = ownerUnit->add_delay_event_poller(this); + if (eventSlot == NULL) + { + log_error("max events!!!!!!"); + struct epoll_event ev; + + ev.events = newEvents; + struct EpollSlot *slot = ownerUnit->get_slot(epslot); + slot->seq++; + ev.data.u64 = ((unsigned long long)slot->seq << 32) + epslot; + if (ownerUnit->Epctl(EPOLL_CTL_MOD, netfd, &ev) == 0) + oldEvents = newEvents; + else + { + log_warning("Epctl: %m"); + return -1; + } + } + + return 0; +} + +int PollerObject::check_link_status(void) +{ + char msg[1] = {0}; + int err = 0; + + err = recv(netfd, msg, sizeof(msg), MSG_DONTWAIT | MSG_PEEK); + + /* client already close connection. */ + if (err == 0 || (err < 0 && errno != EAGAIN)) + return -1; + return 0; +} + +void PollerObject::init_poll_fd(struct pollfd *pfd) +{ + pfd->fd = netfd; + pfd->events = newEvents; + pfd->revents = 0; +} + +void PollerObject::input_notify(void) +{ + enable_input(false); +} + +void PollerObject::output_notify(void) +{ + enable_output(false); +} + +int PollerUnit::totalEventSlot = 40960; + +void PollerObject::hangup_notify(void) +{ + delete this; +} + +PollerUnit::PollerUnit(int mp) +{ + maxPollers = mp; + + eeSize = maxPollers > 1024 ? 1024 : maxPollers; + epfd = -1; + ep_events = NULL; + pollerTable = NULL; + freeSlotList = 0; + usedPollers = 0; + //not initailize eventCnt variable may crash, fix crash bug by linjinming 2014-05-18 + eventCnt = 0; +} + +PollerUnit::~PollerUnit() +{ + // skip first one + for (int i = 1; i < maxPollers; i++) + { + if (pollerTable[i].freeList) + continue; + } + + FREE_CLEAR(pollerTable); + + if (epfd != -1) + { + close(epfd); + epfd = -1; + } + + FREE_CLEAR(ep_events); +} + +int PollerUnit::set_max_pollers(int mp) +{ + if (epfd >= 0) + return -1; + maxPollers = mp; + return 0; +} + +int PollerUnit::initialize_poller_unit(void) +{ + pollerTable = (struct EpollSlot *)CALLOC(maxPollers, sizeof(*pollerTable)); + + if (!pollerTable) + { + log_error("calloc failed, num=%d, %m", maxPollers); + return -1; + } + + // already zero-ed + for (int i = 1; i < maxPollers - 1; i++) + { + pollerTable[i].freeList = i + 1; + } + + pollerTable[maxPollers - 1].freeList = 0; + freeSlotList = 1; + + ep_events = (struct epoll_event *)CALLOC(eeSize, sizeof(struct epoll_event)); + + if (!ep_events) + { + log_error("malloc failed, %m"); + return -1; + } + + if ((epfd = epoll_create(maxPollers)) == -1) + { + log_warning("epoll_create failed, %m"); + return -1; + } + fcntl(epfd, F_SETFD, FD_CLOEXEC); + return 0; +} + +inline int PollerUnit::verify_events(struct epoll_event *ev) +{ + int idx = EPOLL_DATA_SLOT(ev); + + if ((idx >= maxPollers) || (EPOLL_DATA_SEQ(ev) != pollerTable[idx].seq)) + { + return -1; + } + + if (pollerTable[idx].poller == NULL || pollerTable[idx].freeList != 0) + { + log_notice("receive invalid epoll event. idx=%d seq=%d poller=%p freelist=%d event=%x", + idx, (int)EPOLL_DATA_SEQ(ev), pollerTable[idx].poller, + pollerTable[idx].freeList, ev->events); + return -1; + } + return 0; +} + +void PollerUnit::free_epoll_slot(int n) +{ + if (n <= 0) + return; + pollerTable[n].freeList = freeSlotList; + freeSlotList = n; + usedPollers--; + pollerTable[n].seq++; + pollerTable[n].poller = NULL; +} + +int PollerUnit::alloc_epoll_slot() +{ + if (0 == freeSlotList) + { + log_crit("no free epoll slot, usedPollers = %d", usedPollers); + return -1; + } + + int n = freeSlotList; + + usedPollers++; + freeSlotList = pollerTable[n].freeList; + pollerTable[n].freeList = 0; + + return n; +} + +int PollerUnit::Epctl(int op, int fd, struct epoll_event *events) +{ + if (epoll_ctl(epfd, op, fd, events) == -1) + { + log_warning("epoll_ctl error, epfd=%d, fd=%d", epfd, fd); + + return -1; + } + + return 0; +} + +int PollerUnit::wait_poller_events(int timeout) +{ + nrEvents = epoll_wait(epfd, ep_events, eeSize, timeout); + return nrEvents; +} + +void PollerUnit::process_poller_events(void) +{ + for (int i = 0; i < nrEvents; i++) + { + if (verify_events(ep_events + i) == -1) + { + log_notice("verify_events failed, ep_events[%d].data.u64 = %llu", i, (unsigned long long)ep_events[i].data.u64); + continue; + } + + EpollSlot *s = &pollerTable[EPOLL_DATA_SLOT(ep_events + i)]; + PollerObject *p = s->poller; + + p->newEvents = p->oldEvents; + if (ep_events[i].events & (EPOLLHUP | EPOLLERR)) + { + p->hangup_notify(); + continue; + } + + if (ep_events[i].events & EPOLLIN) + p->input_notify(); + + s = &pollerTable[EPOLL_DATA_SLOT(ep_events + i)]; + if (s->poller == p && ep_events[i].events & EPOLLOUT) + p->output_notify(); + + s = &pollerTable[EPOLL_DATA_SLOT(ep_events + i)]; + if (s->poller == p) + p->delay_apply_events(); + } +} + +int PollerUnit::delay_apply_events() +{ + for (int i = 0; i < eventCnt; i++) + { + PollerObject *p = eventSlot[i].poller; + if (p) + { + p->apply_events(); + eventSlot[i].poller = NULL; + p->Cleardelay_apply_events(); + } + } + eventCnt = 0; + return 0; +} diff --git a/src/search_local/index_storage/common/poller.h b/src/search_local/index_storage/common/poller.h new file mode 100644 index 0000000..fbbb87b --- /dev/null +++ b/src/search_local/index_storage/common/poller.h @@ -0,0 +1,161 @@ +/* + * ===================================================================================== + * + * Filename: pooler.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __POLLER_H__ +#define __POLLER_H__ + +#include +#include +#include "myepoll.h" +#include "list.h" + +#define EPOLL_DATA_SLOT(x) ((x)->data.u64 & 0xFFFFFFFF) +#define EPOLL_DATA_SEQ(x) ((x)->data.u64 >> 32) + +class PollerUnit; +class PollerObject; + +struct EpollSlot +{ + PollerObject *poller; + uint32_t seq; + uint32_t freeList; +}; + +struct EventSlot +{ + PollerObject *poller; +}; + +class PollerObject +{ +public: + PollerObject(PollerUnit *o = NULL, int fd = 0) : ownerUnit(o), + netfd(fd), + newEvents(0), + oldEvents(0), + epslot(0) + { + } + + virtual ~PollerObject(); + + virtual void input_notify(void); + virtual void output_notify(void); + virtual void hangup_notify(void); + + void enable_input(void) + { + newEvents |= EPOLLIN; + } + void enable_output(void) + { + newEvents |= EPOLLOUT; + } + void DisableInput(void) + { + newEvents &= ~EPOLLIN; + } + void disable_output(void) + { + newEvents &= ~EPOLLOUT; + } + + void enable_input(bool i) + { + if (i) + newEvents |= EPOLLIN; + else + newEvents &= ~EPOLLIN; + } + void enable_output(bool o) + { + if (o) + newEvents |= EPOLLOUT; + else + newEvents &= ~EPOLLOUT; + } + + int attach_poller(PollerUnit *thread = NULL); + int detach_poller(void); + int apply_events(); + int delay_apply_events(); + void Cleardelay_apply_events() { eventSlot = NULL; } + int check_link_status(void); + + void init_poll_fd(struct pollfd *); + + friend class PollerUnit; + +protected: + PollerUnit *ownerUnit; + int netfd; + int newEvents; + int oldEvents; + int epslot; + struct EventSlot *eventSlot; +}; + +class PollerUnit +{ +public: + friend class PollerObject; + PollerUnit(int mp); + virtual ~PollerUnit(); + + int set_max_pollers(int mp); + int get_max_pollers(void) const { return maxPollers; } + int initialize_poller_unit(void); + int wait_poller_events(int); + void process_poller_events(void); + int get_fd(void) { return epfd; } + int delay_apply_events(); + +private: + int verify_events(struct epoll_event *); + int Epctl(int op, int fd, struct epoll_event *events); + struct EpollSlot *get_slot(int n) { return &pollerTable[n]; } + const struct EpollSlot *get_slot(int n) const { return &pollerTable[n]; } + + void free_epoll_slot(int n); + int alloc_epoll_slot(void); + struct EventSlot *add_delay_event_poller(PollerObject *p) + { + if (eventCnt == totalEventSlot) + return NULL; + eventSlot[eventCnt++].poller = p; + return &eventSlot[eventCnt - 1]; + } + +private: + struct EpollSlot *pollerTable; + struct epoll_event *ep_events; + int epfd; + int eeSize; + int freeSlotList; + int maxPollers; + int usedPollers; + /* FIXME: maybe too small */ + static int totalEventSlot; + struct EventSlot eventSlot[40960]; + int eventCnt; + +protected: + int nrEvents; +}; + +#endif diff --git a/src/search_local/index_storage/common/proc_title.c b/src/search_local/index_storage/common/proc_title.c new file mode 100644 index 0000000..2bb3479 --- /dev/null +++ b/src/search_local/index_storage/common/proc_title.c @@ -0,0 +1,101 @@ +/* + * ===================================================================================== + * + * Filename: proc_title.c + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#ifndef PR_SET_NAME +#define PR_SET_NAME 15 +#endif + +#ifndef PR_GET_NAME +#define PR_GET_NAME 16 +#endif + +static char *arg_start; +static char *arg_end; +static char *env_start; + +void init_proc_title(int argc, char **argv) +{ + int i; + + arg_start = argv[0]; + arg_end = argv[argc - 1] + strlen(argv[argc - 1]) + 1; + env_start = environ[0]; + for (i = 0; i < argc; i++) + argv[i] = STRDUP(argv[i]); +} + +void set_proc_title(const char *title) +{ + int tlen = strlen(title) + 1; + int i; + char *p; + + if (arg_start == NULL) + return; + if (arg_end - arg_start < tlen && env_start == arg_end) + { + char *env_end = env_start; + for (i = 0; environ[i]; i++) + { + if (env_end == environ[i]) + { + env_end = environ[i] + strlen(environ[i]) + 1; + environ[i] = STRDUP(environ[i]); + } + else + break; + } + arg_end = env_end; + env_start = NULL; + } + i = arg_end - arg_start; + if (tlen == i) + { + strncpy(arg_start, title, i); + } + else if (tlen < i) + { + strncpy(arg_start, title, i); + memset(arg_start + tlen, 0, i - tlen); + } + else + { + *(char *)mempcpy(arg_start, title, i - 1) = '\0'; + } + if (env_start) + { + p = strchr(arg_start, ' '); + if (p) + *p = '\0'; + } +} + +void set_proc_name(const char *name) +{ + prctl(PR_SET_NAME, name); +} + +void get_proc_name(char *name) +{ + prctl(PR_GET_NAME, name); +} diff --git a/src/search_local/index_storage/common/proc_title.h b/src/search_local/index_storage/common/proc_title.h new file mode 100644 index 0000000..cf36405 --- /dev/null +++ b/src/search_local/index_storage/common/proc_title.h @@ -0,0 +1,29 @@ +/* + * ===================================================================================== + * + * Filename: proc_title.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_PROCTITLE_H +#define __DTC_PROCTITLE_H + +extern "C" +{ + extern void init_proc_title(int, char **); + extern void set_proc_title(const char *title); + extern void set_proc_name(const char *); + extern void get_proc_name(char *); +} + +#endif diff --git a/src/search_local/index_storage/common/protocol.h b/src/search_local/index_storage/common/protocol.h new file mode 100644 index 0000000..618c10f --- /dev/null +++ b/src/search_local/index_storage/common/protocol.h @@ -0,0 +1,176 @@ +/* + * ===================================================================================== + * + * Filename: protocol.h + * + * Description: request task class base. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_PROTOCOL_H_ +#define __CH_PROTOCOL_H_ +#include + +#define MAXFIELDS_PER_TABLE 255 +#define MAXPACKETSIZE (64 << 20) + +class DField +{ +public: + enum + { + None = 0, // undefined + Signed = 1, // Signed Integer + Unsigned = 2, // Unsigned Integer + Float = 3, // FloatPoint + String = 4, // String, case insensitive, null ended + Binary = 5, // opaque binary data + TotalType + }; + + enum + { + Set = 0, + Add = 1, + SetBits = 2, + OR = 3, + TotalOperation + }; + + enum + { + EQ = 0, + NE = 1, + LT = 2, + LE = 3, + GT = 4, + GE = 5, + TotalComparison + }; +}; + +class DRequest +{ +public: + enum + { + Nop = 0, + result_code = 1, + DTCResultSet = 2, + SvrAdmin = 3, + Get = 4, + Purge = 5, + Insert = 6, + Update = 7, + Delete = 8, + Replace = 12, + Flush = 13, + Invalidate = 14, // OBSOLETED + Monitor = 15, + ReloadConfig = 16, //work helper 重新载入配置文件 + Replicate = 17, // master-slave backup + LocalMigrate = 18, // for cluster scales + }; + + class Flag + { + public: + enum + { + KeepAlive = 1, + NeedTableDefinition = 2, + no_cache = 4, + NoResult = 8, + no_next_server = 16, + MultiKeyValue = 32, + admin_table = 64, + }; + }; + + class Section + { + public: + enum + { + VersionInfo = 0, + table_definition = 1, + RequestInfo = 2, + ResultInfo = 3, + UpdateInfo = 4, + ConditionInfo = 5, + FieldSet = 6, + DTCResultSet = 7, + Total + }; + }; + + class ServerAdminCmd + { + public: + enum CMD + { + RegisterHB = 1, + LogoutHB = 2, + GetKeyList = 3, + GetUpdateKey = 4, + GetRawData = 5, + ReplaceRawData = 6, + AdjustLRU = 7, + VerifyHBT = 8, + GetHBTime = 9, + SET_READONLY = 10, + SET_READWRITE = 11, + QueryServerInfo = 12, + NodeHandleChange = 13, + Migrate = 14, + ReloadClusterNodeList = 15, + SetClusterNodeState = 16, + change_node_address = 17, + GetClusterState = 18, + PurgeForHit = 19, + QUERY_MEM_INFO = 20, + ClearCache = 21, + MigrateDB = 22, + MigrateDBSwitch = 23, + ColExpandStatus = 24, + col_expand = 25, + ColExpandDone = 26, + ColExpandKey = 27, + Cascade = 28, + }; + }; +}; + +struct PacketHeader +{ + uint8_t version; + uint8_t scts; + uint8_t flags; + uint8_t cmd; + uint32_t len[DRequest::Section::Total]; +}; + +struct DTCServerInfo +{ + uint8_t version; + uint8_t reserved[3]; + uint32_t binlog_id; + uint32_t binlog_off; + uint64_t memsize; + uint64_t datasize; +}; + +struct DTCTimeInfo +{ + uint64_t time; +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_client.cc b/src/search_local/index_storage/common/proxy_client.cc new file mode 100644 index 0000000..0eed2c5 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_client.cc @@ -0,0 +1,370 @@ +/* + * ===================================================================================== + * + * Filename: proxy_client.cc + * + * Description: agent client class. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "proxy_client.h" +#include "proxy_client_unit.h" +#include "proxy_receiver.h" +#include "proxy_sender.h" +#include "log.h" +#include "poll_thread.h" +#include "proxy_multi_request.h" +#include "packet.h" +#include "task_request.h" +#include "proxy_multi_request.h" +#include "stat_dtc.h" +#include "table_def_manager.h" + +extern DTCTableDefinition *gTableDef[]; + +static StatItemU32 statAgentAcceptCount; +static StatItemU32 statAgentCurConnCount; +static StatItemU32 statAgentExpireCount; +static StatItemU32 statTaskClientTimeout; + +AgentResultQueue::~AgentResultQueue() +{ + Packet *p; + + while (NULL != (p = packet.Pop())) + { + p->free_result_buff(); + delete p; + } +} + +class AgentReply : public ReplyDispatcher +{ +public: + AgentReply() + { + statInitFlag = false; + } + virtual ~AgentReply() {} + virtual void reply_notify(TaskRequest *task); + +private: + bool statInitFlag; +}; + +void AgentReply::reply_notify(TaskRequest *task) +{ + log_debug("AgentReply::reply_notify start"); + + ClientAgent *client = task->owner_client(); + if (client == NULL) + { + /* client gone, finish this task */ + task->done_one_agent_sub_request(); + return; + } + + client->record_request_process_time(task); + + int client_timeout = task->requestInfo.tag_present(1) == 0 ? task->default_expire_time() : task->requestInfo.get_expire_time(task->versionInfo.CTLibIntVer()); + int req_delaytime = task->responseTimer.live(); + + if (!statInitFlag) + { + statAgentExpireCount = statmgr.get_item_u32(INCOMING_EXPIRE_REQ); + statTaskClientTimeout = statmgr.get_item_u32(TASK_CLIENT_TIMEOUT); + statInitFlag = true; + } + + statTaskClientTimeout = client_timeout; + log_debug("task client_timeout: %d", client_timeout); + + if ((req_delaytime / 1000) >= client_timeout) //ms + { + log_debug("AgentReply::reply_notify client_timeout[%d]ms, req delay time[%d]us", client_timeout, req_delaytime); + task->done_one_agent_sub_request(); + statAgentExpireCount++; + return; + } + Packet *packet = new Packet(); + if (packet == NULL) + { + /* make response error, finish this task */ + task->done_one_agent_sub_request(); + log_crit("no mem new Packet"); + return; + } + + packet->encode_result(task); + task->detach_result_in_result_writer(); + task->done_one_agent_sub_request(); + + client->add_packet(packet); + if (client->send_result() < 0) + { + log_error("cliengAgent send_result error"); + delete client; + return; + } + + log_debug("AgentReply::reply_notify stop"); +} + +static AgentReply agentReply; + +/* sender and receiver should inited ok */ +ClientAgent::ClientAgent(PollThread *o, AgentClientUnit *u, int fd) : PollerObject(o, fd), + ownerThread(o), + owner(u), + tlist(NULL) +{ + tlist = u->get_timer_list(); + sender = new AgentSender(fd); + if (NULL == sender) + { + log_error("no mem to new sender"); + throw(int) - ENOMEM; + } + + if (sender && sender->Init() < 0) + { + delete sender; + sender = NULL; + log_error("no mem to init sender"); + throw(int) - ENOMEM; + } + + if (sender) + { + receiver = new AgentReceiver(fd); + if (NULL == receiver) + { + log_error("no mem to new receiver"); + throw(int) - ENOMEM; + } + + if (receiver && receiver->Init() < 0) + { + log_error("no mem to init receiver"); + throw(int) - ENOMEM; + } + } + + statAgentAcceptCount = statmgr.get_item_u32(AGENT_ACCEPT_COUNT); + statAgentCurConnCount = statmgr.get_item_u32(AGENT_CONN_COUNT); + + statAgentAcceptCount++; + statAgentCurConnCount++; +} + +ClientAgent::~ClientAgent() +{ + log_debug("~ClientAgent start"); + ListObject *node = rememberReqHeader.ListNext(); + AgentMultiRequest *req; + + /* notify all request of this client I'm gone */ + while (node != &rememberReqHeader) + { + req = node->ListOwner(); + req->clear_owner_info(); + req->detach_from_owner_client(); + node = rememberReqHeader.ListNext(); + } + + if (receiver) + delete receiver; + if (sender) + delete sender; + + detach_poller(); + + statAgentCurConnCount--; + log_debug("~ClientAgent end"); +} + +int ClientAgent::attach_thread() +{ + disable_output(); + enable_input(); + + if (attach_poller() < 0) + { + log_error("client agent attach agengInc thread failed"); + return -1; + } + + /* no idle test */ + return 0; +} + +void ClientAgent::remember_request(TaskRequest *request) +{ + request->link_to_owner_client(rememberReqHeader); +} + +TaskRequest *ClientAgent::prepare_request(char *recvbuff, int recvlen, int pktcnt) +{ + TaskRequest *request; + + request = new TaskRequest(TableDefinitionManager::Instance()->get_cur_table_def()); + if (NULL == request) + { + free(recvbuff); + log_crit("no mem allocate for new agent request"); + return NULL; + } + + request->set_hotbackup_table(TableDefinitionManager::Instance()->get_hot_backup_table_def()); + request->set_owner_info(this, 0, NULL); + request->set_owner_client(this); + request->push_reply_dispatcher(&agentReply); + request->save_recved_result(recvbuff, recvlen, pktcnt); + + /* assume only a few sub request decode error */ + if (request->decode_agent_request() < 0) + { + delete request; + return NULL; + } + + /* no mem new task case */ + if (request->is_agent_request_completed()) + { + delete request; + return NULL; + } + + remember_request(request); + + return request; +} + +int ClientAgent::recv_request() +{ + RecvedPacket packets; + char *recvbuff = NULL; + int recvlen = 0; + int pktcnt = 0; + TaskRequest *request = NULL; + + packets = receiver->Recv(); + + if (packets.err < 0) + return -1; + else if (packets.pktCnt == 0) + return 0; + + recvbuff = packets.buff; + recvlen = packets.len; + pktcnt = packets.pktCnt; + + request = prepare_request(recvbuff, recvlen, pktcnt); + if (request != NULL) + owner->task_notify(request); + + return 0; +} + +/* exit when recv error*/ +void ClientAgent::input_notify() +{ + log_debug("ClientAgent::input_notify() start"); + if (recv_request() < 0) + { + log_debug("erro when recv"); + delete this; + return; + } + delay_apply_events(); + log_debug("ClientAgent::input_notify() stop"); + return; +} + +/* +return error if sender broken +*/ +int ClientAgent::send_result() +{ + if (sender->is_broken()) + { + log_error("sender broken"); + return -1; + } + + while (1) + { + + Packet *frontPkt = resQueue.Front(); + if (NULL == frontPkt) + { + + break; + } + + if (frontPkt->vec_count() + sender->vec_count() > SENDER_MAX_VEC) + { + /*这个地方打印error，如果在10s内会5次走入此次分支的话，统计子进程会上报告警*/ + log_error("the sum value of front packet veccount[%d] and sender veccount[%d]is greater than SENDER_MAX_VEC[%d]", + frontPkt->vec_count(), sender->vec_count(), SENDER_MAX_VEC); + break; + } + else + { + Packet *pkt; + pkt = resQueue.Pop(); + if (NULL == pkt) + { + break; + } + if (sender->add_packet(pkt) < 0) + { + return -1; + } + } + } + + if (sender->Send() < 0) + { + log_error("agent client send error"); + return -1; + } + + if (sender->left_len() != 0) + enable_output(); + else + disable_output(); + + delay_apply_events(); + + return 0; +} + +void ClientAgent::output_notify() +{ + log_debug("ClientAgent::output_notify() start"); + if (send_result() < 0) + { + log_debug("error when response"); + delete this; + return; + } + log_debug("ClientAgent::output_notify() stop"); + + return; +} + +void ClientAgent::record_request_process_time(TaskRequest *task) +{ + owner->record_request_time(task); +} diff --git a/src/search_local/index_storage/common/proxy_client.h b/src/search_local/index_storage/common/proxy_client.h new file mode 100644 index 0000000..c93df55 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_client.h @@ -0,0 +1,81 @@ +/* + * ===================================================================================== + * + * Filename: proxy_client.h + * + * Description: agent client class. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_CLIENT_H__ +#define __AGENT_CLIENT_H__ + +#include + +#include "poller.h" +#include "timer_list.h" +#include "lqueue.h" +#include "value.h" +#include "proxy_receiver.h" + +class Packet; +class AgentResultQueue +{ +public: + LinkQueue packet; + + AgentResultQueue() {} + ~AgentResultQueue(); + + inline void Push(Packet *pkt) { packet.Push(pkt); } + inline Packet *Pop() { return packet.Pop(); } + inline Packet *Front() { return packet.Front(); } + inline bool queue_empty() { return packet.queue_empty(); } +}; + +class PollThread; +class AgentClientUnit; +class AgentReceiver; +class AgentSender; +class AgentMultiRequest; +class AgentMultiRequest; +class TaskRequest; +class ClientAgent : public PollerObject, public TimerObject +{ +public: + ClientAgent(PollThread *o, AgentClientUnit *u, int fd); + virtual ~ClientAgent(); + + int attach_thread(); + inline void add_packet(Packet *p) { resQueue.Push(p); } + void remember_request(AgentMultiRequest *agentrequest); + int send_result(); + void record_request_process_time(TaskRequest *task); + + virtual void input_notify(); + virtual void output_notify(); + +private: + PollThread *ownerThread; + AgentClientUnit *owner; + TimerList *tlist; + + AgentReceiver *receiver; + AgentSender *sender; + AgentResultQueue resQueue; + ListObject rememberReqHeader; + + TaskRequest *prepare_request(char *recvbuff, int recvlen, int pktcnt); + int recv_request(); + void remember_request(TaskRequest *request); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_client_unit.cc b/src/search_local/index_storage/common/proxy_client_unit.cc new file mode 100644 index 0000000..a3eae6f --- /dev/null +++ b/src/search_local/index_storage/common/proxy_client_unit.cc @@ -0,0 +1,82 @@ +/* + * ===================================================================================== + * + * Filename: proxy_client_unit.cc + * + * Description: proxy client unit. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "proxy_client_unit.h" +#include "timer_list.h" +#include "task_request.h" +#include "poll_thread.h" + +AgentClientUnit::AgentClientUnit(PollThread *t, int c) : ownerThread(t), + output(t), + check(c) +{ + tlist = t->get_timer_list(c); + + statRequestTime[0] = statmgr.get_sample(REQ_USEC_ALL); + statRequestTime[1] = statmgr.get_sample(REQ_USEC_GET); + statRequestTime[2] = statmgr.get_sample(REQ_USEC_INS); + statRequestTime[3] = statmgr.get_sample(REQ_USEC_UPD); + statRequestTime[4] = statmgr.get_sample(REQ_USEC_DEL); + statRequestTime[5] = statmgr.get_sample(REQ_USEC_FLUSH); + statRequestTime[6] = statmgr.get_sample(REQ_USEC_HIT); + statRequestTime[7] = statmgr.get_sample(REQ_USEC_REPLACE); +} + +AgentClientUnit::~AgentClientUnit() +{ +} + +void AgentClientUnit::bind_dispatcher(TaskDispatcher *proc) +{ + output.bind_dispatcher(proc); +} + +void AgentClientUnit::record_request_time(int hit, int type, unsigned int usec) +{ + static const unsigned char cmd2type[] = + { + /*Nop*/ 0, + /*result_code*/ 0, + /*DTCResultSet*/ 0, + /*HelperAdmin*/ 0, + /*Get*/ 1, + /*Purge*/ 5, + /*Insert*/ 2, + /*Update*/ 3, + /*Delete*/ 4, + /*Other*/ 0, + /*Other*/ 0, + /*Other*/ 0, + /*Replace*/ 7, + /*Flush*/ 5, + + /*Invalidate*/ 0, + /*Monitor*/ 0, + /*ReloadConfig*/ 0, + /*Replicate*/ 1, + /*Migrate*/ 1, + }; + statRequestTime[0].push(usec); + unsigned int t = hit ? 6 : cmd2type[type]; + if (t) + statRequestTime[t].push(usec); +} + +void AgentClientUnit::record_request_time(TaskRequest *req) +{ + record_request_time(req->flag_is_hit(), req->request_code(), req->responseTimer.live()); +} diff --git a/src/search_local/index_storage/common/proxy_client_unit.h b/src/search_local/index_storage/common/proxy_client_unit.h new file mode 100644 index 0000000..52ede1d --- /dev/null +++ b/src/search_local/index_storage/common/proxy_client_unit.h @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: proxy_client_unit.h + * + * Description: agent client unit. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_CLIENT_UNIT_H__ +#define __AGENT_CLIENT_UNIT_H__ + +#include "request_base_all.h" +#include "stat_dtc.h" + +class TimerList; +class PollThread; +class TaskRequest; +class AgentClientUnit +{ +public: + AgentClientUnit(PollThread *t, int c); + virtual ~AgentClientUnit(); + + inline TimerList *get_timer_list() { return tlist; } + void bind_dispatcher(TaskDispatcher *proc); + inline void task_notify(TaskRequest *req) { output.task_notify(req); } + void record_request_time(int hit, int type, unsigned int usec); + void record_request_time(TaskRequest *req); + +private: + PollThread *ownerThread; + RequestOutput output; + int check; + TimerList *tlist; + StatSample statRequestTime[8]; +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_listen_pool.cc b/src/search_local/index_storage/common/proxy_listen_pool.cc new file mode 100644 index 0000000..12f7dc2 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_listen_pool.cc @@ -0,0 +1,189 @@ +/* + * ===================================================================================== + * + * Filename: proxy_listen_pool.cc + * + * Description: agent listen pool + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "proxy_listen_pool.h" +#include "proxy_listener.h" +#include "proxy_client_unit.h" +#include "poll_thread.h" +#include "config.h" +#include "task_request.h" +#include "log.h" +#include "stat_dtc.h" + +AgentListenPool::AgentListenPool() +{ + memset(thread, 0, sizeof(PollThread *) * MAX_AGENT_LISTENER); + memset(out, 0, sizeof(AgentClientUnit *) * MAX_AGENT_LISTENER); + memset(listener, 0, sizeof(AgentListener *) * MAX_AGENT_LISTENER); + + StatItemU32 statAgentCurConnCount = statmgr.get_item_u32(AGENT_CONN_COUNT); + statAgentCurConnCount = 0; +} + +AgentListenPool::~AgentListenPool() +{ + for (int i = 0; i < MAX_AGENT_LISTENER; i++) + { + if (thread[i]) + thread[i]->interrupt(); + delete listener[i]; + delete out[i]; + } +} + +/* part of framework construction */ +int AgentListenPool::Bind(DTCConfig *gc, TaskDispatcher *agentprocess) +{ + char bindstr[64]; + const char *bindaddr; + const char *errmsg = NULL; + char threadname[64]; + int checktime; + int blog; + + checktime = gc->get_int_val("cache", "AgentRcvBufCheck", 5); + blog = gc->get_int_val("cache", "AgentListenBlog", 256); + + for (int i = 0; i < MAX_AGENT_LISTENER; i++) + { + if (i == 0) + snprintf(bindstr, sizeof(bindstr), "BindAddr"); + else + snprintf(bindstr, sizeof(bindstr), "BindAddr%d", i); + + bindaddr = gc->get_str_val("cache", bindstr); + if (NULL == bindaddr) + continue; + + if ((errmsg = addr[i].set_address(bindaddr, (const char *)NULL))) + continue; + + snprintf(threadname, sizeof(threadname), "inc%d", i); + thread[i] = new PollThread(threadname); + if (NULL == thread[i]) + { + log_error("no mem to new agent inc thread %d", i); + return -1; + } + if (thread[i]->initialize_thread() < 0) + { + log_error("agent inc thread %d init error", i); + return -1; + } + + out[i] = new AgentClientUnit(thread[i], checktime); + if (NULL == out[i]) + { + log_error("no mem to new agent client unit %d", i); + return -1; + } + out[i]->bind_dispatcher(agentprocess); + + listener[i] = new AgentListener(thread[i], out[i], addr[i]); + if (NULL == listener[i]) + { + log_error("no mem to new agent listener %d", i); + return -1; + } + if (listener[i]->Bind(blog) < 0) + { + log_error("agent listener %d bind error", i); + return -1; + } + + if (listener[i]->attach_thread() < 0) + return -1; + } + + return 0; +} + +int AgentListenPool::Bind(DTCConfig *gc, TaskDispatcher *agentprocess, PollThread *workThread) +{ + char bindstr[64]; + const char *bindaddr; + const char *errmsg = NULL; + int checktime; + int blog; + + checktime = gc->get_int_val("cache", "AgentRcvBufCheck", 5); + blog = gc->get_int_val("cache", "AgentListenBlog", 256); + + snprintf(bindstr, sizeof(bindstr), "BindAddr"); + + bindaddr = gc->get_str_val("cache", bindstr); + if (NULL == bindaddr) + { + log_error("get cache BindAddr configure failed"); + return -1; + } + + if ((errmsg = addr[0].set_address(bindaddr, (const char *)NULL))) + { + log_error("addr[0] setaddress failed"); + return -1; + } + + thread[0] = workThread; + + out[0] = new AgentClientUnit(thread[0], checktime); + if (NULL == out[0]) + { + log_error("no mem to new agent client unit"); + return -1; + } + out[0]->bind_dispatcher(agentprocess); + + listener[0] = new AgentListener(thread[0], out[0], addr[0]); + if (NULL == listener[0]) + { + log_error("no mem to new agent listener"); + return -1; + } + if (listener[0]->Bind(blog) < 0) + { + log_error("agent listener bind error"); + return -1; + } + if (listener[0]->attach_thread() < 0) + return -1; + + return 0; +} + +int AgentListenPool::Run() +{ + for (int i = 0; i < MAX_AGENT_LISTENER; i++) + { + if (thread[i]) + thread[i]->running_thread(); + } + + return 0; +} + +int AgentListenPool::Match(const char *host, const char *port) +{ + for (int i = 0; i < MAX_AGENT_LISTENER; i++) + { + if (listener[i] == NULL) + continue; + if (addr[i].Match(host, port)) + return 1; + } + return 0; +} diff --git a/src/search_local/index_storage/common/proxy_listen_pool.h b/src/search_local/index_storage/common/proxy_listen_pool.h new file mode 100644 index 0000000..55de09f --- /dev/null +++ b/src/search_local/index_storage/common/proxy_listen_pool.h @@ -0,0 +1,49 @@ +/* + * ===================================================================================== + * + * Filename: proxy_listen_pool.h + * + * Description: agent listen pool + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_LISTEN_POOL_H__ +#define __AGENT_LISTEN_POOL_H__ + +#include "request_base.h" +#include "net_addr.h" + +#define MAX_AGENT_LISTENER 10 + +class AgentClientUnit; +class AgentListener; +class PollThread; +class DTCConfig; +class TaskRequest; +class AgentListenPool +{ +private: + SocketAddress addr[MAX_AGENT_LISTENER]; + PollThread *thread[MAX_AGENT_LISTENER]; + AgentClientUnit *out[MAX_AGENT_LISTENER]; + AgentListener *listener[MAX_AGENT_LISTENER]; + +public: + AgentListenPool(); + ~AgentListenPool(); + + int Bind(DTCConfig *gc, TaskDispatcher *out); + int Bind(DTCConfig *gc, TaskDispatcher *out, PollThread *workThread); + int Run(); + int Match(const char *host, const char *port); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_listener.cc b/src/search_local/index_storage/common/proxy_listener.cc new file mode 100644 index 0000000..b31c405 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_listener.cc @@ -0,0 +1,92 @@ +/* + * ===================================================================================== + * + * Filename: proxy_listener.cc + * + * Description: agent listener + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "proxy_listener.h" +#include "proxy_client.h" +#include "poll_thread.h" +#include "log.h" + +AgentListener::AgentListener(PollThread *t, AgentClientUnit *o, SocketAddress &a) : PollerObject(t, 0), + ownerThread(t), + out(o), + addr(a) +{ +} + +AgentListener::~AgentListener() +{ +} + +/* part of framework construction */ +int AgentListener::Bind(int blog) +{ + if ((netfd = sock_bind(&addr, blog, 0, 0, 1 /*reuse*/, 1 /*nodelay*/, 1 /*defer_accept*/)) == -1) + return -1; + return 0; +} + +int AgentListener::attach_thread() +{ + enable_input(); + if (PollerObject::attach_poller() < 0) + { + log_error("agent listener attach agentInc thread error"); + return -1; + } + return 0; +} + +void AgentListener::input_notify() +{ + while (1) + { + int newfd; + struct sockaddr peer; + socklen_t peersize = sizeof(peer); + + newfd = accept(netfd, &peer, &peersize); + if (newfd < 0) + { + if (EINTR != errno && EAGAIN != errno) + log_error("agent listener accept error, %m"); + break; + } + + log_debug("new CAgentClient accepted!!"); + + ClientAgent *client; + try + { + client = new ClientAgent(ownerThread, out, newfd); + } + catch (int err) + { + return; + } + + if (NULL == client) + { + log_error("no mem for new client agent"); + break; + } + + client->attach_thread(); + client->input_notify(); + } +} diff --git a/src/search_local/index_storage/common/proxy_listener.h b/src/search_local/index_storage/common/proxy_listener.h new file mode 100644 index 0000000..a295949 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_listener.h @@ -0,0 +1,42 @@ +/* + * ===================================================================================== + * + * Filename: proxy_listener.h + * + * Description: agent listener + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_LISTENER_H__ +#define __AGENT_LISTENER_H__ + +#include "poller.h" +#include "net_addr.h" + +class PollThread; +class AgentClientUnit; +class AgentListener : public PollerObject +{ +private: + PollThread *ownerThread; + AgentClientUnit *out; + const SocketAddress addr; + virtual void input_notify(); + +public: + AgentListener(PollThread *t, AgentClientUnit *o, SocketAddress &a); + virtual ~AgentListener(); + + int Bind(int blog); + int attach_thread(); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_multi_request.cc b/src/search_local/index_storage/common/proxy_multi_request.cc new file mode 100644 index 0000000..1514322 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_multi_request.cc @@ -0,0 +1,169 @@ +/* + * ===================================================================================== + * + * Filename: proxy_multi_request.cc + * + * Description: agent multi request. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "proxy_multi_request.h" +#include "task_request.h" +#include "proxy_client.h" +#include "table_def_manager.h" + +extern DTCTableDefinition *gTableDef[]; + +AgentMultiRequest::AgentMultiRequest(TaskRequest *o) : packetCnt(0), + owner(o), + taskList(NULL), + compleTask(0), + ownerClient(NULL) +{ + if (o) + ownerClient = o->owner_client(); +} + +AgentMultiRequest::~AgentMultiRequest() +{ + list_del(); + + if (taskList) + { + for (int i = 0; i < packetCnt; i++) + if (taskList[i].task) + delete taskList[i].task; + + delete[] taskList; + } + + if (!!packets) + free(packets.ptr); +} + +void AgentMultiRequest::complete_task(int index) +{ + if (taskList[index].task) + { + delete taskList[index].task; + taskList[index].task = NULL; + } + + compleTask++; + /* delete owner taskrequest along with us if all sub request's result putted into ClientAgent's send queue */ + + if (compleTask == packetCnt) + { + delete owner; + } +} + +void AgentMultiRequest::clear_owner_info() +{ + ownerClient = NULL; + + if (taskList == NULL) + return; + + for (int i = 0; i < packetCnt; i++) + { + if (taskList[i].task) + taskList[i].task->clear_owner_client(); + } +} + +/* +error case: set this task processed +1. no mem: set task processed +2. decode error: set task processed, reply this task +*/ +void AgentMultiRequest::DecodeOneRequest(char *packetstart, int packetlen, int index) +{ + int err = 0; + TaskRequest *task = NULL; + DecodeResult decoderes; + + task = new TaskRequest(TableDefinitionManager::Instance()->get_cur_table_def()); + if (NULL == task) + { + log_crit("not enough mem for new task creation, client wont recv response"); + compleTask++; + return; + } + + task->set_hotbackup_table(TableDefinitionManager::Instance()->get_hot_backup_table_def()); + decoderes = task->Decode(packetstart, packetlen, 2); + switch (decoderes) + { + default: + case DecodeFatalError: + if (errno != 0) + log_notice("decode fatal error, msg = %m"); + break; + case DecodeDataError: + task->response_timer_start(); + task->mark_as_hit(); + taskList[index].processed = 1; + break; + case DecodeDone: + if ((err = task->prepare_process()) < 0) + { + log_error("build packed key error: %d, %s", err, task->resultInfo.error_message()); + taskList[index].processed = 1; + } + break; + } + + task->set_owner_info(this, index, NULL); + task->set_owner_client(this->ownerClient); + + taskList[index].task = task; + + return; +} + +int AgentMultiRequest::decode_agent_request() +{ + int cursor = 0; + + taskList = new DecodedTask[packetCnt]; + if (NULL == taskList) + { + log_crit("no mem new taskList"); + return -1; + } + memset((void *)taskList, 0, sizeof(DecodedTask) * packetCnt); + + /* whether can work, reply on input buffer's correctness */ + for (int i = 0; i < packetCnt; i++) + { + char *packetstart; + int packetlen; + + packetstart = packets.ptr + cursor; + packetlen = packet_body_len(*(PacketHeader *)packetstart) + sizeof(PacketHeader); + + DecodeOneRequest(packetstart, packetlen, i); + + cursor += packetlen; + } + + return 0; +} + +void AgentMultiRequest::copy_reply_for_sub_task() +{ + for (int i = 0; i < packetCnt; i++) + { + if (taskList[i].task) + taskList[i].task->copy_reply_path(owner); + } +} diff --git a/src/search_local/index_storage/common/proxy_multi_request.h b/src/search_local/index_storage/common/proxy_multi_request.h new file mode 100644 index 0000000..05d4562 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_multi_request.h @@ -0,0 +1,63 @@ +/* + * ===================================================================================== + * + * Filename: proxy_multi_request.h + * + * Description: agent multi request. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef AGENT_MULTI_REQUEST_H___ +#define AGENT_MULTI_REQUEST_H___ + +#include "list.h" +#include "value.h" + +class TaskRequest; +typedef struct +{ + TaskRequest *volatile task; + volatile int processed; +} DecodedTask; + +class ClientAgent; +class AgentMultiRequest : public ListObject +{ +public: + AgentMultiRequest(TaskRequest *o); + virtual ~AgentMultiRequest(); + + int decode_agent_request(); + inline int packet_count() { return packetCnt; } + inline TaskRequest *curr_task(int index) { return taskList[index].task; } + void copy_reply_for_sub_task(); + void clear_owner_info(); + inline bool is_completed() { return compleTask == packetCnt; } + void complete_task(int index); + inline void detach_from_owner_client() { list_del(); } + inline bool is_curr_task_processed(int index) { return taskList[index].processed == 1; } + inline void save_recved_result(char *buff, int len, int pktcnt) + { + packets.Set(buff, len); + packetCnt = pktcnt; + } + +private: + DTCBinary packets; + int packetCnt; + TaskRequest *owner; + DecodedTask *volatile taskList; + volatile int compleTask; + ClientAgent *volatile ownerClient; + void DecodeOneRequest(char *packetstart, int packetlen, int index); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_process.cc b/src/search_local/index_storage/common/proxy_process.cc new file mode 100644 index 0000000..17dcfff --- /dev/null +++ b/src/search_local/index_storage/common/proxy_process.cc @@ -0,0 +1,62 @@ +/* + * ===================================================================================== + * + * Filename: agent_process.cc + * + * Description: agent task process. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "proxy_process.h" +#include "poll_thread.h" +#include "task_request.h" +#include "log.h" + +AgentProcess::AgentProcess(PollThread *o) : TaskDispatcher(o), + ownerThread(o), + output(o) +{ +} + +AgentProcess::~AgentProcess() +{ +} + +void AgentProcess::task_notify(TaskRequest *curr) +{ + curr->copy_reply_for_agent_sub_task(); + log_debug("AgentProcess::task_notify start"); + //there is a race condition here: + //curr may be deleted during process (in task->reply_notify()) + int taskCount = curr->agent_sub_task_count(); + log_debug("AgentProcess::task_notify task count is %d", taskCount); + for (int i = 0; i < taskCount; i++) + { + TaskRequest *task = NULL; + + if (NULL == (task = curr->curr_agent_sub_task(i))) + continue; + + if (curr->is_curr_sub_task_processed(i)) + { + log_debug("AgentProcess::task_notify task reply notify"); + task->reply_notify(); + } + + else + { + log_debug("AgentProcess::task_notify task_notify next process"); + output.task_notify(task); + } + } + + return; +} diff --git a/src/search_local/index_storage/common/proxy_process.h b/src/search_local/index_storage/common/proxy_process.h new file mode 100644 index 0000000..65cb282 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_process.h @@ -0,0 +1,43 @@ +/* + * ===================================================================================== + * + * Filename: proxy_process.h + * + * Description: agent task process. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_PROCESS_H__ +#define __AGENT_PROCESS_H__ + +#include "request_base.h" + +class PollThread; +class TaskRequest; + +class AgentProcess : public TaskDispatcher +{ +private: + PollThread *ownerThread; + RequestOutput output; + +public: + AgentProcess(PollThread *o); + virtual ~AgentProcess(); + + inline void bind_dispatcher(TaskDispatcher *p) + { + output.bind_dispatcher(p); + } + virtual void task_notify(TaskRequest *curr); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_receiver.cc b/src/search_local/index_storage/common/proxy_receiver.cc new file mode 100644 index 0000000..775a517 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_receiver.cc @@ -0,0 +1,302 @@ +/* + * ===================================================================================== + * + * Filename: proxy_receiver.cc + * + * Description: agent receiver. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "proxy_receiver.h" +#include "log.h" +#include "mem_check.h" +#include "protocol.h" +#include "task_request.h" + +AgentReceiver::AgentReceiver(int f) : fd(f), + buffer(NULL), + offset(0), + buffSize(0), + pktTail(0), + pktCnt(0) +{ +} + +AgentReceiver::~AgentReceiver() +{ + /* fd will closed by ClientAgent */ + if (buffer) + free(buffer); +} + +int AgentReceiver::Init() +{ + buffer = (char *)malloc(AGENT_INIT_RECV_BUFF_SIZE); + if (NULL == buffer) + return -1; + buffSize = AGENT_INIT_RECV_BUFF_SIZE; + return 0; +} + +/* +discription: recv +output: +<0: recv error or remote close +=0: nothing recved +>0: recved +*/ +int AgentReceiver::recv_once() +{ + int rv; + + rv = recv(fd, offset + buffer, buffSize - offset, 0); + if (rv < 0) + { + if (EAGAIN == errno || EINTR == errno || EINPROGRESS == errno) + return 0; + log_error("agent receiver recv error: %m, %d", errno); + return -errno; + } + else if (0 == rv) + { + log_debug("remote close connection, fd[%d]", fd); + errno = ECONNRESET; + return -errno; + } + + offset += rv; + + return rv; +} + +/* +return value: +<0: error +=0: nothing recved or have no completed packet +>0: recved and have completed packet +*/ +RecvedPacket AgentReceiver::Recv() +{ + int err = 0; + RecvedPacket packet; + + memset((void *)&packet, 0, sizeof(packet)); + + if ((err = real_recv()) < 0) + { + packet.err = -1; + return packet; + } + else if (err == 0) + return packet; + + if ((err = recv_again()) < 0) + { + packet.err = -1; + return packet; + } + + set_recved_info(packet); + + return packet; +} + +/* +discription: recv data +output: +<0: recv error or remote close +=0: nothing recved +>0: received +*/ +int AgentReceiver::real_recv() +{ + int rv = 0; + + if (is_need_enlarge_buffer()) + { + if (enlarge_buffer() < 0) + { + log_crit("no mme enlarge recv buffer error"); + return -ENOMEM; + } + } + + if ((rv = recv_once()) < 0) + return -1; + + return rv; +} + +/* +output: +=0: no need enlarge recv buffer or enlarge ok but recv nothing +<0: error +>0: recved +*/ +int AgentReceiver::recv_again() +{ + int rv = 0; + + if (!is_need_enlarge_buffer()) + return 0; + + if (enlarge_buffer() < 0) + { + log_crit("no mme enlarge recv buffer error"); + return -ENOMEM; + } + + if ((rv = recv_once()) < 0) + return -1; + + return rv; +} + +int AgentReceiver::decode_header(PacketHeader *header) +{ + if (header->version != 1) + { // version not supported + log_error("version incorrect: %d", header->version); + return -1; + } + + if (header->scts != DRequest::Section::Total) + { // tags# mismatch + log_error("session count incorrect: %d", header->scts); + return -1; + } + + int pktbodylen = 0; + for (int i = 0; i < DRequest::Section::Total; i++) + { +#if __BYTE_ORDER == __BIG_ENDIAN + const unsigned int v = bswap_32(header->len[i]); +#else + const unsigned int v = header->len[i]; +#endif + + if (v > MAXPACKETSIZE) + { + log_error("section [%d] len > MAXPACKETSIZE", i); + return -1; + } + + pktbodylen += v; + } + + if (pktbodylen > MAXPACKETSIZE) + { + log_error("packet len > MAXPACKETSIZE 20M"); + return -1; + } + + return pktbodylen; +} + +int AgentReceiver::count_packet() +{ + char *pos = buffer; + int leftlen = offset; + + pktCnt = 0; + + if (pos == NULL || leftlen == 0) + return 0; + + while (1) + { + int pktbodylen = 0; + PacketHeader *header = NULL; + + if (leftlen < (int)sizeof(PacketHeader)) + break; + + header = (PacketHeader *)pos; + pktbodylen = decode_header(header); + if (pktbodylen < 0) + return -1; + + if (leftlen < (int)sizeof(PacketHeader) + pktbodylen) + break; + + pos += sizeof(PacketHeader) + pktbodylen; + leftlen -= sizeof(PacketHeader) + pktbodylen; + pktCnt++; + } + + pktTail = pos - buffer; + return 0; +} + +/* +return value: +<0: error +=0: contain no packet +>0: contain packet +*/ +void AgentReceiver::set_recved_info(RecvedPacket &packet) +{ + if (count_packet() < 0) + { + packet.err = -1; + return; + } + + /* not even ont packet recved, do nothing */ + if (pktCnt == 0) + return; + + char *tmpbuff; + if (NULL == (tmpbuff = (char *)malloc(buffSize))) + { + /* recved buffer willbe free by outBuff */ + log_crit("no mem malloc new recv buffer"); + packet.err = -1; + return; + } + + if (pktTail < offset) + memcpy(tmpbuff, buffer + pktTail, offset - pktTail); + + packet.buff = buffer; + packet.len = pktTail; + packet.pktCnt = pktCnt; + + buffer = tmpbuff; + offset -= pktTail; + + return; +} + +/* +output: +=0: enlarge ok +<0: enlarge error +*/ +int AgentReceiver::enlarge_buffer() +{ + buffer = (char *)REALLOC(buffer, buffSize * 2); + if (buffer == NULL) + return -1; + buffSize *= 2; + return 0; +} + +bool AgentReceiver::is_need_enlarge_buffer() +{ + return offset == buffSize; +} diff --git a/src/search_local/index_storage/common/proxy_receiver.h b/src/search_local/index_storage/common/proxy_receiver.h new file mode 100644 index 0000000..a280500 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_receiver.h @@ -0,0 +1,63 @@ +/* + * ===================================================================================== + * + * Filename: proxy_receiver.h + * + * Description: agent receiver. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_RECEIVER_H__ +#define __AGENT_RECEIVER_H__ + +#include + +#include "value.h" +#include "protocol.h" + +#define AGENT_INIT_RECV_BUFF_SIZE 4096 + +typedef struct +{ + char *buff; + int len; + int pktCnt; + int err; +} RecvedPacket; + +class AgentReceiver +{ +public: + AgentReceiver(int f); + virtual ~AgentReceiver(); + + int Init(); + RecvedPacket Recv(); + +private: + int fd; + char *buffer; + uint32_t offset; + uint32_t buffSize; + uint32_t pktTail; + int pktCnt; + + int enlarge_buffer(); + bool is_need_enlarge_buffer(); + int recv_once(); + int real_recv(); + int recv_again(); + int decode_header(PacketHeader *header); + void set_recved_info(RecvedPacket &packet); + int count_packet(); +}; + +#endif diff --git a/src/search_local/index_storage/common/proxy_sender.cc b/src/search_local/index_storage/common/proxy_sender.cc new file mode 100644 index 0000000..3a1cf54 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_sender.cc @@ -0,0 +1,180 @@ +/* + * ===================================================================================== + * + * Filename: proxy_sender.cc + * + * Description: agent sender. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "proxy_sender.h" +#include "packet.h" +#include "log.h" + +AgentSender::AgentSender(int f) : fd(f), + vec(NULL), + totalVec(1024), + currVec(0), + packet(NULL), + totalPacket(512), + currPacket(0), + totalLen(0), + sended(0), + leftLen(0), + broken(0) +{ +} + +AgentSender::~AgentSender() +{ + /* fd will closed by ClientAgent */ + for (uint32_t j = 0; j < currPacket; j++) + { + if (packet[j]) + { + packet[j]->free_result_buff(); + delete packet[j]; + } + } + if (vec) + free(vec); + if (packet) + free(packet); +} + +int AgentSender::Init() +{ + vec = (struct iovec *)malloc(1024 * sizeof(struct iovec)); + if (NULL == vec) + { + broken = 1; + return -1; + } + + if (vec) + { + packet = (Packet **)malloc(512 * sizeof(Packet *)); + if (NULL == packet) + { + broken = 1; + return -1; + } + } + + return 0; +} + +int AgentSender::add_packet(Packet *pkt) +{ + if (currVec == totalVec) + { + vec = (struct iovec *)realloc(vec, totalVec * 2 * sizeof(struct iovec)); + if (NULL == vec) + { + broken = 1; + return -ENOMEM; + } + totalVec *= 2; + } + + if (currPacket == totalPacket) + { + packet = (Packet **)realloc(packet, totalPacket * 2 * sizeof(Packet *)); + if (NULL == packet) + { + broken = 1; + return -ENOMEM; + } + totalPacket *= 2; + } + + for (int i = 0; i < pkt->vec_count(); i++) + { + vec[currVec++] = pkt->IOVec()[i]; + } + packet[currPacket++] = pkt; + leftLen += pkt->Bytes(); + + return 0; +} + +int AgentSender::Send() +{ + if (0 == leftLen) + return 0; + + int sd; + struct msghdr msgh; + uint32_t cursor = 0; + struct iovec *v = vec; + uint32_t pcursor = 0; + Packet **p = packet; + + msgh.msg_name = NULL; + msgh.msg_namelen = 0; + msgh.msg_iov = vec; + msgh.msg_iovlen = currVec; + msgh.msg_control = NULL; + msgh.msg_controllen = 0; + msgh.msg_flags = 0; + + sd = sendmsg(fd, &msgh, MSG_DONTWAIT | MSG_NOSIGNAL); + if (sd < 0) + { + if (EINTR == errno || EAGAIN == errno || EINPROGRESS == errno) + return 0; + log_error("agent sender send error. errno: %d, left len: %d, currVec: %d, IOV_MAX: %d", errno, leftLen, currVec, IOV_MAX); + broken = 1; + return -1; + } + + totalLen = leftLen; + sended = sd; + leftLen -= sended; + + if (0 == sd) + return 0; + + while (cursor < currVec && (uint32_t)sd >= v->iov_len) + { + sd -= v->iov_len; + cursor++; + v++; + (*p)->send_done_one_vec(); + if ((*p)->is_send_done()) + { + (*p)->free_result_buff(); + delete *p; + pcursor++; + p++; + } + } + + if (sd > 0) + { + v->iov_base = (char *)v->iov_base + sd; + v->iov_len -= sd; + } + + memmove((void *)vec, (void *)(vec + cursor), (currVec - cursor) * sizeof(struct iovec)); + currVec -= cursor; + + memmove((void *)packet, (void *)(packet + pcursor), (currPacket - pcursor) * sizeof(Packet *)); + currPacket -= pcursor; + + return 0; +} diff --git a/src/search_local/index_storage/common/proxy_sender.h b/src/search_local/index_storage/common/proxy_sender.h new file mode 100644 index 0000000..f534117 --- /dev/null +++ b/src/search_local/index_storage/common/proxy_sender.h @@ -0,0 +1,57 @@ +/* + * ===================================================================================== + * + * Filename: proxy_sender.h + * + * Description: agent sender. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __AGENT_SENDER_H__ +#define __AGENT_SENDER_H__ + +#include + +#define SENDER_MAX_VEC 1024 + +class Packet; +class AgentSender +{ +private: + int fd; + struct iovec *vec; + uint32_t totalVec; + uint32_t currVec; + Packet **packet; + uint32_t totalPacket; + uint32_t currPacket; + + uint32_t totalLen; + uint32_t sended; + uint32_t leftLen; + + int broken; + +public: + AgentSender(int f); + virtual ~AgentSender(); + + int Init(); + int is_broken() { return broken; } + int add_packet(Packet *pkt); + int Send(); + inline uint32_t total_len() { return totalLen; } + inline uint32_t Sended() { return sended; } + inline uint32_t left_len() { return leftLen; } + inline uint32_t vec_count() { return currVec; } +}; + +#endif diff --git a/src/search_local/index_storage/common/receiver.h b/src/search_local/index_storage/common/receiver.h new file mode 100644 index 0000000..a262170 --- /dev/null +++ b/src/search_local/index_storage/common/receiver.h @@ -0,0 +1,84 @@ +/* + * ===================================================================================== + * + * Filename: receiver.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __RECVEIVER_H__ +#define __RECVEIVER_H__ + +#include +#include "protocol.h" + +class SimpleReceiver +{ +private: + struct PacketHeader _header; + char *_packet; + int _netfd; + int _count; + int _offset; + +public: + SimpleReceiver(int fd = -1) : _packet(NULL), _netfd(fd), _count(0), _offset(0){}; + ~SimpleReceiver(void){}; + + void attach(int fd) { _netfd = fd; } + void erase(void) + { + _packet = NULL; + _count = 0; + _offset = 0; + } + int remain(void) const { return _count - _offset; } + int discard(void) + { + char buf[1024]; + int len; + const int bsz = sizeof(buf); + while ((len = remain()) > 0) + { + int rv = read(_netfd, buf, (len > bsz ? bsz : len)); + if (rv <= 0) + return rv; + _offset += rv; + } + return 1; + } + int fill(void) + { + int rv = read(_netfd, _packet + _offset, remain()); + if (rv > 0) + _offset += rv; + return rv; + } + void init(void) + { + _packet = (char *)&_header; + _count = sizeof(_header); + _offset = 0; + } + void set(char *p, int sz) + { + _packet = p; + _count = sz; + _offset = 0; + } + char *c_str(void) { return _packet; } + const char *c_str(void) const { return _packet; } + PacketHeader &header(void) { return _header; } + const PacketHeader &header(void) const { return _header; } +}; + +#endif diff --git a/src/search_local/index_storage/common/relative_hour_calculator.h b/src/search_local/index_storage/common/relative_hour_calculator.h new file mode 100644 index 0000000..a102c00 --- /dev/null +++ b/src/search_local/index_storage/common/relative_hour_calculator.h @@ -0,0 +1,75 @@ +/* + * ===================================================================================== + * + * Filename: relative_hour_calculator.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _RELATIVE_HOUR_CALCULATOR_H_ +#define _RELATIVE_HOUR_CALCULATOR_H_ + +#include +#include "singleton.h" +#include + +class RelativeHourCalculator +{ +public: + RelativeHourCalculator() + { + } + ~RelativeHourCalculator() + { + } + /*由于mktimehelper原因，需要减去八个小时*/ + void SetBaseHour(uint64_t ddwBaseYear) + { + uint64_t ddwRelativeTime = mktime_helper(ddwBaseYear, 1, 1, 0, 0, 0); + m_BaseHour = (ddwRelativeTime / 3600 - 8); + } + uint64_t get_relative_hour() + { + uint64_t ddwCurHour = time(NULL) / 3600; + + return (ddwCurHour - m_BaseHour); + } + uint64_t get_base_hour() + { + return m_BaseHour; + } + + uint64_t mktime_helper(uint64_t year, uint64_t mon, uint64_t day, uint64_t hour, uint64_t min, uint64_t sec) + { + if (0 >= (int)(mon -= 2)) + { /**/ /* 1..12 -> 11,12,1..10 */ + mon += 12; /**/ /* Puts Feb last since it has leap day */ + year -= 1; + } + + return (((( + (unsigned long)(year / 4 - year / 100 + year / 400 + 367 * mon / 12 + day) + + year * 365 - 719499) * + 24 + + hour /**/ /* now have hours */ + ) * 60 + + min /**/ /* now have minutes */ + ) * 60 + + sec); + /**/ /* finally seconds */ + } + +private: + uint64_t m_BaseHour; /*本业务对应的ModuleId*/ +}; +#define RELATIVE_HOUR_CALCULATOR Singleton::Instance() +#endif diff --git a/src/search_local/index_storage/common/request_base.h b/src/search_local/index_storage/common/request_base.h new file mode 100644 index 0000000..ff5cdd6 --- /dev/null +++ b/src/search_local/index_storage/common/request_base.h @@ -0,0 +1,189 @@ +/* + * ===================================================================================== + * + * Filename: request_base.h + * + * Description: request task class base. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_REQUEST_TEMP_H__ +#define __H_DTC_REQUEST_TEMP_H__ + +#include +#include + +#include "lqueue.h" +#include "log.h" +#include "timer_list.h" + +template +class TaskPipe; + +class PollThread; + +template +class TaskOneWayDispatcher +{ +public: + PollThread *owner; + +public: + inline TaskOneWayDispatcher(PollThread *p = NULL) : owner(p) {} + virtual inline ~TaskOneWayDispatcher() {} + virtual void task_notify(T) = 0; + + inline PollThread *owner_thread(void) { return owner; } + inline void attach_thread(PollThread *thread) + { + if (owner == NULL) + owner = thread; + } +}; + +template +class TaskDispatcher : public TaskOneWayDispatcher +{ +public: + inline TaskDispatcher(PollThread *p = NULL) : TaskOneWayDispatcher(p) {} + virtual inline ~TaskDispatcher() {} +}; + +template +class ReplyDispatcher +{ +public: + inline ReplyDispatcher() {} + virtual inline ~ReplyDispatcher() {} + virtual void reply_notify(T *) = 0; +}; + +template +class RequestOutput : private TimerObject +{ +private: + PollThread *owner; + TaskDispatcher *proc; + LinkQueue queue; + typename LinkQueue::allocator allocator; + +protected: + int queueDisabled; + +private: + inline virtual void timer_notify(void) + { + T *p; + while ((p = queue.Pop()) != NULL) + task_notify(p); + } + +public: + inline RequestOutput(PollThread *o) : owner(o), + proc(NULL), + queue(&allocator), + queueDisabled(0) + //reply(NULL) + { + } + inline ~RequestOutput() {} + inline PollThread *owner_thread(void) { return owner; } + inline void bind_dispatcher(TaskDispatcher *out) + { + if (owner == out->owner_thread()) + { + proc = out; + } + else + { + TaskPipe *tp = new TaskPipe(); + tp->bind_dispatcher(this, out); + } + } + inline void task_notify(T *p) + { + proc->task_notify(p); + } + inline void indirect_notify(T *p) + { + if (queueDisabled) + task_notify(p); + else + { + queue.Push(p); + attach_ready_timer(owner); + } + } + inline TaskDispatcher *get_dispatcher(void) { return proc; } + inline void disable_queue(void) { queueDisabled = 1; } +}; + +template +class TaskReplyList +{ +public: + ReplyDispatcher *proc[maxcount]; + int count; + +public: + inline TaskReplyList(void) { count = 0; }; + inline virtual ~TaskReplyList(void){}; + inline void Clean() + { + memset(proc, 0, sizeof(ReplyDispatcher *) * maxcount); + count = 0; + } + inline void copy_reply_path(TaskReplyList *org) + { + memcpy(proc, org->proc, sizeof(ReplyDispatcher *) * maxcount); + count = org->count; + } + inline int Push(ReplyDispatcher *p) + { + if (count >= maxcount) + return -1; + proc[count++] = p; + return 0; + } + inline ReplyDispatcher *Pop(void) + { + return count == 0 ? NULL : proc[--count]; + } + inline void Dump(void) + { + for (int i = 0; i < count; i++) + log_debug("replyproc%d: %p", i, proc[i]); + } + inline void push_reply_dispatcher(ReplyDispatcher *proc) + { + if (proc == NULL) + static_cast(this)->Panic("push_reply_dispatcher: dispatcher is NULL, check your code"); + else if (Push(proc) != 0) + static_cast(this)->Panic("push_reply_dispatcher: push queue failed, possible memory exhausted"); + } + inline void reply_notify(void) + { + ReplyDispatcher *proc = Pop(); + if (proc == NULL) + static_cast(this)->Panic("reply_notify: no more dispatcher, possible double reply"); + else + proc->reply_notify(static_cast(this)); + } + void Panic(const char *msg); +}; + +template +void TaskReplyList::Panic(const char *msg) +{ + log_crit("Internal Error Encountered: %s", msg); +} + +#endif diff --git a/src/search_local/index_storage/common/request_base_all.h b/src/search_local/index_storage/common/request_base_all.h new file mode 100644 index 0000000..edb34b6 --- /dev/null +++ b/src/search_local/index_storage/common/request_base_all.h @@ -0,0 +1,7 @@ +#ifndef REQUEST_BASE_ALL_H +#define REQUEST_BASE_ALL_H + +#include "request_base.h" +#include "pipetask.h" + +#endif diff --git a/src/search_local/index_storage/common/request_threading.h b/src/search_local/index_storage/common/request_threading.h new file mode 100644 index 0000000..d0e808f --- /dev/null +++ b/src/search_local/index_storage/common/request_threading.h @@ -0,0 +1,146 @@ +/* + * ===================================================================================== + * + * Filename: request_threading.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_REQUEST_THREAD_H__ +#define __H_DTC_REQUEST_THREAD_H__ + +#include "mtpqueue.h" +#include "wait_queue.h" +#include "request_base_all.h" + +template +class ThreadingOutputDispatcher +{ +private: // internal implementation + class InternalTaskDispatcher : public ThreadingPipeQueue + { + public: + TaskDispatcher *proc; + + public: + InternalTaskDispatcher() : proc(0) {} + virtual ~InternalTaskDispatcher() {} + void task_notify(T *p) + { + proc->task_notify(p); + }; + }; + + class InternalReplyDispatcher : public ReplyDispatcher, public threading_wait_queue + { + public: + InternalReplyDispatcher *freenext; + InternalReplyDispatcher *allnext; + + public: + InternalReplyDispatcher() : freenext(0), allnext(0) {} + virtual ~InternalReplyDispatcher() {} + virtual void reply_notify(T *p) + { + Push(p); + }; + }; + +private: + InternalTaskDispatcher incQueue; + pthread_mutex_t lock; + // lock management, protect freeList and allList + inline void Lock(void) { pthread_mutex_lock(&lock); } + inline void Unlock(void) { pthread_mutex_unlock(&lock); } + InternalReplyDispatcher *volatile freeList; + InternalReplyDispatcher *volatile allList; + volatile int stopping; + +public: + ThreadingOutputDispatcher() : freeList(0), allList(0), stopping(0) + { + pthread_mutex_init(&lock, NULL); + } + ~ThreadingOutputDispatcher() + { + InternalReplyDispatcher *q; + while (allList) + { + q = allList; + allList = q->allnext; + delete q; + } + pthread_mutex_destroy(&lock); + } + + void Stop(void) + { + stopping = 1; + InternalReplyDispatcher *p; + for (p = allList; p; p = p->allnext) + p->Stop(NULL); + } + + int Stopping(void) { return stopping; } + + int execute(T *p) + { + InternalReplyDispatcher *q; + + // aborted without side-effect + if (Stopping()) + return -1; + + /* freelist被别的线程在lock锁住的时候被别的线程置成了NULL */ + Lock(); + if (freeList) + { + q = freeList; + freeList = q->freenext; + q->Clear(); + q->freenext = NULL; + } + else + { + q = new InternalReplyDispatcher(); + q->allnext = allList; + allList = q; + } + Unlock(); + + p->push_reply_dispatcher(q); + incQueue.Push(p); + // has side effect + if (q->Pop() == NULL) + { + // q leaked by purpose + // because an pending reply didn't Popped + return -2; + } + + Lock(); + q->freenext = freeList; + freeList = q; + Unlock(); + return 0; + } + + int bind_dispatcher(TaskDispatcher *to) + { + log_debug("Create ThreadingOutputDispatcher to thread %s", + to->owner_thread()->Name()); + incQueue.proc = to; + return incQueue.attach_poller(to->owner_thread()); + } +}; + +#endif diff --git a/src/search_local/index_storage/common/result.h b/src/search_local/index_storage/common/result.h new file mode 100644 index 0000000..48a9bea --- /dev/null +++ b/src/search_local/index_storage/common/result.h @@ -0,0 +1,220 @@ +/* + * ===================================================================================== + * + * Filename: result.h + * + * Description: result write/ result packet. + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_RESUL_H__ +#define __CH_RESUL_H__ +#include "field.h" + +class ResultSet : public DTCFieldSet +{ +private: + int err; + DTCBinary init; + DTCBinary curr; + int numRows; + int rowno; + RowValue row; + +public: + /*fieldset at largest size*/ + ResultSet(const uint8_t *idtab, int num, int total, DTCTableDefinition *t) : DTCFieldSet(idtab, num, total), + err(0), + init(DTCBinary::Make(NULL, 0)), + curr(DTCBinary::Make(NULL, 0)), + numRows(0), + rowno(0), + row(t) + { + } + + ResultSet(const DTCFieldSet &f, DTCTableDefinition *t) : DTCFieldSet(f), + err(0), + init(DTCBinary::Make(NULL, 0)), + curr(DTCBinary::Make(NULL, 0)), + numRows(0), + rowno(0), + row(t) + { + } + + ~ResultSet(void) + { + } + + inline void Clean() + { + DTCFieldSet::Clean(); + err = 0; + init = DTCBinary::Make(NULL, 0); + curr = DTCBinary::Make(NULL, 0); + numRows = 0; + rowno = 0; + row.Clean(); + } + + inline int field_set_max_fields() + { + return DTCFieldSet::max_fields(); + } + + inline void realloc_field_set(int total) + { + DTCFieldSet::Realloc(total); + } + + /* clean before set*/ + inline void Set(const uint8_t *idtab, int num) + { + DTCFieldSet::Set(idtab, num); + //row.set_table_definition(t); + } + + inline void set_value_data(int nr, DTCBinary b) + { + rowno = 0; + numRows = nr; + init = b; + curr = b; + } + + int total_rows(void) const + { + return numRows; + } + + const DTCValue &operator[](int id) const { return row[id]; } + const DTCValue *field_value(int id) const + { + return row.field_value(id); + } + const DTCValue *field_value(const char *id) const + { + return row.field_value(id); + } + + int decode_row(void); + + const RowValue *row_value(void) const + { + if (err) + return NULL; + return &row; + } + + const RowValue *fetch_row(void) + { + if (decode_row()) + return NULL; + return &row; + } + + RowValue *_fetch_row(void) + { + if (decode_row()) + return NULL; + return &row; + } + + int error_num(void) const { return err; } + + int rewind(void) + { + curr = init; + err = 0; + rowno = 0; + return 0; + } + + int data_len(void) const { return init.len; } +}; + +class ResultWriter +{ + ResultWriter(const ResultWriter &); // NOT IMPLEMENTED +public: + const DTCFieldSet *fieldSet; + unsigned int limitStart, limitNext; + unsigned int totalRows; + unsigned int numRows; + + ResultWriter(const DTCFieldSet *, unsigned int, unsigned int); + virtual ~ResultWriter(void) {} + + inline virtual void Clean() + { + totalRows = 0; + numRows = 0; + } + virtual void detach_result() = 0; + inline virtual int Set(const DTCFieldSet *fs, unsigned int st, unsigned int cnt) + { + if (cnt == 0) + st = 0; + limitStart = st; + limitNext = st + cnt; + const int nf = fs == NULL ? 0 : fs->num_fields(); + fieldSet = nf ? fs : NULL; + + return 0; + } + virtual int append_row(const RowValue &) = 0; + virtual int merge_no_limit(const ResultWriter *rp) = 0; + + int set_rows(unsigned int rows); + void set_total_rows(unsigned int totalrows) { totalRows = totalrows; } + void add_total_rows(int n) { totalRows += n; } + int is_full(void) const { return limitNext > 0 && totalRows >= limitNext; } + int in_range(unsigned int total, unsigned int begin = 0) const + { + if (limitNext == 0) + return (1); + if (total <= limitStart || begin >= limitNext) + return (0); + return (1); + } +}; + +class ResultPacket : public ResultWriter +{ + ResultPacket(const ResultPacket &); // NOT IMPLEMENTED +public: + /* acctually just one buff */ + BufferChain *bc; + unsigned int rowDataBegin; + + ResultPacket(const DTCFieldSet *, unsigned int, unsigned int); + virtual ~ResultPacket(void); + + void Clean(); + inline virtual void detach_result() { bc = NULL; } + virtual int Set(const DTCFieldSet *fieldList, unsigned int st, unsigned int ct); + virtual int append_row(const RowValue &); + virtual int merge_no_limit(const ResultWriter *rp); +}; + +class ResultBuffer : public ResultWriter +{ + ResultBuffer(const ResultBuffer &); // NOT IMPLEMENTED +public: + ResultBuffer(const DTCFieldSet *, unsigned int, unsigned int); + virtual ~ResultBuffer(void); + + virtual int append_row(const RowValue &); + virtual int merge_no_limit(const ResultWriter *rp); +}; + +#endif diff --git a/src/search_local/index_storage/common/section.cc b/src/search_local/index_storage/common/section.cc new file mode 100644 index 0000000..4db52ab --- /dev/null +++ b/src/search_local/index_storage/common/section.cc @@ -0,0 +1,88 @@ +/* + * ===================================================================================== + * + * Filename: section.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include "section.h" +#include "field.h" +#include "buffer_error.h" + +#if MAX_STATIC_SECTION >= 1 && MAX_STATIC_SECTION < 18 +#error MAX_STATIC_SECTION must >= 18 +#endif +const SectionDefinition versionInfoDefinition = { + DRequest::Section::VersionInfo, + 20, + { + // version info: + DField::None, + DField::String, // 1 -- table name + DField::Binary, // 2 -- data table hash [for hb register use, unless always send 0x00000...] + DField::Unsigned, // 3 -- serial# + DField::Binary, // 4 -- real table hash + DField::String, // 5 -- client version + DField::String, // 6 -- ctlib version + DField::String, // 7 -- helper version [unuse] + DField::Unsigned, // 8 -- keepalive timeout + DField::Unsigned, // 9 -- key type + DField::Unsigned, // 10 -- key field count + DField::Unsigned, // 11 -- key value count + DField::Binary, // 12 -- key type list + DField::Binary, // 13 -- key name list + DField::Unsigned, // 14 -- hot backup ID + DField::Signed, // 15 -- hot backup master timestamp + DField::Signed, // 16 -- hot backup slave timestamp + DField::Unsigned, // 17 -- agent client id + DField::String, // 18 -- accessKey + DField::Unsigned // 19 -- reconnet flag + }, +}; + +const SectionDefinition requestInfoDefinition = { + DRequest::Section::VersionInfo, 8, { +#if MAX_STATIC_SECTION >= 1 && MAX_STATIC_SECTION < 8 +#error MAX_STATIC_SECTION must >= 8 +#endif + // request info: + DField::Binary, // 0 -- key + DField::Unsigned, // 1 -- timeout + DField::Unsigned, // Limit start + DField::Unsigned, // Limit count + DField::String, // 4 -- trace msg + DField::Unsigned, // 5 -- Cache ID -- OBSOLETED + DField::String, // 6 -- raw config string + DField::Unsigned, // 7 -- admin cmd code + }}; + +const SectionDefinition resultInfoDefinition = { + DRequest::Section::ResultInfo, 10, { +#if MAX_STATIC_SECTION >= 1 && MAX_STATIC_SECTION < 9 +#error MAX_STATIC_SECTION must >= 9 +#endif + // result info: + DField::Binary, // 0 -- key, original key or INSERT_ID + DField::Signed, // 1 -- code + DField::String, // 2 -- trace msg + DField::String, // 3 -- from + DField::Signed, // 4 -- affected rows, by delete/update + DField::Unsigned, // 5 -- total rows, not used. for SELECT SQL_CALC_FOUND_ROWS... + DField::Unsigned, // 6 -- insert_id, not used. for SELECT SQL_CALC_FOUND_ROWS... + DField::Binary, // 7 -- query server info: version, binlogid,.... + DField::Unsigned, // 8 -- server timestamp + DField::Unsigned, // 9 -- Hit Flag + }}; diff --git a/src/search_local/index_storage/common/section.h b/src/search_local/index_storage/common/section.h new file mode 100644 index 0000000..18a5366 --- /dev/null +++ b/src/search_local/index_storage/common/section.h @@ -0,0 +1,486 @@ +/* + * ===================================================================================== + * + * Filename: section.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_SECTION_H__ +#define __CH_SECTION_H__ + +#include "mem_check.h" +#include +#include +#include +#include + +#include "value.h" +#include "protocol.h" +#include "log.h" + +#define MAX_STATIC_SECTION 20 + +/* + * tag: a pair of id:value, the value type if predefined + * section: a set of id:value + */ +struct SectionDefinition +{ + uint8_t sectionId; + uint8_t maxTags; + uint8_t tagType[256]; + + uint8_t tag_type(uint8_t i) const { return tagType[i]; }; + int max_tags(void) const { return maxTags; } + int section_id(void) const { return sectionId; } +}; + +class SimpleSection +{ +private: + const SectionDefinition *definition; + uint8_t numTags; +#if MAX_STATIC_SECTION + uint8_t fieldMask[(MAX_STATIC_SECTION + 7) / 8]; +#else + uint8_t fieldMask[32]; +#endif + +public: +#if MAX_STATIC_SECTION + DTCValue tagValue[MAX_STATIC_SECTION]; +#else + DTCValue *tagValue; +#endif + + SimpleSection(const SectionDefinition &d) : definition(&d), numTags(0) + { +#if MAX_STATIC_SECTION + for (int i = 0; i < d.max_tags(); i++) + tagValue[i].Set(NULL, 0); +#else + tagValue = (DTCValue *)calloc(d.max_tags(), sizeof(DTCValue)); + if (tagValue == NULL) + throw std::bad_alloc(); +#endif + FIELD_ZERO(fieldMask); + } + + virtual ~SimpleSection() + { +#if !MAX_STATIC_SECTION + FREE_IF(tagValue); +#endif + } + SimpleSection(const SimpleSection &orig) + { +#if !MAX_STATIC_SECTION + tagValue = (DTCValue *)malloc(orig.definition->max_tags(), sizeof(DTCValue)); + if (tagValue == NULL) + throw std::bad_alloc(); +#endif + Copy(orig); + } + void Copy(const SimpleSection &orig) + { + for (int i = 0; i < orig.definition->max_tags(); i++) + tagValue[i] = orig.tagValue[i]; + memcpy(fieldMask, orig.fieldMask, sizeof(fieldMask)); + definition = orig.definition; + numTags = orig.numTags; + } + + inline virtual void Clean() + { + memset(tagValue, 0, sizeof(DTCValue) * definition->max_tags()); + FIELD_ZERO(fieldMask); + numTags = 0; + } + + int max_tags(void) const + { + return definition->max_tags(); + } + + int num_tags(void) const + { + return numTags; + } + + int section_id(void) const + { + return definition->section_id(); + } + + uint8_t tag_type(uint8_t id) const + { + return definition->tag_type(id); + } + + int tag_present(uint8_t id) const + { + return id >= MAX_STATIC_SECTION ? 0 : FIELD_ISSET(id, fieldMask); + } + + void set_tag(uint8_t id, const DTCValue &val) + { + if (tag_type(id) != DField::None) + { + tagValue[id] = val; + if (!FIELD_ISSET(id, fieldMask)) + { + numTags++; + FIELD_SET(id, fieldMask); + } + } + } + + /* no check of dumplicate tag */ + void SetTagMask(uint8_t id) + { + numTags++; + FIELD_SET(id, fieldMask); + } + + void set_tag(uint8_t id, int64_t val) + { + set_tag(id, DTCValue::Make(val)); + } + void set_tag(uint8_t id, uint64_t val) + { + set_tag(id, DTCValue::Make(val)); + } + void set_tag(uint8_t id, int32_t val) + { + if (tag_type(id) == DField::Signed) + set_tag(id, DTCValue::Make(val)); + else if (tag_type(id) == DField::Unsigned) + { + if (val < 0) + val = 0; + set_tag(id, DTCValue::Make((uint64_t)val)); + } + } + void set_tag(uint8_t id, uint32_t val) + { + //fix Unsigned to Signed + if (tag_type(id) == DField::Signed) + set_tag(id, DTCValue::Make((int64_t)val)); + else if (tag_type(id) == DField::Unsigned) + set_tag(id, DTCValue::Make(val)); + } + void set_tag(uint8_t id, double val) + { + if (tag_type(id) == DField::Float) + set_tag(id, DTCValue::Make(val)); + } + void set_tag(uint8_t id, const char *val, int len) + { + if (tag_type(id) == DField::String || tag_type(id) == DField::Binary) + set_tag(id, DTCValue::Make(val, len)); + } + void set_tag(uint8_t id, const char *val) + { + if (tag_type(id) == DField::String || tag_type(id) == DField::Binary) + set_tag(id, DTCValue::Make(val)); + } + + const DTCValue *get_tag(uint8_t id) const + { + return tag_present(id) ? &tagValue[id] : NULL; + } + + /* no check */ + DTCValue *get_this_tag(uint8_t id) + { + return &tagValue[id]; + } + + const DTCValue *Key(void) const + { + return tag_present(0) ? &tagValue[0] : NULL; + } + void set_key(const DTCValue &k) { set_tag(0, k); } +}; + +extern const SectionDefinition versionInfoDefinition; +extern const SectionDefinition requestInfoDefinition; +extern const SectionDefinition resultInfoDefinition; + +class DTCVersionInfo : public SimpleSection +{ +public: + DTCVersionInfo() : SimpleSection(versionInfoDefinition) {} + ~DTCVersionInfo() {} + DTCVersionInfo(const DTCVersionInfo &orig) : SimpleSection(orig) {} + + const DTCBinary &table_name(void) const { return tagValue[1].str; } + void set_table_name(const char *n) { set_tag(1, n); } + const DTCBinary &table_hash(void) const { return tagValue[4].str; } + void set_table_hash(const char *n) { set_tag(4, n, 16); } + const DTCBinary &data_table_hash(void) const { return tagValue[2].str; } + void set_data_table_hash(const char *n) { set_tag(2, n, 16); } + + const DTCBinary &CTLibVer(void) const { return tagValue[6].str; } + const int CTLibIntVer(void) const + { + int major = 3; //, minor=0, micro=0; + + /* 3.x系列的批量拆包之后没有version信息 */ + if (NULL == CTLibVer().ptr) + { + log_debug("multi task have no version info"); + return major; + } + + char buf = CTLibVer().ptr[7]; + if (buf >= '1' && buf <= '9') + { + major = buf - '0'; + log_debug("client major version:%d,version string:%s", major, CTLibVer().ptr); + return major; + } + else + { + log_debug("unknown client api version: %s", CTLibVer().ptr); + return major; + } + } + + int ReConnect(void) const { return tagValue[19].u64; } + int key_type(void) const { return tagValue[9].u64; } + void set_key_type(int n) { set_tag(9, n); } +#if MAX_STATIC_SECTION >= 1 && MAX_STATIC_SECTION < 10 +#error MAX_STATIC_SECTION must >= 10 +#endif + + const uint64_t serial_nr(void) const + { + return tagValue[3].u64; + } + void set_serial_nr(uint64_t u) { set_tag(3, u); } + const int keep_alive_timeout(void) const + { + return (int)tagValue[8].u64; + } + void set_keep_alive_timeout(uint32_t u) { set_tag(8, u); } + + void set_hot_backup_id(uint64_t u) { set_tag(14, u); } + uint64_t hot_backup_id() const { return tagValue[14].u64; } + void set_master_hb_timestamp(int64_t t) { set_tag(15, t); } + int64_t master_hb_timestamp(void) const { return tagValue[15].s64; } + void set_slave_hb_timestamp(int64_t t) { set_tag(16, t); } + int64_t slave_hb_timestamp(void) const { return tagValue[16].s64; } + uint64_t get_agent_client_id() const { return tagValue[17].u64; } + void set_agent_client_id(uint64_t id) { set_tag(17, id); } + + void set_access_key(const char *token) { set_tag(18, token); } + const DTCBinary &access_key(void) const { return tagValue[18].str; } +}; + +class DTCRequestInfo : public SimpleSection +{ +public: + DTCRequestInfo() : SimpleSection(requestInfoDefinition) {} + ~DTCRequestInfo() {} + DTCRequestInfo(const DTCRequestInfo &orig) : SimpleSection(orig) {} + + uint64_t get_expire_time(int version) const + { + /* server内部全部按照ms单位来处理超时 */ + if (version >= 3) + /* 3.x 系列客户端发送的超时时间单位：us */ + return tagValue[1].u64 >> 10; + else + /* 2.x 系列客户端发送的超时时间单位: ms */ + return tagValue[1].u64; + } + void set_timeout(uint32_t n) { set_tag(1, (uint64_t)n << 10); } + uint32_t limit_start(void) const { return tagValue[2].u64; } + uint32_t limit_count(void) const { return tagValue[3].u64; } + void set_limit_start(uint32_t n) { set_tag(2, (uint64_t)n); } + void set_limit_count(uint32_t n) { set_tag(3, (uint64_t)n); } + + uint32_t admin_code() const { return tagValue[7].u64; } + void set_admin_code(uint32_t code) { set_tag(7, (uint64_t)code); } +}; + +class DTCResultInfo : public SimpleSection +{ +private: + char *szErrMsg; + char *s_info; + char *t_info; + +public: + DTCResultInfo() : SimpleSection(resultInfoDefinition), szErrMsg(NULL), s_info(NULL), t_info(NULL) {} + ~DTCResultInfo() + { + FREE_CLEAR(szErrMsg); + FREE_CLEAR(s_info); + FREE_CLEAR(t_info); + } + DTCResultInfo(const DTCResultInfo &orig) : SimpleSection(orig) + { + if (orig.szErrMsg) + { + szErrMsg = STRDUP(orig.szErrMsg); + set_tag(3, szErrMsg, strlen(szErrMsg)); + } + if (orig.s_info) + { + s_info = NULL; + set_server_info((DTCServerInfo *)orig.s_info); + } + if (orig.t_info) + { + t_info = NULL; + set_time_info((DTCTimeInfo *)orig.t_info); + } + } + + inline virtual void Clean() + { + SimpleSection::Clean(); + FREE_CLEAR(szErrMsg); + FREE_CLEAR(s_info); + FREE_CLEAR(t_info); + } + void Copy(const DTCResultInfo &orig) + { + SimpleSection::Copy(orig); + FREE_CLEAR(szErrMsg); + if (orig.szErrMsg) + { + szErrMsg = STRDUP(orig.szErrMsg); + set_tag(3, szErrMsg, strlen(szErrMsg)); + } + FREE_CLEAR(s_info); + if (orig.s_info) + { + set_server_info((DTCServerInfo *)orig.s_info); + } + FREE_CLEAR(t_info); + if (orig.t_info) + { + set_time_info((DTCTimeInfo *)orig.t_info); + } + } + + void set_error(int err, const char *from, const char *msg) + { + set_tag(1, err); + if (from) + set_tag(2, (char *)from, strlen(from)); + if (msg) + set_tag(3, (char *)msg, strlen(msg)); + } + void set_error_dup(int err, const char *from, const char *msg) + { + FREE_IF(szErrMsg); + szErrMsg = STRDUP(msg); + set_tag(1, err); + if (from) + set_tag(2, (char *)from, strlen(from)); + if (msg) + set_tag(3, szErrMsg, strlen(szErrMsg)); + } + int result_code(void) const { return tagValue[1].s64; } + const char *error_from(void) const { return tagValue[2].str.ptr; } + const char *error_message(void) const { return tagValue[3].str.ptr; } + uint64_t affected_rows(void) const { return tagValue[4].s64; } + void set_affected_rows(uint64_t nr) { set_tag(4, nr); } + uint64_t total_rows(void) const { return tagValue[5].s64; } + void set_total_rows(uint64_t nr) { set_tag(5, nr); } + uint64_t insert_id(void) const { return tagValue[6].u64; } + void set_insert_id(uint64_t nr) { set_tag(6, nr); } + char *server_info(void) const + { + if (tagValue[7].str.len != sizeof(DTCServerInfo)) + return NULL; + return tagValue[7].str.ptr; + } + + void set_server_info(DTCServerInfo *si) + { + FREE_IF(s_info); + int len = sizeof(DTCServerInfo); + s_info = (char *)CALLOC(1, len); + memcpy(s_info, si, len); + set_tag(7, s_info, len); + return; + } + + char *time_info(void) const + { + if (tagValue[7].str.len != sizeof(DTCTimeInfo)) + return NULL; + return tagValue[7].str.ptr; + } + + void set_time_info(DTCTimeInfo *ti) + { + FREE_IF(t_info); + int len = sizeof(DTCTimeInfo); + t_info = (char *)CALLOC(1, len); + memcpy(t_info, ti, len); + set_tag(7, t_info, len); + return; + } + + uint32_t Timestamp(void) const { return tagValue[8].s64; } + void set_time_info(uint32_t nr) { set_tag(8, nr); } + /*set hit flag by tomchen 20140604*/ + void set_hit_flag(uint32_t nr) { set_tag(9, nr); } + uint32_t hit_flag() { return tagValue[9].s64; }; + + /*一个请求有多个key的时候，要统计这个请求不同key对应的业务命中率和计数命中率*/ + /*命中率字段中，前16位放业务命中率，后16位放技术命中率*/ + uint32_t get_tech_hit_num() + { + uint32_t uHitFlag = (uint32_t)hit_flag(); + uint32_t uTaskTechHitNum = uHitFlag & 0xFFFF; + return uTaskTechHitNum; + } + + uint32_t get_business_hit_num() + { + uint32_t uHitFlag = (uint32_t)hit_flag(); + uint32_t uTashBusinessHitNum = (uHitFlag & 0xFFFF0000) >> 16; + return uTashBusinessHitNum; + } + + void incr_tech_hit_num() + { + + uint32_t uHitFlag = (uint32_t)hit_flag(); + uint32_t uTaskTechHitNum = uHitFlag & 0xFFFF; + uTaskTechHitNum++; + uHitFlag = uHitFlag & 0xFFFF0000; + uHitFlag = uHitFlag | uTaskTechHitNum; + set_hit_flag(uHitFlag); + } + + void incr_bussiness_hit_num() + { + uint32_t uHitFlag = (uint32_t)hit_flag(); + uint32_t uTashBusinessHitNum = (uHitFlag & 0xFFFF0000) >> 16; + uTashBusinessHitNum++; + uTashBusinessHitNum = uTashBusinessHitNum << 16; + uHitFlag = uHitFlag & 0xFFFF; + uHitFlag = uHitFlag | uTashBusinessHitNum; + set_hit_flag(uHitFlag); + } +}; + +#endif diff --git a/src/search_local/index_storage/common/shmem.cc b/src/search_local/index_storage/common/shmem.cc new file mode 100644 index 0000000..1cd20ea --- /dev/null +++ b/src/search_local/index_storage/common/shmem.cc @@ -0,0 +1,137 @@ +/* + * ===================================================================================== + * + * Filename: sheme.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "shmem.h" +#include "system_lock.h" + +SharedMemory::SharedMemory() : m_key(0), m_id(0), m_size(0), m_ptr(NULL), lockfd(-1) +{ +} + +SharedMemory::~SharedMemory() +{ + Detach(); + Unlock(); +} + +unsigned long SharedMemory::Open(int key) +{ + if (m_ptr) + Detach(); + + if (key == 0) + return 0; + + m_key = key; + if ((m_id = shmget(m_key, 0, 0)) == -1) + return 0; + + struct shmid_ds ds; + + if (shmctl(m_id, IPC_STAT, &ds) < 0) + return 0; + + return m_size = ds.shm_segsz; +} + +unsigned long SharedMemory::Create(int key, unsigned long size) +{ + if (m_ptr) + Detach(); + + m_key = key; + + if (m_key == 0) + { + m_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (m_ptr != MAP_FAILED) + m_size = size; + else + m_ptr = NULL; + } + else + { + if ((m_id = shmget(m_key, size, IPC_CREAT | IPC_EXCL | IPC_PERM)) == -1) + return 0; + + struct shmid_ds ds; + + if (shmctl(m_id, IPC_STAT, &ds) < 0) + return 0; + + m_size = ds.shm_segsz; + } + return m_size; +} + +void *SharedMemory::Attach(int ro) +{ + if (m_ptr) + return m_ptr; + + m_ptr = shmat(m_id, NULL, ro ? SHM_RDONLY : 0); + if (m_ptr == MAP_FAILED) + m_ptr = NULL; + return m_ptr; +} + +void SharedMemory::Detach(void) +{ + if (m_ptr) + { + if (m_key == 0) + munmap(m_ptr, m_size); + else + shmdt(m_ptr); + } + m_ptr = NULL; +} + +int SharedMemory::Lock(void) +{ + if (lockfd > 0 || m_key == 0) + return 0; + lockfd = unix_socket_lock("tlock-shm-%u", m_key); + if (lockfd < 0) + return -1; + return 0; +} + +void SharedMemory::Unlock(void) +{ + if (lockfd > 0) + { + close(lockfd); + lockfd = -1; + } +} + +int SharedMemory::Delete(void) +{ + Detach(); + + if (shmctl(m_id, IPC_RMID, NULL) < 0) + return -1; + return 0; +} diff --git a/src/search_local/index_storage/common/shmem.h b/src/search_local/index_storage/common/shmem.h new file mode 100644 index 0000000..6c36a09 --- /dev/null +++ b/src/search_local/index_storage/common/shmem.h @@ -0,0 +1,44 @@ +/* + * ===================================================================================== + * + * Filename: sheme.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#define IPC_PERM 0644 + +class SharedMemory +{ +private: + int m_key; + int m_id; + unsigned long m_size; + void *m_ptr; + int lockfd; + +public: + SharedMemory(); + ~SharedMemory(); + + unsigned long Open(int key); + unsigned long Create(int key, unsigned long size); + unsigned long Size(void) const { return m_size; } + void *Ptr(void) const { return m_ptr; } + void *Attach(int ro = 0); + void Detach(void); + int Lock(void); + void Unlock(void); + + /* 删除共享内存 */ + int Delete(void); +}; diff --git a/src/search_local/index_storage/common/singleton.h b/src/search_local/index_storage/common/singleton.h new file mode 100644 index 0000000..40bb7eb --- /dev/null +++ b/src/search_local/index_storage/common/singleton.h @@ -0,0 +1,108 @@ +/* + * ===================================================================================== + * + * Filename: singleton.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __SINGLETON_H__ +#define __SINGLETON_H__ +#include "lock.h" +#include "namespace.h" + +DTC_BEGIN_NAMESPACE + +template +struct CreateUsingNew +{ + static T *Create(void) + { + return new T; + } + + static void Destroy(T *p) + { + delete p; + } +}; + +template class CreationPolicy = CreateUsingNew> +class Singleton +{ +public: + static T *Instance(void); + static void Destroy(void); + +private: + Singleton(void); + Singleton(const Singleton &); + Singleton &operator=(const Singleton &); + +private: + static T *_instance; + static Mutex _mutex; +}; + +DTC_END_NAMESPACE + +DTC_USING_NAMESPACE + +//implement +template class CreationPolicy> +Mutex Singleton::_mutex; + +template class CreationPolicy> +T *Singleton::_instance = 0; + +template class CreationPolicy> +T *Singleton::Instance(void) +{ + if (0 == _instance) + { + ScopedLock guard(_mutex); + + if (0 == _instance) + { + _instance = CreationPolicy::Create(); + } + } + + return _instance; +} + +/* BUGFIX by ada*/ +#if 0 +template class CreationPolicy> +void Singleton::Destroy (void) +{ + return CreationPolicy::Destroy (_instance); +} +#endif + +template class CreationPolicy> +void Singleton::Destroy(void) +{ + if (0 != _instance) + { + ScopedLock guard(_mutex); + if (0 != _instance) + { + CreationPolicy::Destroy(_instance); + _instance = 0; + } + } + + return; +} + +#endif //__SINGLETON_H__ diff --git a/src/search_local/index_storage/common/stop_watch.h b/src/search_local/index_storage/common/stop_watch.h new file mode 100644 index 0000000..7d0eeee --- /dev/null +++ b/src/search_local/index_storage/common/stop_watch.h @@ -0,0 +1,91 @@ +/* + * ===================================================================================== + * + * Filename: stop_watch.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __STOPWATCH_TIMER_H__ +#define __STOPWATCH_TIMER_H__ + +#include + +class stopwatch_unit_t +{ +public: + class sec + { + public: + static inline unsigned int gettime(void) + { + return time(NULL); + }; + }; + + class msec + { + public: + static inline unsigned int gettime(void) + { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000 + tv.tv_usec / 1000; + }; + }; + + class usec + { + public: + static inline unsigned int gettime(void) + { + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; + }; + }; +}; + +template +class stopwatch_t +{ +private: + unsigned int _value; + +public: + inline stopwatch_t(void) { _value = 0; } + inline stopwatch_t(bool started) { _value = started ? T::gettime() : 0; } + inline stopwatch_t(unsigned int v) { _value = v; } + inline stopwatch_t(const stopwatch_t &v) { _value = v._value; } + inline ~stopwatch_t(void) {} + + /* start timer now */ + inline void start(void) { _value = T::gettime(); } + /* start from timestamp:v */ + inline void init(unsigned int v) { _value = v; } + inline stopwatch_t &operator=(unsigned int v) + { + _value = v; + return *this; + } + /* stop watch */ + inline void stop(void) { _value = T::gettime() - _value; } + /* get counter _value */ + inline unsigned int get(void) { return _value; } + inline operator unsigned int(void) const { return _value; } + inline unsigned int live(void) const { return T::gettime() - _value; } +}; + +typedef stopwatch_t stopwatch_sec_t; +typedef stopwatch_t stopwatch_msec_t; +typedef stopwatch_t stopwatch_usec_t; +#endif diff --git a/src/search_local/index_storage/common/system_lock.c b/src/search_local/index_storage/common/system_lock.c new file mode 100644 index 0000000..6f6b4af --- /dev/null +++ b/src/search_local/index_storage/common/system_lock.c @@ -0,0 +1,54 @@ +/* + * ===================================================================================== + * + * Filename: system_lock.c + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#include "system_lock.h" + +int unix_socket_lock(const char *format, ...) +{ + int lockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (lockfd >= 0) + { + va_list ap; + struct sockaddr_un addr; + int len; + + addr.sun_family = AF_UNIX; + + va_start(ap, format); + vsnprintf(addr.sun_path + 1, sizeof(addr.sun_path) - 1, format, ap); + va_end(ap); + + addr.sun_path[0] = '@'; + len = SUN_LEN(&addr); + addr.sun_path[0] = '\0'; + if (bind(lockfd, (struct sockaddr *)&addr, len) == 0) + fcntl(lockfd, F_SETFD, FD_CLOEXEC); + else + { + close(lockfd); + lockfd = -1; + } + } + return lockfd; +} diff --git a/src/search_local/index_storage/common/system_lock.h b/src/search_local/index_storage/common/system_lock.h new file mode 100644 index 0000000..cd73a32 --- /dev/null +++ b/src/search_local/index_storage/common/system_lock.h @@ -0,0 +1,27 @@ +/* + * ===================================================================================== + * + * Filename: system_lock.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __SYSTEM_LOCK_H +#define __SYSTEM_LOCK_H + +#include + +__BEGIN_DECLS +int unix_socket_lock(const char *fmt, ...); +__END_DECLS + +#endif diff --git a/src/search_local/index_storage/common/table_def.cc b/src/search_local/index_storage/common/table_def.cc new file mode 100644 index 0000000..8b64737 --- /dev/null +++ b/src/search_local/index_storage/common/table_def.cc @@ -0,0 +1,517 @@ +/* + * ===================================================================================== + * + * Filename: table_def.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +// skip scope test here, we need full decleration +#undef CLIENTAPI +#include "table_def.h" +#include "md5.h" +#include "decode.h" +#include "buffer_error.h" +#include "mem_check.h" + +#define HASHSTOP 0xFFFF + +#if MAX_STATIC_SECTION >= 1 && MAX_STATIC_SECTION < 4 +#error MAX_STATIC_SECTION must >= 4 +#endif +const SectionDefinition tableAttributeDefinition = { + DRequest::Section::table_definition, 4, { + DField::None, + DField::Unsigned, // flags + DField::Unsigned, // masks + DField::Unsigned, // key field count + }}; + +DTCTableDefinition::DTCTableDefinition(int m) : maxFields(m), + usedFields(0), + numFields(0), + keyFields(0) +{ + // memset 0xFF, == all value 0xFFFF (HASHSTOP) + memset(nameHash, 0xFF, sizeof(nameHash)); + tableName.Set(NULL, 0); + // hash didn't, it will be inited by build_info_cache(server) or Unpack(client) + // max_key_size always calculated by set_key_fields(server), unused in client side + + if (maxFields > 0) + { + // server side code + // used in early initialize, assume always has enough memory + fieldList = (FieldDefinition *)calloc(maxFields, sizeof(FieldDefinition)); + defaultValue = (DTCValue *)calloc(maxFields, sizeof(DTCValue)); + uniqFields = (uint8_t *)calloc(maxFields, sizeof(uint8_t)); + packedTableDefinition.Set(NULL, 0); + packedFullFieldSet.Set(NULL, 0); + packedFieldSet.Set(NULL, 0); + uniqFieldCnt = 0; + keysAsUniqField = 0; + hasAutoInc = -1; + hasLastacc = -1; + hasLastmod = -1; + hasLastcmod = -1; + hasCompress = -1; + hasExpireTime = -1; + keyFormat = 0; + rowSize = 0; + hasDiscard = 0; + indexFields = 0; // by TREE_DATA + } + else + { + // client side code + fieldList = NULL; + // default Value used as flag determine client/server side object + defaultValue = NULL; + // don't init following members: + // packedTableDefinition + // packedFullFieldSet + // packedFieldSet + // uniqFields + // uniqFieldCnt + // keysAsUniqField + // hasAutoInc + // keyFormat + // rowSize + // indexFields + // because client side don't use it, and save a lot of CPU cycle + } +} + +DTCTableDefinition::~DTCTableDefinition(void) +{ + FREE_IF(tableName.ptr); + if (fieldList != NULL) + { + for (int i = 0; i <= numFields; i++) + { + FREE_IF(fieldList[i].fieldName); + } + FREE_IF(fieldList); + } + if (defaultValue != NULL) + { + // client side didn't init this members + FREE_IF(packedTableDefinition.ptr); + FREE_IF(packedFullFieldSet.ptr); + FREE_IF(packedFieldSet.ptr); + FREE_IF(uniqFields); + FREE(defaultValue); + } +} + +// used for bitmap svr +int DTCTableDefinition::add_field(int id, const char *name, uint8_t type, + int size, int bsize, int boffset) +{ + // permit duplicate add to overwrite the old one + if (id < 0 || id > MAXFIELDS_PER_TABLE || id >= maxFields) + return -EC_BAD_FIELD_ID; + if (size <= 0) + return -EC_BAD_FIELD_SIZE; + switch (type) + { + case DField::Signed: + case DField::Unsigned: + if (size > 8) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(0); + break; + case DField::Float: + if (size != 8 && size != 4) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(0.0); + break; + case DField::String: + case DField::Binary: + if (size > MAXPACKETSIZE) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(NULL, 0); + break; + default: + return -EC_BAD_FIELD_TYPE; + } + int n = strlen(name); + if (n == 0 || n > 255) + return -EC_BAD_FIELD_NAME; + + char *lname = STRDUP(name); + if (lname == NULL) + throw std::bad_alloc(); + + unsigned h = 0; + for (int i = 0; i < n; i++) + h = h * 11 + (name[i] << 4) + (name[i] >> 4); + h &= 0xFF; + + fieldList[id].fieldName = lname; + fieldList[id].fieldType = type; + fieldList[id].fieldSize = size; //Bytes + fieldList[id].bsize = bsize; //bits + fieldList[id].boffset = boffset; + fieldList[id].nameLen = n; + fieldList[id].flags = id == 0 ? FieldDefinition::FF_READONLY : 0; + fieldList[id].next = nameHash[h]; + nameHash[h] = id; + if (id > numFields) + numFields = id; + if (id) + usedFields++; + return 0; +} + +int DTCTableDefinition::add_field(int id, const char *name, uint8_t type, int size) +{ + if (id < 0 || id > MAXFIELDS_PER_TABLE || id >= maxFields || fieldList[id].fieldName) + return -EC_BAD_FIELD_ID; + if (size <= 0) + return -EC_BAD_FIELD_SIZE; + switch (type) + { + case DField::Signed: + case DField::Unsigned: + if (size > 8) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(0); + break; + case DField::Float: + if (size != 8 && size != 4) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(0.0); + break; + case DField::String: + case DField::Binary: + if (size > MAXPACKETSIZE) + return -EC_BAD_FIELD_SIZE; + if (defaultValue != NULL) + defaultValue[id].Set(NULL, 0); + break; + default: + return -EC_BAD_FIELD_TYPE; + } + int n = strlen(name); + if (n == 0 || n > 255) + return -EC_BAD_FIELD_NAME; + + char *lname = STRDUP(name); + if (lname == NULL) + throw std::bad_alloc(); + + unsigned h = 0; + for (int i = 0; i < n; i++) + h = h * 11 + (name[i] << 4) + (name[i] >> 4); + h &= 0xFF; + + fieldList[id].fieldName = lname; + fieldList[id].fieldType = type; + fieldList[id].fieldSize = size; + fieldList[id].nameLen = n; + fieldList[id].flags = id == 0 ? FieldDefinition::FF_READONLY : 0; + fieldList[id].next = nameHash[h]; + nameHash[h] = id; + if (id > numFields) + numFields = id; + if (id) + usedFields++; + return 0; +} + +void DTCTableDefinition::set_default_value(int id, const DTCValue *dval) +{ + if (id <= 0 || id > 255 || id >= maxFields || defaultValue == NULL) + return; + defaultValue[id] = *dval; + /* string value cross reference to dbconfig */ +} + +void DTCTableDefinition::get_default_row(DTCValue *row) const +{ + if (defaultValue) + memcpy(row + 1, defaultValue + 1, sizeof(DTCValue) * numFields); + else + memset(row + 1, 0, sizeof(DTCValue) * numFields); +} + +int DTCTableDefinition::field_id(const char *name, int len) const +{ + if (len == 0) + len = strlen(name); + unsigned h = 0; + for (int i = 0; i < len; i++) + h = h * 11 + (name[i] << 4) + (name[i] >> 4); + h &= 0xFF; + + for (uint16_t id = nameHash[h]; id != HASHSTOP; id = fieldList[id].next) + { + if (fieldList[id].nameLen != len) + continue; + if (memcmp(fieldList[id].fieldName, name, len) == 0) + return id; + } + return -1; +} + +int DTCTableDefinition::set_table_name(const char *name) +{ + char *p = STRDUP(name); + if (p == NULL) + return -ENOMEM; + tableName.Set(p); + return 0; +} + +int DTCTableDefinition::set_key_fields(int n) +{ + if (n < 0) + n = 1; + if (n <= 0) + return 0; + if (n > 32) + return -1; + if (n > numFields + 1) + n = numFields + 1; + keyFields = n; + attr.set_key_field_count(n); + + int j; + keysAsUniqField = 2; /* SUBSET */ + int i, maxSize = 0, nvar = 0; + for (i = 0; i < keyFields; i++) + { + mark_as_read_only(i); + fieldList[i].offset = maxSize; + switch (field_type(i)) + { + case DField::Signed: + case DField::Unsigned: //整数 + case DField::Float: //浮点数 + maxSize += field_size(i); + break; + case DField::String: //以null结尾的字符串 + case DField::Binary: //二进制数据 + default: + nvar++; + maxSize = 1 + field_size(i); + break; + } + if (keysAsUniqField) + { + for (j = 0; j < uniqFieldCnt; j++) + if (uniqFields[j] == i) + break; + if (j >= uniqFieldCnt) + keysAsUniqField = 0; /* NOT */ + } + } + if (uniqFieldCnt == n && keysAsUniqField > 0) + keysAsUniqField = 1; /* EXACT */ + if (nvar == 0) + { + if (maxKeySize >= 256) + return -1; + keyFormat = maxSize; + maxKeySize = maxSize; + } + else + { + if (maxKeySize >= 256 + 1) + return -1; + if (keyFields != 1) + return -1; + keyFormat = 0; + maxKeySize = maxSize + (nvar > 1); + } + return 0; +} + +/* + * table definition packing: + * 16 bytes, MD5 of remain data + * 1 byte, ==1 + * 1 byte, total field# exclude key + * 1 byte, length of table name + * bytes, table name + * list + * 1 byte, field ID + * 1 byte, field type + * 1 byte, field size + * 1 byte, length of field name + * bytes, field name + * table attribute, Section encoding format + */ +int DTCTableDefinition::build_info_cache(void) +{ + if (numFields <= 0) + return 0; + + packedFullFieldSet.ptr = (char *)MALLOC(1 + usedFields + 2); + char *p = packedFullFieldSet.ptr + 1; + *p++ = 0; // field-id 0 encode: 00 + *p++ = 0; + for (int i = 1; i <= numFields; i++) + { + if (fieldList[i].fieldName == NULL) + continue; + *p++ = i; + } + packedFullFieldSet.len = p - packedFullFieldSet.ptr; + packedFullFieldSet.ptr[0] = packedFullFieldSet.len - 1 - 1; // real num fields + + packedFieldSet.ptr = (char *)MALLOC(1 + usedFields); + p = packedFieldSet.ptr + 1; + for (int i = 1; i <= numFields; i++) + { + if (fieldList[i].fieldName == NULL) + continue; + *p++ = i; + } + packedFieldSet.len = p - packedFieldSet.ptr; + packedFieldSet.ptr[0] = packedFieldSet.len - 1; // real num fields + + int n = 16 /*hash*/ + 1 /*ver*/ + 1 + /*nf*/ +1 /*nl*/ + tableName.len; + int nf = 0; + for (int i = 0; i <= numFields; i++) + { + if (fieldList[i].fieldName == NULL) + continue; + nf++; + n += 1 /*id*/ + 1 /*type*/ + 5 /*size*/ + 1 /*nl*/ + fieldList[i].nameLen; + } + n += encoded_bytes_simple_section(attr, DField::None); + + packedTableDefinition.ptr = (char *)MALLOC(n); + p = packedTableDefinition.ptr + 16; + *p++ = 1; /*ver*/ + + *p++ = tableName.len; + memcpy(p, tableName.ptr, tableName.len); + p += tableName.len; + + *p++ = nf; + for (int i = 0; i <= numFields; i++) + { + if (fieldList[i].fieldName == NULL) + continue; + *p++ = i; + *p++ = fieldList[i].fieldType; + p = encode_length(p, fieldList[i].fieldSize); + *p++ = fieldList[i].nameLen; + memcpy(p, fieldList[i].fieldName, fieldList[i].nameLen); + p += fieldList[i].nameLen; + } + + p = encode_simple_section(p, attr, DField::None); + + packedTableDefinition.len = p - packedTableDefinition.ptr; + + MD5Context md5; + MD5Init(&md5); + MD5Update(&md5, + (unsigned char *)packedTableDefinition.ptr + 16, + packedTableDefinition.len - 16); + MD5Final((unsigned char *)packedTableDefinition.ptr, &md5); + + memcpy(hash, packedTableDefinition.ptr, 16); + + return 0; +} + +int DTCTableDefinition::Unpack(const char *ptr, int len) +{ + if (numFields != 0 || maxFields != 0) + return -EC_TABLE_REDEFINED; + DTCBinary b = {len, (char *)ptr}; + + if (b < 24) + return -EC_BAD_SECTION_LENGTH; + + { // verify MD5 + MD5Context md5; + unsigned char h[16]; + MD5Init(&md5); + MD5Update(&md5, (unsigned char *)ptr + 16, len - 16); + MD5Final(h, &md5); + if (memcmp(h, ptr, 16) != 0) + return -EC_VERSION_MISMATCH; //EC_HASH_MISMATCH; + memcpy(hash, ptr, 16); + } + + b += 16; + uint8_t n; + n = *b++; + if (n != 1) + return -EC_VERSION_MISMATCH; + n = *b++; + + if (n == 0 || b < n + 1) + return -EC_BAD_SECTION_LENGTH; + tableName.ptr = (char *)MALLOC(n + 1); + if (tableName.ptr == NULL) + throw std::bad_alloc(); + memcpy(tableName.ptr, b.ptr, n); + tableName.ptr[n] = '\0'; + tableName.len = n; + b += n; + + int nf = *b++; + if (nf == 0) + return -EC_BAD_SECTION_LENGTH; + + maxFields = nf; + fieldList = maxFields == 0 ? NULL : (FieldDefinition *)calloc(maxFields, sizeof(FieldDefinition)); + // don't allocate defaultValue and uniqFields buffer, it's useless at client side + for (int i = 0; i < nf; i++) + { + if (b < 5) + return -EC_BAD_SECTION_LENGTH; + uint8_t id = *b++; + uint8_t type = *b++; + uint32_t size; + int rv; + char name[256]; + rv = decode_length(b, size); + if (rv) + return rv; + if (b < 2) + return -EC_BAD_SECTION_LENGTH; + n = *b++; + if (n == 0 || b < n) + return -EC_BAD_SECTION_LENGTH; + memcpy(name, b.ptr, n); + b += n; + name[n] = '\0'; + rv = add_field(id, name, type, size); + if (rv) + return rv; + } + + int rv = decode_simple_section(b, attr, DField::None); + if (rv) + return rv; + + return 0; +} diff --git a/src/search_local/index_storage/common/table_def.h b/src/search_local/index_storage/common/table_def.h new file mode 100644 index 0000000..4c9c921 --- /dev/null +++ b/src/search_local/index_storage/common/table_def.h @@ -0,0 +1,332 @@ +/* + * ===================================================================================== + * + * Filename: table_def.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_TABLEDEF_H_ +#define __CH_TABLEDEF_H_ + +#include "protocol.h" +#include "section.h" +#include "value.h" +#include "atomic.hpp" + +#include + +struct FieldDefinition +{ +public: + enum + { + FF_READONLY = 1, + FF_VOLATILE = 2, + FF_DISCARD = 4, + FF_DESC = 0x10, + FF_TIMESTAMP = 0x20, + }; + + typedef uint8_t fieldflag_t; + +public: + char *fieldName; + int fieldSize; //bytes + uint8_t fieldType; + uint8_t nameLen; + uint8_t offset; + fieldflag_t flags; + + int boffset; //field的bits起始偏移 + int bsize; //bits + uint16_t next; +}; + +extern const SectionDefinition tableAttributeDefinition; + +class TableAttribute : public SimpleSection +{ +public: + enum + { + SingleRow = 0x1, + admin_table = 0x2, + }; + + TableAttribute() : SimpleSection(tableAttributeDefinition) {} + ~TableAttribute() {} + + int is_single_row(void) const { return (int)(get_tag(1) ? (get_tag(1)->u64 & SingleRow) : 0); } + void set_single_row(void) { set_tag(1, get_tag(1) ? (get_tag(1)->u64 | SingleRow) : SingleRow); } + int is_admin_tab(void) const { return (int)(get_tag(1) ? (get_tag(1)->u64 & admin_table) : 0); } + void set_admin_tab(void) { set_tag(1, get_tag(1) ? (get_tag(1)->u64 | admin_table) : admin_table); } + unsigned int key_field_count(void) const { return get_tag(3) ? get_tag(3)->u64 : 0; } + void set_key_field_count(unsigned int n) { set_tag(3, n); } + + //压缩字段用来表示设置compressflag的字段id，传给client端使用。 + //该标识占用tag1的高八位，共计一个字节 + int compress_field_id(void) const { return get_tag(1) ? (((get_tag(1)->u64) >> 56) & 0xFF) : -1; } + void set_compress_flag(int n) + { + uint64_t tmp = (((uint64_t)n & 0xFF) << 56); + set_tag(1, get_tag(1) ? (get_tag(1)->u64 & tmp) : tmp); + } +}; + +class DTCTableDefinition +{ +private: + FieldDefinition *fieldList; + uint16_t nameHash[256]; + TableAttribute attr; + int maxFields; + int usedFields; + int numFields; + int keyFields; + DTCBinary tableName; + char hash[16]; + + // only used by client side + AtomicS32 count; + + // client side use this determine local/internal object + DTCValue *defaultValue; + + // only used by server side + DTCBinary packedTableDefinition; + DTCBinary packedFullFieldSet; + DTCBinary packedFieldSet; // field[0] not include + uint8_t *uniqFields; + int rowSize; //bits + uint16_t maxKeySize; // max real key size, maybe 0-256. (256 is valid) + int16_t hasAutoInc; // the autoinc field id, -1:none 0-254, fieldid + int16_t hasLastacc; + int16_t hasLastmod; + int16_t hasLastcmod; + int16_t hasCompress; //flag for compress + int16_t hasExpireTime; + uint8_t keyFormat; // 0:varsize, 1-255:fixed, large than 255 is invalid + uint8_t hasDiscard; + int8_t indexFields; // TREE_DATA, disabled in this release + uint8_t uniqFieldCnt; //the size of uniqFields member + uint8_t keysAsUniqField; /* 0 == NO, 1 == EXACT, 2 == SUBSET */ + + // no copy + DTCTableDefinition(const DTCTableDefinition &); + +public: + DTCTableDefinition(int m = 0); + ~DTCTableDefinition(void); + + // client side only + int Unpack(const char *, int); + int INC(void) + { + if (defaultValue == 0) + return ++count; + else + return 2; + }; + int DEC(void) + { + if (defaultValue == 0) + return --count; + else + return 1; + }; + + // common methods, by both side + const char *table_name(void) const { return tableName.ptr; } + const char *table_hash(void) const { return hash; } + int is_same_table(const DTCBinary &n) const { return string_equal(tableName, n); } + int is_same_table(const char *n) const { return string_equal(tableName, n); } + int is_same_table(const DTCTableDefinition &r) const { return !memcmp(hash, r.hash, sizeof(hash)); } + int is_same_table(const DTCTableDefinition *r) const { return r ? is_same_table(*r) : 0; } + int hash_equal(const DTCBinary &n) const { return n.len == 16 && !memcmp(hash, n.ptr, 16); } + int num_fields(void) const { return numFields; } + int key_fields(void) const { return keyFields; } + int field_id(const char *, int = 0) const; + const char *field_name(int id) const { return fieldList[id].fieldName; } + int field_type(int id) const { return fieldList[id].fieldType; } + int field_size(int id) const { return fieldList[id].fieldSize; } + int max_field_size(void) const + { + int max = 0; + for (int i = key_fields(); i <= num_fields(); ++i) + { + if (field_size(i) > max) + { + max = field_size(i); + } + } + return max; + } + unsigned int field_flags(int id) const { return fieldList[id].flags; } + const char *key_name(void) const { return fieldList[0].fieldName; } + int key_type(void) const { return fieldList[0].fieldType; } + int key_size(void) const { return fieldList[0].fieldSize; } + int is_read_only(int n) const { return fieldList[n].flags & FieldDefinition::FF_READONLY; } + int is_volatile(int n) const { return fieldList[n].flags & FieldDefinition::FF_VOLATILE; } + int is_discard(int n) const { return fieldList[n].flags & FieldDefinition::FF_DISCARD; } + int is_desc_order(int n) const { return fieldList[n].flags & FieldDefinition::FF_DESC; } + int is_timestamp(int n) const { return fieldList[n].flags & FieldDefinition::FF_TIMESTAMP; } + + int is_single_row(void) const { return attr.is_single_row(); } + int is_admin_table(void) const { return attr.is_admin_tab(); } + void set_admin_table(void) { return attr.set_admin_tab(); } + int compress_field_id(void) const { return attr.compress_field_id(); } + +#if !CLIENTAPI + // this macro test is scope-test only, didn't affected the class implementation + + // server side only + //to support Bitmap svr + const int row_size(void) const { return (rowSize + 7) / 8; }; //返回rowsize 单位bytes + const int b_row_size(void) const { return rowSize; }; //返回rowsize 单位bits + void set_row_size(int size) { rowSize = size; }; + int field_b_size(int id) const { return fieldList[id].bsize; } + int field_b_offset(int id) const { return fieldList[id].boffset; } + int field_offset(int id) const { return fieldList[id].offset; } + + void Pack(DTCBinary &v) { v = packedTableDefinition; } + int build_info_cache(void); + const DTCBinary &packed_field_set(void) const { return packedFullFieldSet; } + const DTCBinary &packed_field_set(int withKey) const { return withKey ? packedFullFieldSet : packedFieldSet; } + const DTCBinary &packed_definition(void) const { return packedTableDefinition; } + + int set_key(const char *name, uint8_t type, int size) { return add_field(0, name, type, size); } + void set_single_row(void) { attr.set_single_row(); }; + int set_table_name(const char *); + void set_default_value(int id, const DTCValue *val); + void get_default_row(DTCValue *) const; + const DTCValue *default_value(int id) const { return &defaultValue[id]; } + void set_auto_increment(int n) + { + if (n >= 0 && n <= numFields) + hasAutoInc = n; + } + void set_lastacc(int n) + { + if (n >= 0 && n <= numFields) + hasLastacc = n; + fieldList[n].flags |= FieldDefinition::FF_TIMESTAMP; + } + void set_lastmod(int n) + { + if (n >= 0 && n <= numFields) + hasLastmod = n; + fieldList[n].flags |= FieldDefinition::FF_TIMESTAMP; + } + void set_lastcmod(int n) + { + if (n >= 0 && n <= numFields) + hasLastcmod = n; + fieldList[n].flags |= FieldDefinition::FF_TIMESTAMP; + } + void set_compress_flag(int n) + { + if (n >= 0 && n <= numFields) + attr.set_compress_flag(n); + } + void set_expire_time(int n) + { + if (n >= 0 && n <= numFields) + hasExpireTime = n; + } + int has_discard(void) const { return hasDiscard; } + int has_auto_increment(void) const { return hasAutoInc >= 0; } + int key_auto_increment(void) const { return hasAutoInc == 0; } + int auto_increment_field_id(void) const { return hasAutoInc; } + int lastacc_field_id(void) const { return hasLastacc; } + int lastmod_field_id(void) const { return hasLastmod; } + int lastcmod_field_id(void) const { return hasLastcmod; } + int expire_time_field_id(void) const { return hasExpireTime; } + void mark_as_read_only(int n) { fieldList[n].flags |= FieldDefinition::FF_READONLY; } + void mark_as_volatile(int n) { fieldList[n].flags |= FieldDefinition::FF_VOLATILE; } + void mark_as_discard(int n) + { + fieldList[n].flags |= FieldDefinition::FF_DISCARD; + hasDiscard = 1; + } + void mark_order_desc(int n) { fieldList[n].flags |= FieldDefinition::FF_DESC; } + void mark_order_asc(int n) { fieldList[n].flags &= ~FieldDefinition::FF_DESC; } + uint8_t uniq_fields() const { return uniqFieldCnt; } + uint8_t *uniq_fields_list() { return uniqFields; } + const uint8_t *uniq_fields_list() const { return uniqFields; } + void mark_uniq_field(int n) { uniqFields[uniqFieldCnt++] = n; } + int key_as_uniq_field() const { return keysAsUniqField == 1 ? true : false; } + int key_part_of_uniq_field() const { return keysAsUniqField; } /* 0 -- NOT, 1 -- EXACT, 2 -- SUBSET */ + int index_fields() const { return indexFields; } + void set_index_fields(int n) { indexFields = n; } + + int set_key_fields(int n = 1); + int key_format(void) const { return keyFormat; } + int max_key_size(void) const { return maxKeySize; } + + DTCValue packed_key(const char *pk) + { + DTCValue key; + key.bin.ptr = (char *)pk; + key.bin.len = key_format() ?: (*(unsigned char *)pk + 1); + return key; + } + + int add_field(int id, const char *name, uint8_t type, int size); + int add_field(int id, const char *name, uint8_t type, int size, int bsize, int boffset); +#endif +}; + +class TableReference +{ +private: + DTCTableDefinition *tableDef; + +public: + DTCTableDefinition *table_definition(void) const { return tableDef; } + operator DTCTableDefinition *(void) { return tableDef; } + TableReference(DTCTableDefinition *t = 0) { tableDef = t; } + TableReference(const TableReference &c) { tableDef = c.table_definition(); } + virtual ~TableReference(void) {} + + inline void set_table_definition(DTCTableDefinition *t) { tableDef = t; } + + int num_fields(void) const { return tableDef->num_fields(); } + int key_fields(void) const { return tableDef->key_fields(); } + int key_type(void) const { return tableDef->key_type(); } + const char *key_name(void) const { return tableDef->key_name(); } + int key_size(void) const { return tableDef->key_size(); } + int field_type(int n) const { return tableDef->field_type(n); } + int field_size(int n) const { return tableDef->field_size(n); } + int field_id(const char *n) const { return tableDef->field_id(n); } + const char *field_name(int id) const { return tableDef->field_name(id); } + int is_same_table(const DTCBinary &n) const { return tableDef->is_same_table(n); } + int is_same_table(const char *n) const { return tableDef->is_same_table(n); } + int is_same_table(const DTCTableDefinition &r) const { return tableDef->is_same_table(r); } + int is_same_table(const DTCTableDefinition *r) const { return tableDef->is_same_table(r); } + int hash_equal(const DTCBinary &n) const { return tableDef->hash_equal(n); } + const char *table_name(void) const { return tableDef->table_name(); } + const char *table_hash(void) const { return tableDef->table_hash(); } + +#if !CLIENTAPI + // this macro test is scope-test only, didn't affected the class implementation + int key_format(void) const { return tableDef->key_format(); } + int key_auto_increment(void) const { return tableDef->key_auto_increment(); } + int auto_increment_field_id(void) const { return tableDef->auto_increment_field_id(); } + int field_offset(int n) const { return tableDef->field_offset(n); } + int field_b_size(int n) const { return tableDef->field_b_size(n); } + int field_b_offset(int n) const { return tableDef->field_b_offset(n); } + void get_default_row(DTCValue *value) const { tableDef->get_default_row(value); } +#endif +}; + +#endif diff --git a/src/search_local/index_storage/common/table_def_manager.cc b/src/search_local/index_storage/common/table_def_manager.cc new file mode 100644 index 0000000..a414b23 --- /dev/null +++ b/src/search_local/index_storage/common/table_def_manager.cc @@ -0,0 +1,202 @@ +/* + * ===================================================================================== + * + * Filename: table_def_manager.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "table_def_manager.h" +#include "mem_check.h" +#include "admin_tdef.h" + +TableDefinitionManager::TableDefinitionManager() +{ + _cur = 0; + _new = 0; + _def[0] = NULL; + _def[1] = NULL; +} + +TableDefinitionManager::~TableDefinitionManager() +{ + DEC_DELETE(_def[0]); + DEC_DELETE(_def[1]); +} + +bool TableDefinitionManager::set_new_table_def(DTCTableDefinition *t, int idx) +{ + _new = idx; + DEC_DELETE(_def[_new % 2]); + _def[_new % 2] = t; + _def[_new % 2]->INC(); + return true; +} + +bool TableDefinitionManager::set_cur_table_def(DTCTableDefinition *t, int idx) +{ + _cur = idx; + DEC_DELETE(_def[_cur % 2]); + _def[_cur % 2] = t; + _def[_cur % 2]->INC(); + return true; +} + +bool TableDefinitionManager::renew_cur_table_def() +{ + _cur = _new; + return true; +} + +bool TableDefinitionManager::save_new_table_conf() +{ + _save_dbconfig->cfgObj->Dump("../conf/table.conf", false); + _save_dbconfig->Destroy(); + _save_dbconfig = NULL; + return true; +} + +DTCTableDefinition *TableDefinitionManager::get_cur_table_def() +{ + return _def[_cur % 2]; +} + +DTCTableDefinition *TableDefinitionManager::get_new_table_def() +{ + return _def[_new % 2]; +} + +DTCTableDefinition *TableDefinitionManager::get_old_table_def() +{ + return _def[(_cur + 1) % 2]; +} + +int TableDefinitionManager::get_cur_table_idx() +{ + return _cur; +} + +DTCTableDefinition *TableDefinitionManager::get_table_def_by_idx(int idx) +{ + return _def[idx % 2]; +} + +DTCTableDefinition *TableDefinitionManager::get_hot_backup_table_def() +{ + return _hotbackup; +} + +bool TableDefinitionManager::build_hot_backup_table_def() +{ + _hotbackup = build_hot_backup_table(); + if (_hotbackup) + return true; + return false; +} + +bool TableDefinitionManager::save_db_config() +{ + if (_save_dbconfig) + { + log_error("_save_dbconfig not empty, maybe error"); + _save_dbconfig->Destroy(); + } + _save_dbconfig = _dbconfig; + _dbconfig = NULL; + return true; +} + +DTCTableDefinition *TableDefinitionManager::load_buffered_table(const char *buf) +{ + DTCTableDefinition *table = NULL; + char *bufLocal = (char *)MALLOC(strlen(buf) + 1); + memset(bufLocal, 0, strlen(buf) + 1); + strcpy(bufLocal, buf); + if (_dbconfig) + { + _dbconfig->Destroy(); + _dbconfig = NULL; + } + _dbconfig = DbConfig::load_buffered(bufLocal); + FREE(bufLocal); + if (!_dbconfig) + { + log_error("new dbconfig error"); + return false; + } + table = _dbconfig->build_table_definition(); + if (!table) + log_error("build table def error"); + return table; +} + +DTCTableDefinition *TableDefinitionManager::load_table(const char *file) +{ + char *buf; + int fd, len, unused; + DTCTableDefinition *ret; + if (!file || file[0] == '\0' || (fd = open(file, O_RDONLY)) < 0) + { + log_error("open config file error"); + return NULL; + } + lseek(fd, 0L, SEEK_SET); + len = lseek(fd, 0L, SEEK_END); + lseek(fd, 0L, SEEK_SET); + _buf = (char *)MALLOC(len + 1); + buf = (char *)MALLOC(len + 1); + unused = read(fd, _buf, len); + _buf[len] = '\0'; + close(fd); + // should copy one, as load_buffered_table will modify buf + strncpy(buf, _buf, len); + buf[len] = '\0'; + log_debug("read file to buf, len: %d, content: %s", len, _buf); + ret = load_buffered_table(buf); + if (!ret) + FREE_CLEAR(_buf); + FREE_CLEAR(buf); + _table = ret; + return ret; +} + +DTCTableDefinition *TableDefinitionManager::table_file_table_def() +{ + return _table; +} + +const char *TableDefinitionManager::table_file_buffer() +{ + return _buf; +} + +bool TableDefinitionManager::release_table_file_def_and_buffer() +{ + FREE_CLEAR(_buf); + DEC_DELETE(_table); + return true; +} + +bool TableDefinitionManager::renew_table_file_def(const char *buf, int len) +{ + FREE_CLEAR(_buf); + _buf = (char *)malloc(len + 1); + strncpy(_buf, buf, len); + _buf[len] = '\0'; + return true; +} diff --git a/src/search_local/index_storage/common/table_def_manager.h b/src/search_local/index_storage/common/table_def_manager.h new file mode 100644 index 0000000..9929290 --- /dev/null +++ b/src/search_local/index_storage/common/table_def_manager.h @@ -0,0 +1,69 @@ +/* + * ===================================================================================== + * + * Filename: table_def_manager.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_TABLEDEF_MANAGER_H_ +#define __DTC_TABLEDEF_MANAGER_H_ + +#include "singleton.h" +#include "table_def.h" +#include "dbconfig.h" + +class TableDefinitionManager +{ +public: + TableDefinitionManager(); + ~TableDefinitionManager(); + + static TableDefinitionManager *Instance() { return Singleton::Instance(); } + static void Destroy() { Singleton::Destroy(); } + + bool set_new_table_def(DTCTableDefinition *t, int idx); + bool set_cur_table_def(DTCTableDefinition *t, int idx); + bool renew_cur_table_def(); + bool save_new_table_conf(); + + DTCTableDefinition *get_cur_table_def(); + DTCTableDefinition *get_new_table_def(); + DTCTableDefinition *get_old_table_def(); + int get_cur_table_idx(); + DTCTableDefinition *get_table_def_by_idx(int idx); + + bool build_hot_backup_table_def(); + DTCTableDefinition *get_hot_backup_table_def(); + + bool save_db_config(); + DTCTableDefinition *load_buffered_table(const char *buff); + DTCTableDefinition *load_table(const char *file); + + DTCTableDefinition *table_file_table_def(); + const char *table_file_buffer(); + bool release_table_file_def_and_buffer(); + bool renew_table_file_def(const char *buf, int len); + +private: + int _cur; + int _new; + DTCTableDefinition *_def[2]; + DTCTableDefinition *_hotbackup; + // for cold start + char *_buf; + DTCTableDefinition *_table; + DbConfig *_dbconfig; + DbConfig *_save_dbconfig; +}; + +#endif diff --git a/src/search_local/index_storage/common/task_api.cc b/src/search_local/index_storage/common/task_api.cc new file mode 100644 index 0000000..f1c99ee --- /dev/null +++ b/src/search_local/index_storage/common/task_api.cc @@ -0,0 +1,369 @@ +/* + * ===================================================================================== + * + * Filename: task_api.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include "version.h" + +#include "decode.h" +#include "protocol.h" +#include "buffer_error.h" +#include "log.h" +#include "task_request.h" +#include "result.h" +#include + +inline int DTCFieldSet::Copy(const FieldSetByName &rq) +{ + const int num = rq.num_fields(); + for (int n = 0; n < num; n++) + { + const int id = rq.field_id(n); + // filter out invalid and duplicated fieldId + if (id >= 0 && id < INVALID_FIELD_ID && !field_present(id)) + add_field(id); + } + return 0; +} + +int DTCFieldValue::Copy(const FieldValueByName &rq, int mode, const DTCTableDefinition *tdef) +{ + int clear = 0; + const int num = rq.num_fields(); + unsigned knum = (int)tdef->key_fields(); + for (int n = 0; n < num; n++) + { + unsigned id = rq.field_id(n); + const DTCValue *val = rq.field_value(n); + unsigned op = rq.field_operation(n); + unsigned vt = rq.field_type(n); + + const int ft = tdef->field_type(id); + + if (vt == 0 || vt >= DField::TotalType) + { + // value type is invalid + return -EC_BAD_VALUE_TYPE; + } + + if (mode == 0) + { + // condition information + if (op >= DField::TotalComparison || DTCTask::validcomps[ft][op] == 0) + { + // invalid comparator id or + // the field type don't support the comparator + return -EC_BAD_OPERATOR; + } + + if (id < knum) + { + // part of cache hash key fields + if (op != DField::EQ) + { + // key must always EQ + return -EC_BAD_MULTIKEY; + } + } + else + { + // non-key condition exists, clear all_rows flag + clear = 1; + } + + if (DTCTask::validxtype[DField::Set][ft][vt] == 0) + { + // value type must compatible with field type + return -EC_BAD_OPERATOR; + } + } + else if (mode == 1) + { + // mode 1 is insert/replace + if (op != DField::Set) + { + // insert values, must be assignment + return -EC_BAD_OPERATOR; + } + if (DTCTask::validxtype[op][ft][vt] == 0) + { + // value type must compatible with field type + return -EC_BAD_OPERATOR; + } + } + else + { + // update operation + if (tdef->is_read_only(id)) + { + // read-only field cannot be updated + return -EC_READONLY_FIELD; + } + if (op >= DField::TotalOperation || DTCTask::validxtype[op][ft][vt] == 0) + { + // invalid operator id or + // the field type don't support the operator + return -EC_BAD_OPERATOR; + } + } + + // everything is fine + add_value(id, op, vt, *val); // val never bu NULL + // non key field, mark its type + update_type_mask(tdef->field_flags(id)); + } + + return clear; +} + +int TaskRequest::Copy(NCRequest &rq, const DTCValue *kptr) +{ + if (1) + { + /* timeout present */ + int client_timeout = requestInfo.tag_present(1) ? requestInfo.get_expire_time(3) : default_expire_time(); + + //log_debug("client api set timeout %d ms", client_timeout); + struct timeval now; + gettimeofday(&now, NULL); + + responseTimer = (int)(now.tv_sec * 1000000ULL + now.tv_usec); + expire_time = now.tv_sec * 1000ULL + now.tv_usec / 1000 + client_timeout; + timestamp = now.tv_sec; + } + + // this Copy() may be failed + int ret = DTCTask::Copy(rq, kptr); + if (ret < 0) + return ret; + + // inline from prepare_process() + if ((requestFlags & DRequest::Flag::MultiKeyValue)) + { + if (rq.kvl.key_fields() != key_fields()) + { + set_error(-EC_KEY_NEEDED, "decoder", "key field count incorrect"); + return -EC_KEY_NEEDED; + } + if (rq.kvl.key_count() < 1) + { + set_error(-EC_KEY_NEEDED, "decoder", "require key value"); + return -EC_KEY_NEEDED; + } + // set batch key + keyList = &rq.kvl; + } + else if (request_code() != DRequest::SvrAdmin) + { + if (build_packed_key() < 0) + return -1; // ERROR + calculate_barrier_key(); + } + + return 0; +} + +#define ERR_RET(ret_msg, fmt, args...) \ + do \ + { \ + set_error(err, "decoder", ret_msg); \ + log_debug(fmt, ##args); \ + return err; \ + } while (0) +int DTCTask::Copy(NCRequest &rq, const DTCValue *kptr) +{ + NCServer *sv = rq.server; + + TableReference::set_table_definition(rq.tdef); + stage = DecodeStageDone; + role = TaskRoleServer; + requestCode = rq.cmd; + requestType = cmd2type[requestCode]; + +#define PASSING_FLAGS ( \ + DRequest::Flag::no_cache | \ + DRequest::Flag::NoResult | \ + DRequest::Flag::no_next_server | \ + DRequest::Flag::MultiKeyValue | \ + 0) + + requestFlags = DRequest::Flag::KeepAlive | (rq.flags & PASSING_FLAGS); + + // tablename & hash + versionInfo.set_table_name(rq.tablename); + if (rq.tdef) + versionInfo.set_table_hash(rq.tdef->table_hash()); + versionInfo.set_serial_nr(sv->NextSerialNr()); + // app version + if (sv->appname) + versionInfo.set_tag(5, sv->appname); + // lib version + versionInfo.set_tag(6, "ctlib-v" DTC_VERSION); + versionInfo.set_tag(9, rq.keytype); + + // hot backup id + versionInfo.set_hot_backup_id(rq.hotBackupID); + + // hot backup timestamp + versionInfo.set_master_hb_timestamp(rq.master_hb_timestamp); + versionInfo.set_slave_hb_timestamp(rq.slave_hb_timestamp); + if (sv->tdef && rq.adminCode != 0) + versionInfo.set_data_table_hash(sv->tdef->table_hash()); + + if (rq.flags & DRequest::Flag::MultiKeyValue) + { + kptr = NULL; + if (sv->simple_batch_key() && rq.kvl.key_count() == 1) + { + /* single field single key batch, convert to normal */ + kptr = rq.kvl.val; + requestFlags &= ~DRequest::Flag::MultiKeyValue; + } + } + + key = kptr; + processFlags = PFLAG_ALLROWS; + + if (kptr) + { + requestInfo.set_key(*kptr); + } + // cmd + if (sv->get_timeout()) + { + requestInfo.set_timeout(sv->get_timeout()); + } + //limit + if (rq.limitCount) + { + requestInfo.set_limit_start(rq.limitStart); + requestInfo.set_limit_count(rq.limitCount); + } + if (rq.adminCode > 0) + { + requestInfo.set_admin_code(rq.adminCode); + } + + if (rq.fs.num_fields() > 0) + { + fieldList = new DTCFieldSet(rq.fs.num_fields()); + fieldList->Copy(rq.fs); // never failed + } + + if (rq.ui.num_fields() > 0) + { + /* decode updateInfo */ + updateInfo = new DTCFieldValue(rq.ui.num_fields()); + const int mode = requestCode == DRequest::Update ? 2 : 1; + int err = updateInfo->Copy(rq.ui, mode, rq.tdef); + if (err < 0) + ERR_RET("decode update info error", "decode update info error: %d", err); + } + + if (rq.ci.num_fields() > 0) + { + /* decode conditionInfo */ + conditionInfo = new DTCFieldValue(rq.ci.num_fields()); + int err = conditionInfo->Copy(rq.ci, 0, rq.tdef); + if (err < 0) + ERR_RET("decode condition info error", "decode update info error: %d", err); + if (err > 0) + { + clear_all_rows(); + } + } + + return 0; +} + +int DTCTask::process_internal_result(uint32_t ts) +{ + ResultPacket *rp = result_code() >= 0 ? get_result_packet() : NULL; + + if (rp) + { + resultInfo.set_total_rows(rp->totalRows); + } + else + { + resultInfo.set_total_rows(0); + if (result_code() == 0) + set_error(0, NULL, NULL); + } + + if (ts) + { + resultInfo.set_time_info(ts); + } + versionInfo.set_tag(6, "ctlib-v" DTC_VERSION); + if (result_key() == NULL && request_key() != NULL) + set_result_key(*request_key()); + + replyFlags = DRequest::Flag::KeepAlive | flag_multi_key_val(); + if (rp == NULL) + { + replyCode = DRequest::result_code; + } + else + { + replyCode = DRequest::DTCResultSet; + DTCBinary v; + v.ptr = rp->bc->data + rp->rowDataBegin; + v.len = rp->bc->usedBytes - rp->rowDataBegin; + if (rp->fieldSet) + result = new ResultSet(*rp->fieldSet, table_definition()); + else + result = new ResultSet((const uint8_t *)"", 0, num_fields(), table_definition()); + result->set_value_data(rp->numRows, v); + } + return 0; +} + +int DTCTask::update_key_expire_time(int max) +{ + int fid = dataTableDef->expire_time_field_id(); + for (int i = 0; i < updateInfo->num_fields(); ++i) + { + if (updateInfo->field_id(i) == fid) + { + // found updateInfo with expire time, update it + DTCValue *val = updateInfo->field_value(i); + if (val->s64 > 0 && val->s64 <= max) + { + log_debug("key expire time: %ld", val->s64); + val->s64 += time(NULL); + log_debug("set key expire at time: %ld", val->s64); + } + else if (val->s64 == 0) + { + log_debug("key expire time: 0, never expire"); + } + else + { + log_error("unknown key expire time"); + return -1; + } + break; + } + } + return 0; +} diff --git a/src/search_local/index_storage/common/task_base.cc b/src/search_local/index_storage/common/task_base.cc new file mode 100644 index 0000000..980a915 --- /dev/null +++ b/src/search_local/index_storage/common/task_base.cc @@ -0,0 +1,886 @@ +/* + * ===================================================================================== + * + * Filename: task_base.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "table_def_manager.h" +#include "task_base.h" +#include "decode.h" +#include "protocol.h" +#include "buffer_error.h" +#include "log.h" +#include "md5.h" + +int DTCTask::check_packet_size(const char *buf, int len) +{ + const PacketHeader *header = (PacketHeader *)buf; + if (len < (int)sizeof(PacketHeader)) + return 0; + + if (header->version != 1) + { // version not supported + return -1; + } + + if (header->scts != DRequest::Section::Total) + { // tags# mismatch + return -2; + } + + int i; + int64_t n; + for (i = 0, n = 0; i < DRequest::Section::Total; i++) + { +#if __BYTE_ORDER == __BIG_ENDIAN + n += bswap_32(header->len[i]); +#else + n += header->len[i]; +#endif + } + + if (n > MAXPACKETSIZE) + { // oversize + return -4; + } + + return (int)n + sizeof(PacketHeader); +} + +void DTCTask::decode_stream(SimpleReceiver &receiver) +{ + int rv; + + switch (stage) + { + default: + break; + case DecodeStageFatalError: + case DecodeStageDataError: + case DecodeStageDone: + return; + + case DecodeStageIdle: + receiver.init(); + case DecodeStageWaitHeader: + rv = receiver.fill(); + + if (rv == -1) + { + if (errno != EAGAIN && errno != EINTR && errno != EINPROGRESS) + stage = DecodeStageFatalError; + return; + } + + if (rv == 0) + { + errno = role == TaskRoleServer && stage == DecodeStageIdle ? 0 : ECONNRESET; + stage = DecodeStageFatalError; + return; + } + + stage = DecodeStageWaitHeader; + + if (receiver.remain() > 0) + { + return; + } + + if ((rv = decode_header(receiver.header(), &receiver.header())) < 0) + { + return; + } + + char *buf = packetbuf.Allocate(rv, role); + receiver.set(buf, rv); + if (buf == NULL) + { + set_error(-ENOMEM, "decoder", "Insufficient Memory"); + stage = DecodeStageDiscard; + } + } + + if (stage == DecodeStageDiscard) + { + rv = receiver.discard(); + if (rv == -1) + { + if (errno != EAGAIN && errno != EINTR) + { + stage = DecodeStageFatalError; + } + + return; + } + + if (rv == 0) + { + stage = DecodeStageFatalError; + return; + } + + stage = DecodeStageDataError; + return; + } + + rv = receiver.fill(); + + if (rv == -1) + { + if (errno != EAGAIN && errno != EINTR) + { + stage = DecodeStageFatalError; + } + + return; + } + + if (rv == 0) + { + stage = DecodeStageFatalError; + return; + } + + if (receiver.remain() <= 0) + { + decode_request(receiver.header(), receiver.c_str()); + } + + return; +} + +// Decode data from packet +// type 0: clone packet +// type 1: eat(keep&free) packet +// type 2: use external packet +void DTCTask::decode_packet(char *packetIn, int packetLen, int type) +{ + PacketHeader header; +#if __BYTE_ORDER == __BIG_ENDIAN + PacketHeader out[1]; +#else + PacketHeader *const out = &header; +#endif + + switch (stage) + { + default: + break; + case DecodeStageFatalError: + case DecodeStageDataError: + case DecodeStageDone: + return; + } + + if (packetLen < (int)sizeof(PacketHeader)) + { + stage = DecodeStageFatalError; + return; + } + + memcpy((char *)&header, packetIn, sizeof(header)); + + int rv = decode_header(header, out); + if (rv < 0) + { + return; + } + + if ((int)(sizeof(PacketHeader) + rv) > packetLen) + { + stage = DecodeStageFatalError; + return; + } + + char *buf = (char *)packetIn + sizeof(PacketHeader); + switch (type) + { + default: + case 0: + // clone packet buffer + buf = packetbuf.Clone(buf, rv, role); + if (buf == NULL) + { + set_error(-ENOMEM, "decoder", "Insufficient Memory"); + stage = DecodeStageDataError; + return; + } + break; + + case 1: + packetbuf.Push(packetIn); + break; + case 2: + break; + } + + decode_request(*out, buf); + return; +} + +int DTCTask::decode_header(const PacketHeader &header, PacketHeader *out) +{ + + if (header.version != 1) + { // version not supported + stage = DecodeStageFatalError; + log_debug("version incorrect: %d", header.version); + + return -1; + } + + if (header.scts != DRequest::Section::Total) + { // tags# mismatch + stage = DecodeStageFatalError; + log_debug("session count incorrect: %d", header.scts); + + return -2; + } + + int i; + int64_t n; + for (i = 0, n = 0; i < DRequest::Section::Total; i++) + { +#if __BYTE_ORDER == __BIG_ENDIAN + const unsigned int v = bswap_32(header.len[i]); + if (out) + out->len[i] = v; +#else + const unsigned int v = header.len[i]; +#endif + n += v; + } + + if (n > MAXPACKETSIZE) + { // oversize + stage = DecodeStageFatalError; + log_debug("package size error: %ld", (long)n); + + return -4; + } + + if (header.cmd == DRequest::result_code || header.cmd == DRequest::DTCResultSet) + { + replyCode = header.cmd; + replyFlags = header.flags; + } + else + { + requestCode = header.cmd; + requestFlags = header.flags; + requestType = cmd2type[requestCode]; + } + + stage = DecodeStageWaitData; + + return (int)n; +} + +int DTCTask::validate_section(PacketHeader &header) +{ + int i; + int m; + for (i = 0, m = 0; i < DRequest::Section::Total; i++) + { + if (header.len[i]) + m |= 1 << i; + } + + if (header.cmd > 17 || !(validcmds[role] & (1 << header.cmd))) + { + set_error(-EC_BAD_COMMAND, "decoder", "Invalid Command"); + return -1; + } + + if ((m & validsections[header.cmd][0]) != validsections[header.cmd][0]) + { + set_error(-EC_MISSING_SECTION, "decoder", "Missing Section"); + return -1; + } + + if ((m & ~validsections[header.cmd][1]) != 0) + { + log_error("m[%x] valid[%x]", m, validsections[header.cmd][1]); + set_error(-EC_EXTRA_SECTION, "decoder", "Extra Section"); + return -1; + } + + return 0; +} + +void DTCTask::decode_request(PacketHeader &header, char *p) +{ +#if !CLIENTAPI + if (DRequest::ReloadConfig == requestCode && TaskTypeHelperReloadConfig == requestType) + { + log_error("TaskTypeHelperReloadConfig == requestType"); + stage = DecodeStageDone; + return; + } +#endif + int id = 0; + int err = 0; + +#define ERR_RET(ret_msg, fmt, args...) \ + do \ + { \ + set_error(err, "decoder", ret_msg); \ + log_debug(fmt, ##args); \ + goto error; \ + } while (0) + + if (header.len[id]) + { + /* decode version info */ + err = decode_simple_section(p, header.len[id], versionInfo, DField::None); + if (err) + ERR_RET("decode version info error", "decode version info error: %d", err); + + /* backup serialNr */ + serialNr = versionInfo.serial_nr(); + + if (header.cmd == DRequest::SvrAdmin) + { + set_table_definition(hotbackupTableDef); // 管理命令，换成管理表定义 + log_debug("hb table ptr: %p, name: %s", hotbackupTableDef, hotbackupTableDef->table_name()); + } + + if (role == TaskRoleServer) + { + + /* local storage no need to check table, because it always set it to "@HOT_BACKUP", checking tablename */ + if (requestCode != DRequest::Replicate && !is_same_table(versionInfo.table_name())) + { + err = -EC_TABLE_MISMATCH; + requestFlags |= DRequest::Flag::NeedTableDefinition; + + ERR_RET("table mismatch", "table mismatch: %d, client[%.*s], server[%s]", err, + versionInfo.table_name().len, + versionInfo.table_name().ptr, + table_name()); + } + + /* check table hash */ + if (requestCode != DRequest::Replicate && versionInfo.table_hash().len > 0 && !hash_equal(versionInfo.table_hash())) + { +#if !CLIENTAPI + + DTCTableDefinition *oldDef = TableDefinitionManager::Instance()->get_old_table_def(); + if (oldDef && oldDef->hash_equal(versionInfo.table_hash())) + { + requestFlags |= DRequest::Flag::NeedTableDefinition; + log_error("mismatch new table, but match old table, so notify update tabledefinition! pid [%d]", getpid()); + } + else + { + err = -EC_CHECKSUM_MISMATCH; + requestFlags |= DRequest::Flag::NeedTableDefinition; + + ERR_RET("table mismatch", "table mismatch: %d", err); + } + +#else + + err = -EC_CHECKSUM_MISMATCH; + requestFlags |= DRequest::Flag::NeedTableDefinition; + + ERR_RET("table mismatch", "table mismatch: %d", err); + +#endif + } + /* checking keytype */ + /* local storage no need to check table, because it always set it to "unsigned int", not the + * acutal key type */ + if (requestCode != DRequest::Replicate) + { + const unsigned int t = versionInfo.key_type(); + const int rt = key_type(); + if (!(requestFlags & DRequest::Flag::MultiKeyValue) && (t >= DField::TotalType || !validktype[t][rt])) + { + err = -EC_BAD_KEY_TYPE; + requestFlags |= DRequest::Flag::NeedTableDefinition; + + ERR_RET("key type incorrect", "key type incorrect: %d", err); + } + } + } + + } //end of version info + + if (requestCode != DRequest::Replicate && validate_section(header) < 0) + { + stage = DecodeStageDataError; + return; + } + + if (header.cmd == DRequest::Nop && role == TaskRoleServer) + { + stage = DecodeStageDataError; + return; + } + + p += header.len[id++]; + + // only client use remote table + if (header.len[id] && allow_remote_table()) + { + /* decode table definition */ + DTCTableDefinition *tdef; + try + { + tdef = new DTCTableDefinition; + } + catch (int e) + { + err = e; + goto error; + } + int rv = tdef->Unpack(p, header.len[id]); + if (rv != 0) + { + delete tdef; + + ERR_RET("unpack table info error", "unpack table info error: %d", rv); + } + DTCTableDefinition *old = table_definition(); + DEC_DELETE(old); + set_table_definition(tdef); + tdef->INC(); + if (!(header.flags & DRequest::Flag::admin_table)) + mark_has_remote_table(); + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode request info */ + if (requestFlags & DRequest::Flag::MultiKeyValue) + err = decode_simple_section(p, header.len[id], requestInfo, + DField::Binary); + else + err = decode_simple_section(p, header.len[id], requestInfo, + table_definition() ? field_type(0) : DField::None); + + if (err) + ERR_RET("decode request info error", "decode request info error: %d", err); + + /* key present */ + key = requestInfo.Key(); + + if (header.cmd == DRequest::SvrAdmin && requestInfo.admin_code() == DRequest::ServerAdminCmd::RegisterHB) + { + if (versionInfo.data_table_hash().len <= 0 || !dataTableDef->hash_equal(versionInfo.data_table_hash())) + { + err = -EC_CHECKSUM_MISMATCH; + requestFlags |= DRequest::Flag::NeedTableDefinition; + + ERR_RET("table mismatch", "table mismatch: %d", err); + } + } + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode result code */ + err = decode_simple_section(p, header.len[id], resultInfo, + table_definition() ? field_type(0) : DField::None); + + if (err) + ERR_RET("decode result info error", "decode result info error: %d", err); + rkey = resultInfo.Key(); + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode updateInfo */ + const int mode = requestCode == DRequest::Update ? 2 : 1; + err = decode_field_value(p, header.len[id], mode); + + if (err) + ERR_RET("decode update info error", "decode update info error: %d", err); + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode conditionInfo */ + err = decode_field_value(p, header.len[id], 0); + + if (err) + ERR_RET("decode condition error", "decode condition error: %d", err); + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode fieldset */ + err = decode_field_set(p, header.len[id]); + + if (err) + ERR_RET("decode field set error", "decode field set error: %d", err); + } + + p += header.len[id++]; + + if (header.len[id]) + { + /* decode resultset */ + err = decode_result_set(p, header.len[id]); + + if (err) + ERR_RET("decode result set error", "decode result set error: %d", err); + } + + stage = DecodeStageDone; + return; + +#undef ERR_RET + +error: + set_error(err, "decoder", NULL); + stage = DecodeStageDataError; +} + +int DTCTask::decode_field_set(char *d, int l) +{ + uint8_t mask[32]; + FIELD_ZERO(mask); + + DTCBinary bin = {l, d}; + if (!bin) + return -EC_BAD_SECTION_LENGTH; + + const int num = *bin++; + if (num == 0) + return 0; + int realnum = 0; + uint8_t idtab[num]; + + /* old: buf -> local -> id -> idtab -> task */ + /* new: buf -> local -> idtab -> task */ + for (int i = 0; i < num; i++) + { + int nd = 0; + int rv = decode_field_id(bin, idtab[realnum], table_definition(), nd); + if (rv != 0) + return rv; + if (FIELD_ISSET(idtab[realnum], mask)) + continue; + FIELD_SET(idtab[realnum], mask); + realnum++; + if (nd) + requestFlags |= DRequest::Flag::NeedTableDefinition; + } + + if (!!bin.len) + return -EC_EXTRA_SECTION_DATA; + + /* allocate max field in field set, first byte indicate real field num */ + if (!fieldList) + { + try + { + fieldList = new DTCFieldSet(idtab, + realnum, + num_fields() + 1); + } + catch (int err) + { + return -ENOMEM; + } + } + else + { + if (fieldList->max_fields() < num_fields() + 1) + { + fieldList->Realloc(num_fields() + 1); + } + if (fieldList->Set(idtab, realnum) < 0) + { + log_warning("fieldList not exist in pool"); + return -EC_TASKPOOL; + } + } + + return 0; +} + +int DTCTask::decode_field_value(char *d, int l, int mode) +{ + DTCBinary bin = {l, d}; + if (!bin) + return -EC_BAD_SECTION_LENGTH; + + const int num = *bin++; + if (num == 0) + return 0; + + /* conditionInfo&updateInfo at largest size in pool, numFields indicate real field */ + DTCFieldValue *fv; + if (mode == 0 && conditionInfo) + { + fv = conditionInfo; + } + else if (mode != 0 && updateInfo) + { + fv = updateInfo; + } + else + { + try + { + fv = new DTCFieldValue(num); + } + catch (int err) + { + return err; + } + if (mode == 0) + conditionInfo = fv; + else + updateInfo = fv; + } + + if (fv->max_fields() < num) + { + fv->Realloc(num); + } + + int err; + int keymask = 0; + for (int i = 0; i < num; i++) + { + uint8_t id; + DTCValue val; + uint8_t op; + uint8_t vt; + + if (!bin) + { + err = -EC_BAD_SECTION_LENGTH; + goto bad; + } + + op = *bin++; + vt = op & 0xF; + op >>= 4; + + int nd = 0; + /* buf -> local -> id */ + err = decode_field_id(bin, id, table_definition(), nd); + if (err != 0) + goto bad; + err = -EC_BAD_INVALID_FIELD; + if (id > num_fields()) + goto bad; + + const int ft = field_type(id); + + if (vt >= DField::TotalType) + { + err = -EC_BAD_VALUE_TYPE; + goto bad; + } + + err = -EC_BAD_OPERATOR; + + if (mode == 0) + { + if (op >= DField::TotalComparison || validcomps[ft][op] == 0) + goto bad; + if (id < key_fields()) + { + if (op != DField::EQ) + { + err = -EC_BAD_MULTIKEY; + goto bad; + } + } + else + clear_all_rows(); + + if (validxtype[DField::Set][ft][vt] == 0) + goto bad; + } + else if (mode == 1) + { + if (op != DField::Set) + goto bad; + if (validxtype[op][ft][vt] == 0) + goto bad; + } + else + { + if (table_definition()->is_read_only(id)) + { + err = -EC_READONLY_FIELD; + goto bad; + } + if (op >= DField::TotalOperation || validxtype[op][ft][vt] == 0) + goto bad; + } + + /* avoid one more copy of DTCValue*/ + /* int(len, value): buf -> local; buf -> local -> tag */ + /* str(len, value): buf -> local -> tag; buf -> tag */ + //err = decode_data_value(bin, val, vt); + err = decode_data_value(bin, *fv->next_field_value(), vt); + if (err != 0) + goto bad; + //fv->add_value(id, op, vt, val); + fv->add_value_no_val(id, op, vt); + if (nd) + requestFlags |= DRequest::Flag::NeedTableDefinition; + if (id < key_fields()) + { + if ((keymask & (1 << id)) != 0) + { + err = -EC_BAD_MULTIKEY; + goto bad; + } + keymask |= 1 << id; + } + else + { + fv->update_type_mask(table_definition()->field_flags(id)); + } + } + + if (!!bin) + { + err = -EC_EXTRA_SECTION_DATA; + goto bad; + } + + return 0; +bad: + /* free when distructed instread free here*/ + //delete fv; + return err; +} + +int DTCTask::decode_result_set(char *d, int l) +{ + uint32_t nrows; + int num; + + uint8_t mask[32]; + FIELD_ZERO(mask); + + DTCBinary bin = {l, d}; + + int err = decode_length(bin, nrows); + if (err) + return err; + + if (!bin) + return -EC_BAD_SECTION_LENGTH; + + num = *bin++; + + /* buf -> id */ + //check duplicate or bad field id + int haskey = 0; + if (num > 0) + { + if (bin < num) + return -EC_BAD_SECTION_LENGTH; + + for (int i = 0; i < num; i++) + { + int t = bin[i]; + if (t == 0) + haskey = 1; + if (t == 255) + return -EC_BAD_FIELD_ID; + if (FIELD_ISSET(t, mask)) + return -EC_DUPLICATE_FIELD; + FIELD_SET(t, mask); + } + } + + /* result's fieldset at largest size */ + /* field ids: buf -> result */ + if (!result) + { + try + { + result = new ResultSet((uint8_t *)bin.ptr, + num, + num_fields() + 1, + table_definition()); + } + catch (int err) + { + return err; + } + } + else + { + if (result->field_set_max_fields() < num_fields() + 1) + result->realloc_field_set(num_fields() + 1); + result->Set((uint8_t *)bin.ptr, num); + } + + bin += num; + result->set_value_data(nrows, bin); + return 0; +} + +int ResultSet::decode_row(void) +{ + if (err) + return err; + if (rowno == numRows) + return -EC_NO_MORE_DATA; + for (int i = 0; i < num_fields(); i++) + { + const int id = field_id(i); + err = decode_data_value(curr, row[id], row.field_type(id)); + if (err) + return err; + } + rowno++; + return 0; +} + +int packet_body_len(PacketHeader &header) +{ + int pktbodylen = 0; + for (int i = 0; i < DRequest::Section::Total; i++) + { +#if __BYTE_ORDER == __BIG_ENDIAN + const unsigned int v = bswap_32(header.len[i]); +#else + const unsigned int v = header.len[i]; +#endif + pktbodylen += v; + } + return pktbodylen; +} diff --git a/src/search_local/index_storage/common/task_base.h b/src/search_local/index_storage/common/task_base.h new file mode 100644 index 0000000..e7182ee --- /dev/null +++ b/src/search_local/index_storage/common/task_base.h @@ -0,0 +1,512 @@ +/* + * ===================================================================================== + * + * Filename: task_base.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_TASK_H__ +#define __CH_TASK_H__ + +#include "table_def.h" +#include "section.h" +#include "field.h" +#include "packet.h" +#include "result.h" +#include "buffer_error.h" +#include "timestamp.h" +#include +#include "log.h" +#include "buffer.h" +#include "receiver.h" + +class NCRequest; + +enum DecodeResult +{ + DecodeFatalError, // no response needed + DecodeDataError, // response with error code + DecodeIdle, // no data received + DecodeWaitData, // partial decoding + DecodeDone // full packet +}; + +enum DecodeStage +{ // internal use + DecodeStageFatalError, // same as result + DecodeStageDataError, // same as result + DecodeStageIdle, // same as result + DecodeStageWaitHeader, // partial header + DecodeStageWaitData, // partial data + DecodeStageDiscard, // error mode, discard remain data + DecodeStageDone // full packet +}; + +enum TaskRole +{ + TaskRoleServer = 0, /* server, for incoming thread */ + TaskRoleClient, /* client, reply from server */ + TaskRoleHelperReply, /* helper, reply from server */ +}; + +enum TaskType +{ + TaskTypeAdmin = 0, + TaskTypeRead, /* Read Only operation */ + TaskTypeWrite, /* Modify data */ + TaskTypeCommit, /* commit dirty data */ + TaskTypeWriteHbLog, + TaskTypeReadHbLog, + TaskTypeWriteLruHbLog, + TaskTypeRegisterHbLog, + TaskTypeQueryHbLogInfo, + TaskTypeHelperReloadConfig, +}; + +enum CHITFLAG +{ + HIT_INIT = 0, // hit init flag + HIT_SUCCESS = 1, // hit success flag +}; + +class DTCTask : public TableReference +{ +public: + static const DecodeResult stage2result[]; + DecodeResult get_decode_result(void) { return stage2result[stage]; }; + static const uint32_t validcmds[]; + static const uint16_t cmd2type[]; + static const uint16_t validsections[][2]; + static const uint8_t validktype[DField::TotalType][DField::TotalType]; + static const uint8_t validxtype[DField::TotalOperation][DField::TotalType][DField::TotalType]; + static const uint8_t validcomps[DField::TotalType][DField::TotalComparison]; + +protected: + class BufferPool + { + // simple buffer pool, only keep 2 buffers + public: + char *ptr[2]; + int len[2]; + BufferPool() + { + ptr[0] = NULL; + ptr[1] = NULL; + } + ~BufferPool() + { + FREE_IF(ptr[0]); + FREE_IF(ptr[1]); + } + void Push(char *v) + { + ptr[ptr[0] ? 1 : 0] = v; + } + + inline char *Allocate(int size, TaskRole role) + { + if (role == TaskRoleServer || role == TaskRoleClient) + { + CreateBuff(size, len[0], &ptr[0]); + if (ptr[0] == NULL) + return NULL; + return ptr[0]; + } + else + { + CreateBuff(size, len[1], &ptr[1]); + if (ptr[1] == NULL) + return NULL; + return ptr[1]; + } + } + char *Clone(char *buff, int size, TaskRole role) + { + if (role != TaskRoleClient) + return NULL; + char *p; + if (ptr[0]) + p = ptr[1] = (char *)MALLOC(size); + else + p = ptr[0] = (char *)MALLOC(size); + + if (p) + memcpy(p, buff, size); + return p; + } + }; + +public: + char *migratebuf; + +protected: // decoder informations + DecodeStage stage; + TaskRole role; + BufferPool packetbuf; + + //don't use it except packet decoding + DTCTableDefinition *dataTableDef; + DTCTableDefinition *hotbackupTableDef; + + // used by replicate table definition + DTCTableDefinition *replicateTableDef; + +protected: // packet info, read-only + DTCFieldValue *updateInfo; + DTCFieldValue *conditionInfo; + DTCFieldSet *fieldList; + +public: + ResultSet *result; + +public: // packet info, read-write + DTCVersionInfo versionInfo; + DTCRequestInfo requestInfo; + DTCResultInfo resultInfo; + +protected: // working data + uint64_t serialNr; /* derived from packet */ + const DTCValue *key; /* derived from packet */ + const DTCValue *rkey; /* processing */ + /* resultWriter only create once in task entire life */ + ResultWriter *resultWriter; /* processing */ + int resultWriterReseted; + uint8_t requestCode; /* derived from packet */ + uint8_t requestType; /* derived from packet */ + uint8_t requestFlags; /* derived from packet */ + uint8_t replyCode; /* processing */ + uint8_t replyFlags; /* derived from packet */ + enum + { + PFLAG_REMOTETABLE = 1, + PFLAG_ALLROWS = 2, + PFLAG_PASSTHRU = 4, + PFLAG_ISHIT = 8, + PFLAG_FETCHDATA = 0x10, + PFLAG_ALLOWREMOTETABLE = 0x20, + PFLAG_FIELDSETWITHKEY = 0x40, + PFLAG_BLACKHOLED = 0x80, + }; + uint8_t processFlags; /* processing */ + +protected: + // return total packet size + int decode_header(const PacketHeader &in, PacketHeader *out = NULL); + int validate_section(PacketHeader &header); + void decode_request(PacketHeader &header, char *p); + int decode_field_value(char *d, int l, int m); + int decode_field_set(char *d, int l); + int decode_result_set(char *d, int l); + +public: + DTCTask(DTCTableDefinition *tdef = NULL, TaskRole r = TaskRoleServer, int final = 0) : TableReference(tdef), + migratebuf(NULL), + stage(final ? DecodeStageDataError : DecodeStageIdle), + role(r), + dataTableDef(tdef), + hotbackupTableDef(NULL), + replicateTableDef(NULL), + updateInfo(NULL), + conditionInfo(NULL), + fieldList(NULL), + result(NULL), + serialNr(0), + key(NULL), + rkey(NULL), + resultWriter(NULL), + resultWriterReseted(0), + requestCode(0), + requestType(0), + requestFlags(0), + replyCode(0), + replyFlags(0), + processFlags(PFLAG_ALLROWS) + { + } + + virtual ~DTCTask(void) + { + DELETE(updateInfo); + DELETE(conditionInfo); + DELETE(fieldList); + DELETE(resultWriter); + DELETE(result); + FREE_IF(migratebuf); + } + // linked clone + inline DTCTask(const DTCTask &orig) + { + DTCTask(); + Copy(orig); + } + + // these Copy()... only apply to empty DTCTask + // linked clone + int Copy(const DTCTask &orig); + // linked clone with replace key + int Copy(const DTCTask &orig, const DTCValue *newkey); + // replace row + int Copy(const RowValue &); + // internal API + int Copy(NCRequest &rq, const DTCValue *key); + + inline void Clean() + { + TableReference::set_table_definition(NULL); + if (updateInfo) + updateInfo->Clean(); + if (conditionInfo) + conditionInfo->Clean(); + if (fieldList) + fieldList->Clean(); + if (result) + result->Clean(); + versionInfo.Clean(); + requestInfo.Clean(); + resultInfo.Clean(); + + //serialNr = 0; + key = NULL; + rkey = NULL; + if (resultWriter) + { + resultWriter->Clean(); + resultWriterReseted = 0; + } + requestCode = 0; + requestType = 0; + requestFlags = 0; + replyCode = 0; + replyFlags = 0; + processFlags = PFLAG_ALLROWS; + } + + //////////// some API access request property + inline void set_data_table(DTCTableDefinition *t) + { + TableReference::set_table_definition(t); + dataTableDef = t; + } + inline void set_hotbackup_table(DTCTableDefinition *t) { hotbackupTableDef = t; } + inline void set_replicate_table(DTCTableDefinition *t) { replicateTableDef = t; } + inline DTCTableDefinition *get_replicate_table() { return replicateTableDef; } +#if 0 + DTCTableDefinition *table_definition(void) const { return tableDef; } + int key_format(void) const { return tableDef->key_format(); } + int key_fields(void) const { return tableDef->key_fields(); } + int key_auto_increment(void) const { return tableDef->key_auto_increment(); } + int auto_increment_field_id(void) const { return tableDef->auto_increment_field_id(); } + int field_type(int n) const { return tableDef->field_type(n); } + int field_offset(int n) const { return tableDef->field_type(n); } + int field_id(const char *n) const { return tableDef->field_id(n); } + const char* field_name(int id) const { return tableDef->field_name(id); } +#endif + + // This code has to value (not very usefull): + // DRequest::ResultInfo --> result/error code/key only + // DRequest::DTCResultSet --> result_code() >=0, with DTCResultSet + // please use result_code() for detail error code + int reply_code(void) const { return replyCode; } + + // Retrieve request key + int has_request_key(void) const { return key != NULL; } + const DTCValue *request_key(void) const { return key; } + unsigned int int_key(void) const { return (unsigned int)(request_key()->u64); } + void update_key(RowValue *r) { (*r)[0] = *request_key(); } + + // only for test suit + void set_request_condition(DTCFieldValue *cond) { conditionInfo = cond; } + void set_request_key(DTCValue *val) { key = val; } + + const DTCFieldValue *request_condition(void) const { return conditionInfo; } + const DTCFieldValue *request_operation(void) const { return updateInfo; } + + //for migrate + void set_request_operation(DTCFieldValue *ui) { updateInfo = ui; } + const DTCFieldSet *request_fields(void) const { return fieldList; } + const uint64_t request_serial(void) const { return serialNr; } + + // result key + const DTCValue *result_key(void) const { return rkey; } + // only if insert w/o key + void set_result_key(const DTCValue &v) + { + resultInfo.set_key(v); + rkey = &v; + } + + static int max_header_size(void) { return sizeof(PacketHeader); } + static int min_header_size(void) { return sizeof(PacketHeader); } + static int check_packet_size(const char *buf, int len); + + // Decode data from fd + void decode_stream(SimpleReceiver &receiver); + DecodeResult Decode(SimpleReceiver &receiver) + { + decode_stream(receiver); + return get_decode_result(); + } + + // Decode data from packet + // type 0: clone packet + // type 1: eat(keep&free) packet + // type 2: use external packet + void decode_packet(char *packetIn, int packetLen, int type); + DecodeResult Decode(char *packetIn, int packetLen, int type) + { + decode_packet(packetIn, packetLen, type); + return get_decode_result(); + } + + DecodeResult Decode(const char *packetIn, int packetLen) + { + decode_packet((char *)packetIn, packetLen, 0); + return get_decode_result(); + }; + + inline void BeginStage() { stage = DecodeStageIdle; } + + // change role from TaskRoleServer to TaskRoleHelperReply + // cleanup decode state, prepare reply from helper + inline void prepare_decode_reply(void) + { + role = TaskRoleHelperReply; + stage = DecodeStageIdle; + } + + inline void set_role_as_server() { role = TaskRoleServer; } + + // set error code before Packet::encode_result(); + // err is positive errno + void set_error(int err, const char *from, const char *msg) + { + resultInfo.set_error(err, from, msg); + } + void set_error_dup(int err, const char *from, const char *msg) + { + resultInfo.set_error_dup(err, from, msg); + } + // retrieve previous result code + // >= 0 success + // < 0 err, negative value of set_error() + int result_code(void) { return resultInfo.result_code(); } + int allow_remote_table(void) const { return processFlags & PFLAG_ALLOWREMOTETABLE; } + void mark_allow_remote_table(void) { processFlags |= PFLAG_ALLOWREMOTETABLE; } + void mark_has_remote_table(void) { processFlags |= PFLAG_REMOTETABLE; } + DTCTableDefinition *remote_table_definition(void) + { + if (processFlags & PFLAG_REMOTETABLE) + return table_definition(); + return NULL; + } + + //////////// some API for request processing + // Client Request Code + int Role(void) const { return role; } + int request_code(void) const { return requestCode; } + void set_request_code(uint8_t code) { requestCode = code; } + int request_type(void) const { return requestType; } + void set_request_type(TaskType type) { requestType = type; } + int flag_keep_alive(void) const { return requestFlags & DRequest::Flag::KeepAlive; } + int flag_table_definition(void) const { return requestFlags & DRequest::Flag::NeedTableDefinition; } + int flag_no_cache(void) const { return requestFlags & DRequest::Flag::no_cache; } + int flag_no_result(void) const { return requestFlags & DRequest::Flag::NoResult; } + int flag_no_next_server(void) const { return requestFlags & DRequest::Flag::no_next_server; } + int flag_multi_key_val(void) const { return requestFlags & DRequest::Flag::MultiKeyValue; } + int flag_multi_key_result(void) const { return replyFlags & DRequest::Flag::MultiKeyValue; } + int flag_admin_table(void) const { return requestFlags & DRequest::Flag::admin_table; } + int flag_pass_thru(void) const { return processFlags & PFLAG_PASSTHRU; } + int flag_fetch_data(void) const { return processFlags & PFLAG_FETCHDATA; } + int flag_is_hit(void) const { return processFlags & PFLAG_ISHIT; } + int flag_field_set_with_key(void) const { return processFlags & PFLAG_FIELDSETWITHKEY; } + int flag_black_hole(void) const { return processFlags & PFLAG_BLACKHOLED; } + void mark_as_pass_thru(void) { processFlags |= PFLAG_PASSTHRU; } + void mark_as_fetch_data(void) { processFlags |= PFLAG_FETCHDATA; } + void mark_as_hit(void) { processFlags |= PFLAG_ISHIT; } + void mark_field_set_with_key(void) { processFlags |= PFLAG_FIELDSETWITHKEY; } + void mark_as_black_hole(void) { processFlags |= PFLAG_BLACKHOLED; } + void set_result_hit_flag(CHITFLAG hitFlag) { resultInfo.set_hit_flag((uint32_t)hitFlag); } + + // API for key expire time + int update_key_expire_time(int max); + + // this is count only request + int count_only(void) const { return fieldList == NULL; } + // this is non-contional request + void clear_all_rows(void) { processFlags &= ~PFLAG_ALLROWS; } + int all_rows(void) const { return processFlags & PFLAG_ALLROWS; } + // apply insertValues, updateOperations to row + int update_row(RowValue &row) + { + return updateInfo == NULL ? 0 : updateInfo->Update(row); + } + // checking the condition + int compare_row(const RowValue &row, int iCmpFirstNRows = 256) const + { + return all_rows() ? 1 : conditionInfo->Compare(row, iCmpFirstNRows); + } + + // prepare an DTCResultSet, for afterward operation + int prepare_result(int start, int count); + inline int prepare_result(void) + { + return prepare_result(requestInfo.limit_start(), requestInfo.limit_count()); + } + inline int prepare_result_no_limit(void) + { + return prepare_result(0, 0); + } + + inline void detach_result_in_result_writer() + { + if (resultWriter) + resultWriter->detach_result(); + } + + // new a countonly DTCResultSet with 'nr' rows + // No extra append_row() allowed + int set_total_rows(unsigned int nr, int Force = 0) + { + if (!Force) + { + if (resultWriter || prepare_result() == 0) + return resultWriter->set_rows(nr); + } + else + { + resultWriter->set_total_rows(nr); + } + return 0; + } + void add_total_rows(int n) { resultWriter->add_total_rows(n); } + int in_range(unsigned int nr, unsigned int begin = 0) const { return resultWriter->in_range(nr, begin); } + int result_full(void) const { return resultWriter && resultWriter->is_full(); } + // append_row, from row 'r' + int append_row(const RowValue &r) { return resultWriter->append_row(r); } + int append_row(const RowValue *r) { return r ? resultWriter->append_row(*r) : 0; } + + // Append Row from DTCResultSet with condition operation + int append_result(ResultSet *rs); + // Append All Row from DTCResultSet + int pass_all_result(ResultSet *rs); + // Merge all row from sub-task + int merge_result(const DTCTask &task); + // Get Encoded Result Packet + ResultPacket *get_result_packet(void) { return (ResultPacket *)resultWriter; } + // Process Internal Results + int process_internal_result(uint32_t ts = 0); +}; + +extern int packet_body_len(PacketHeader &header); + +#endif diff --git a/src/search_local/index_storage/common/task_const.cc b/src/search_local/index_storage/common/task_const.cc new file mode 100644 index 0000000..d122756 --- /dev/null +++ b/src/search_local/index_storage/common/task_const.cc @@ -0,0 +1,177 @@ +/* + * ===================================================================================== + * + * Filename: task_const.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "task_base.h" + +// convert stage to result code +const DecodeResult DTCTask::stage2result[] = { + /*DecodeStageFatalError */ DecodeFatalError, + /*DecodeStageDataError */ DecodeDataError, + /*DecodeStageIdle */ DecodeIdle, + /*DecodeStageWaitHeader */ DecodeWaitData, + /*DecodeStageWaitData */ DecodeWaitData, + /*DecodeStageDiscard */ DecodeWaitData, + /*DecodeStageDone */ DecodeDone, +}; + +const uint32_t DTCTask::validcmds[] = { + //00000001 NOP + //00000002 RESULTCODE + //00000004 RESULTSET + //00000008 SVRADMIN + //00000010 GET + //00000020 PURGE + //00000040 INSERT + //00000080 UPDATE + //00000100 DELETE + //00000200 OBSOLETED + //00000400 OBSOLETED + //00000800 OBSOLETED + //00001000 REPLACE + //00002000 FLUSH + //00004000 Invalidate + //00008000 bit15 + 0x000071F9 /* Client --> Server/Helper */, + 0x00000006 /* Server --> Client */, + 0x00000006 /* Helper --> Server */, +}; + +const uint16_t DTCTask::cmd2type[] = { + TaskTypeAdmin, // NOP + TaskTypeAdmin, // RESULTCODE + TaskTypeAdmin, // RESULTSET + TaskTypeAdmin, // SVRADMIN + TaskTypeRead, // GET + TaskTypeAdmin, // PURGE + TaskTypeWrite, // INSERT + TaskTypeWrite, // UPDATE + TaskTypeWrite, // DELETE + TaskTypeAdmin, // OBSOLETED + TaskTypeAdmin, // OBSOLETED + TaskTypeAdmin, // OBSOLETED + // TaskTypeCommit, // REPLACE + TaskTypeWrite, // REPLACE + TaskTypeWrite, // Flush + TaskTypeWrite, // Invalidate + TaskTypeAdmin, // Monitor,此条是为了占位 + TaskTypeHelperReloadConfig, //Reload +}; + +const uint16_t DTCTask::validsections[][2] = { + //0001 VersionInfo + //0002 DataDefinition + //0004 RequestInfo + //0008 ResultInfo + //0010 UpdateInfo + //0020 ConditionInfo + //0040 FieldSet + //0080 DTCResultSet + {0x0000, 0x0005}, // NOP + {0x0008, 0x000B}, // RESULTCODE + {0x0089, 0x008B}, // RESULTSET + {0x0005, 0x0075}, // SVRADMIN + {0x0005, 0x0065}, // GET + {0x0005, 0x0025}, // PURGE + {0x0001, 0x0015}, // INSERT + {0x0015, 0x0035}, // UPDATE + {0x0005, 0x0025}, // DELETE + {0x0001, 0x0051}, // OBSOLETED + {0x0015, 0x0055}, // OBSOLETED + {0x0005, 0x0045}, // OBSOLETED + {0x0001, 0x0015}, // REPLACE + {0x0005, 0x0025}, // Flush + {0x0005, 0x0025}, // Invalidate +}; + +/* [clientkey][serverkey] */ +const uint8_t DTCTask::validktype[DField::TotalType][DField::TotalType] = { + /* -, d, u, f, s, b */ + {0, 0, 0, 0, 0, 0}, /* - */ + {0, 1, 1, 0, 0, 0}, /* d */ + {0, 1, 1, 0, 0, 0}, /* u */ + {0, 0, 0, 0, 0, 0}, /* f */ + {0, 0, 0, 0, 1, 1}, /* s */ + {0, 0, 0, 0, 1, 1}, /* b */ +}; + +#if 0 +/* [fieldtype][op] */ +const uint8_t DTCTask::validops[DField::TotalType][DField::TotalOperation] = { + /* =, +, b */ + { 0, 0, 0 }, /* - */ + { 1, 1, 1 }, /* d */ + { 1, 1, 1 }, /* u */ + { 1, 1, 0 }, /* f */ + { 1, 0, 1 }, /* s */ + { 1, 0, 1 }, /* b */ +}; +#endif + +/* [fieldtype][ valuetype] */ +const uint8_t DTCTask::validxtype[DField::TotalOperation][DField::TotalType][DField::TotalType] = { + //set + { + /* -, d, u, f, s, b */ + {0, 0, 0, 0, 0, 0}, /* - */ + {0, 1, 1, 0, 0, 0}, /* d */ + {0, 1, 1, 0, 0, 0}, /* u */ + {0, 1, 1, 1, 0, 0}, /* f */ + {0, 0, 0, 0, 1, 0}, /* s */ + {0, 0, 0, 0, 1, 1}, /* b */ + }, + //add + { + /* -, d, u, f, s, b */ + {0, 0, 0, 0, 0, 0}, /* - */ + {0, 1, 1, 0, 0, 0}, /* d */ + {0, 1, 1, 0, 0, 0}, /* u */ + {0, 1, 1, 1, 0, 0}, /* f */ + {0, 0, 0, 0, 0, 0}, /* s */ + {0, 0, 0, 0, 0, 0}, /* b */ + }, + //setbits + { + /* -, d, u, f, s, b */ + {0, 0, 0, 0, 0, 0}, /* - */ + {0, 1, 1, 0, 0, 0}, /* d */ + {0, 1, 1, 0, 0, 0}, /* u */ + {0, 0, 0, 0, 0, 0}, /* f */ + {0, 1, 1, 0, 0, 0}, /* s */ + {0, 1, 1, 0, 0, 0}, /* b */ + }, + //OR + { + /* -, d, u, f, s, b */ + {0, 0, 0, 0, 0, 0}, /* - */ + {0, 1, 1, 0, 0, 0}, /* d */ + {0, 1, 1, 0, 0, 0}, /* u */ + {0, 0, 0, 0, 0, 0}, /* f */ + {0, 0, 0, 0, 0, 0}, /* s */ + {0, 0, 0, 0, 0, 0}, /* b */ + }, +}; + +/* [fieldtype][cmp] */ +const uint8_t DTCTask::validcomps[DField::TotalType][DField::TotalComparison] = { + /* EQ,NE,LT,LE,GT,GE */ + {0, 0, 0, 0, 0, 0}, /* - */ + {1, 1, 1, 1, 1, 1}, /* d */ + {1, 1, 1, 1, 1, 1}, /* u */ + {0, 0, 0, 0, 0, 0}, /* f */ + {1, 1, 0, 0, 0, 0}, /* s */ + {1, 1, 0, 0, 0, 0}, /* b */ +}; diff --git a/src/search_local/index_storage/common/task_copy.cc b/src/search_local/index_storage/common/task_copy.cc new file mode 100644 index 0000000..2abafe5 --- /dev/null +++ b/src/search_local/index_storage/common/task_copy.cc @@ -0,0 +1,260 @@ +/* + * ===================================================================================== + * + * Filename: task_copy.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "decode.h" +#include "protocol.h" +#include "buffer_error.h" +#include "log.h" +#include "task_request.h" +#include "task_pkey.h" + +// all Copy() only copy raw request state, +// don't copy in-process data + +// vanilla Copy(), un-used yet +int DTCTask::Copy(const DTCTask &rq) +{ + //straight member-by-member copy or duplicate + TableReference::set_table_definition(rq.table_definition()); + stage = DecodeStageDone; + role = TaskRoleServer; + + versionInfo.Copy(rq.versionInfo); + requestInfo.Copy(rq.requestInfo); + resultInfo.Copy(rq.resultInfo); + + key = rq.request_key(); + requestCode = rq.requestCode; + requestType = rq.requestType; + requestFlags = rq.requestFlags; + processFlags = rq.processFlags & (PFLAG_ALLROWS | PFLAG_FIELDSETWITHKEY | PFLAG_PASSTHRU); + if (table_definition()) + table_definition()->INC(); + fieldList = rq.fieldList ? new DTCFieldSet(*rq.fieldList) : NULL; + updateInfo = rq.updateInfo ? new DTCFieldValue(*rq.updateInfo) : NULL; + conditionInfo = rq.conditionInfo ? new DTCFieldValue(*rq.conditionInfo) : NULL; + + return 0; +} + +// vanilla Copy(), un-used yet +int TaskRequest::Copy(const TaskRequest &rq) +{ + DTCTask::Copy(rq); // always success + expire_time = rq.expire_time; + timestamp = rq.timestamp; + + // don't use generic build_packed_key, copy manually + barHash = rq.barHash; + // Dup multi-fields key + if (rq.multiKey) + { + multiKey = (DTCValue *)MALLOC(key_fields() * sizeof(DTCValue)); + if (multiKey == NULL) + throw -ENOMEM; + memcpy(multiKey, rq.multiKey, key_fields() * sizeof(DTCValue)); + } + // Dup packed key + if (rq.packedKey) + { + int pksz = TaskPackedKey::packed_key_size(packedKey, key_format()); + packedKey = (char *)MALLOC(pksz); + if (packedKey == NULL) + throw -ENOMEM; + memcpy(packedKey, rq.packedKey, pksz); + } + // Copy internal API batch key list + keyList = rq.internal_key_val_list(); + return 0; +} + +// only for batch request spliting +int DTCTask::Copy(const DTCTask &rq, const DTCValue *newkey) +{ + // copy non-field depend informations, see above Copy() variant + TableReference::set_table_definition(rq.table_definition()); + stage = DecodeStageDone; + role = TaskRoleServer; + versionInfo.Copy(rq.versionInfo); + //如果是批量请求，拷贝批量task的versioninfo之后记得强制set一下keytype + //因为有些老的api发过来的请求没有设置全局的keytype + //这样批量的时候可能会在heper端出现-2024错误 + if (((TaskRequest *)&rq)->is_batch_request()) + { + if (table_definition()) + versionInfo.set_key_type(table_definition()->key_type()); + else + log_notice("table_definition() is null.please check it"); + } + requestInfo.Copy(rq.requestInfo); + resultInfo.Copy(rq.resultInfo); + key = newkey; + requestCode = rq.requestCode; + requestType = rq.requestType; + requestFlags = rq.requestFlags & ~DRequest::Flag::MultiKeyValue; + processFlags = rq.processFlags & (PFLAG_ALLROWS | PFLAG_FIELDSETWITHKEY | PFLAG_PASSTHRU); + if (table_definition()) + table_definition()->INC(); + + // Need() field list always same + fieldList = rq.fieldList ? new DTCFieldSet(*rq.fieldList) : NULL; + + // primary key set to first key component + requestInfo.set_key(newkey[0]); + + // k1 means key field number minus 1 + const int k1 = rq.key_fields() - 1; + if (k1 <= 0) + { + // single-field key, copy straight + updateInfo = rq.updateInfo ? new DTCFieldValue(*rq.updateInfo) : NULL; + conditionInfo = rq.conditionInfo ? new DTCFieldValue(*rq.conditionInfo) : NULL; + } + else if (rq.requestCode == DRequest::Insert || rq.requestCode == DRequest::Replace) + { + // multi-field key, insert or replace + conditionInfo = rq.conditionInfo ? new DTCFieldValue(*rq.conditionInfo) : NULL; + + // enlarge update information, add non-primary key using Set() operator + updateInfo = rq.updateInfo ? new DTCFieldValue(*rq.updateInfo, k1) : new DTCFieldValue(k1); + for (int i = 1; i <= k1; i++) + { // last one is k1 + updateInfo->add_value(i, DField::Set, /*field_type(i)*/ DField::Signed, newkey[i]); + // FIXME: server didn't support non-integer multi-field key + } + } + else + { + // multi-field key, other commands + updateInfo = rq.updateInfo ? new DTCFieldValue(*rq.updateInfo) : NULL; + + // enlarge condition information, add non-primary key using EQ() comparator + conditionInfo = rq.conditionInfo ? new DTCFieldValue(*rq.conditionInfo, k1) : new DTCFieldValue(k1); + for (int i = 1; i <= k1; i++) + { // last one is k1 + conditionInfo->add_value(i, DField::EQ, /*field_type(i)*/ DField::Signed, newkey[i]); + // FIXME: server didn't support non-integer multi-field key + } + } + + return 0; +} + +int TaskRequest::Copy(const TaskRequest &rq, const DTCValue *newkey) +{ + DTCTask::Copy(rq, newkey); // always success + expire_time = rq.expire_time; + timestamp = rq.timestamp; + // splited, now NOT batch request + // re-calculate packed and barrier key + if (request_code() != DRequest::SvrAdmin) + { + if (build_packed_key() < 0) + return -1; // ERROR + calculate_barrier_key(); + } + return 0; +} + +// only for commit row, replace whole row +int DTCTask::Copy(const RowValue &r) +{ + TableReference::set_table_definition(r.table_definition()); + stage = DecodeStageDone; + role = TaskRoleServer; + requestCode = DRequest::Replace; + requestType = cmd2type[requestCode]; + requestFlags = DRequest::Flag::KeepAlive; + processFlags = PFLAG_ALLROWS | PFLAG_FIELDSETWITHKEY; + + const int n = r.num_fields(); + + // dup strings&binary field value to packetbuf + DTCValue row[n + 1]; + int len = 0; + for (int i = 0; i <= n; i++) + { + int t = r.field_type(i); + if (t == DField::String || t == DField::Binary) + len += r[i].bin.len + 1; + } + char *p = packetbuf.Allocate(len, role); + for (int i = 0; i <= n; i++) + { + int t = r.field_type(i); + if (t == DField::String || t == DField::Binary) + { + int l = r[i].bin.len; + row[i].bin.len = l; + row[i].bin.ptr = p; + memcpy(p, r[i].bin.ptr, l); + p[l] = 0; + p += l + 1; + } + else + { + row[i] = r[i]; + } + } + + // tablename & hash + versionInfo.set_table_name(table_name()); + versionInfo.set_table_hash(table_hash()); + versionInfo.set_serial_nr(0); + versionInfo.set_tag(9, key_type()); + + // key + requestInfo.set_key(row[0]); + // cmd + requestInfo.set_tag(1, DRequest::Replace); + + // a replace command with all fields set to desired value + updateInfo = new DTCFieldValue(n); + for (int i = 1; i <= n; i++) + updateInfo->add_value(i, DField::Set, + //api cast all uint to long long,and set fieldtype to uint + //so here should be consistent + (r.field_type(i) == DField::Unsigned) ? DField::Signed : r.field_type(i), + row[i]); + + // first field always is key + // bug fix, key should not point to local row[0] + key = requestInfo.Key(); + ; + + return 0; +} + +int TaskRequest::Copy(const RowValue &row) +{ + DTCTask::Copy(row); // always success + + // ALWAYS a replace request + // calculate packed and barrier key + if (build_packed_key() < 0) + return -1; // ERROR + calculate_barrier_key(); + return 0; +} diff --git a/src/search_local/index_storage/common/task_multiplexer.cc b/src/search_local/index_storage/common/task_multiplexer.cc new file mode 100644 index 0000000..f993d9e --- /dev/null +++ b/src/search_local/index_storage/common/task_multiplexer.cc @@ -0,0 +1,142 @@ +/* + * ===================================================================================== + * + * Filename: task_multiplexer.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "task_request.h" +#include "task_multiplexer.h" +#include "log.h" + +class ReplyMultiplexer : public ReplyDispatcher +{ +public: + ReplyMultiplexer(void) {} + virtual ~ReplyMultiplexer(void); + virtual void reply_notify(TaskRequest *task); +}; + +ReplyMultiplexer::~ReplyMultiplexer(void) {} + +void ReplyMultiplexer::reply_notify(TaskRequest *cur) +{ + log_debug("ReplyMultiplexer::reply_notify start"); + MultiRequest *req = cur->get_batch_key_list(); + /* reset BatchKey state */ + cur->set_batch_cursor(-1); + if (cur->result_code() < 0) + { + req->second_pass(-1); + } + else if (req->remain_count() <= 0) + { + req->second_pass(0); + } + else if (req->split_task() != 0) + { + log_error("split task error"); + cur->set_error(-ENOMEM, __FUNCTION__, "split task error"); + req->second_pass(-1); + } + else + { + req->second_pass(0); + } + log_debug("ReplyMultiplexer::reply_notify end"); +} + +static ReplyMultiplexer replyMultiplexer; + +TaskMultiplexer::~TaskMultiplexer(void) +{ +} + +void TaskMultiplexer::task_notify(TaskRequest *cur) +{ + log_debug("TaskMultiplexer::task_notify start, flag_multi_key_val: %d", cur->flag_multi_key_val()); + + if (!cur->flag_multi_key_val()) + { + // single task, no dispatcher needed + output.task_notify(cur); + return; + } + +#if 0 + // multi-task + if(cur->flag_pass_thru()){ + log_error("multi-task not support under pass-through mode"); + cur->set_error(-EC_TOO_MANY_KEY_VALUE, __FUNCTION__, "multi-task not support under pass-through mode"); + cur->reply_notify(); + return; + } +#endif + + switch (cur->request_code()) + { + case DRequest::Insert: + case DRequest::Replace: + if (cur->table_definition()->has_auto_increment()) + { + log_error("table has autoincrement field, multi-insert/replace not support"); + cur->set_error(-EC_TOO_MANY_KEY_VALUE, __FUNCTION__, "table has autoincrement field, multi-insert/replace not support"); + cur->reply_notify(); + return; + } + break; + + case DRequest::Get: + if (cur->requestInfo.limit_start() != 0 || cur->requestInfo.limit_count() != 0) + { + log_error("multi-task not support limit()"); + cur->set_error(-EC_TOO_MANY_KEY_VALUE, __FUNCTION__, "multi-task not support limit()"); + cur->reply_notify(); + return; + } + case DRequest::Delete: + case DRequest::Purge: + case DRequest::Update: + case DRequest::Flush: + break; + + default: + log_error("multi-task not support other than get/insert/update/delete/purge/replace/flush request"); + cur->set_error(-EC_TOO_MANY_KEY_VALUE, __FUNCTION__, "bad batch request type"); + cur->reply_notify(); + return; + } + + MultiRequest *req = new MultiRequest(this, cur); + if (req == NULL) + { + log_error("new MultiRequest error: %m"); + cur->set_error(-ENOMEM, __FUNCTION__, "new MultiRequest error"); + cur->reply_notify(); + return; + } + + if (req->decode_key_list() <= 0) + { + /* empty batch or errors */ + cur->reply_notify(); + delete req; + return; + } + + cur->set_batch_key_list(req); + replyMultiplexer.reply_notify(cur); + + log_debug("TaskMultiplexer::task_notify end"); + return; +} diff --git a/src/search_local/index_storage/common/task_multiplexer.h b/src/search_local/index_storage/common/task_multiplexer.h new file mode 100644 index 0000000..c09c906 --- /dev/null +++ b/src/search_local/index_storage/common/task_multiplexer.h @@ -0,0 +1,42 @@ +/* + * ===================================================================================== + * + * Filename: task_multiplexer.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_TASK_MULTIPLEXER_H__ +#define __H_TASK_MULTIPLEXER_H__ + +#include "request_base.h" +#include "multi_request.h" + +class TaskRequest; + +class TaskMultiplexer : public TaskDispatcher +{ +public: + TaskMultiplexer(PollThread *o) : TaskDispatcher(o), output(o), fastUpdate(0){}; + virtual ~TaskMultiplexer(void); + + void bind_dispatcher(TaskDispatcher *p) { output.bind_dispatcher(p); } + void push_task_queue(TaskRequest *req) { output.indirect_notify(req); } + void enable_fast_update(void) { fastUpdate = 1; } + +private: + RequestOutput output; + virtual void task_notify(TaskRequest *); + int fastUpdate; +}; + +#endif diff --git a/src/search_local/index_storage/common/task_pkey.cc b/src/search_local/index_storage/common/task_pkey.cc new file mode 100644 index 0000000..96491bb --- /dev/null +++ b/src/search_local/index_storage/common/task_pkey.cc @@ -0,0 +1,116 @@ +/* + * ===================================================================================== + * + * Filename: task_pkey.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "value.h" +#include "table_def.h" +#include "task_pkey.h" + +void TaskPackedKey::unpack_key(const DTCTableDefinition *tdef, const char *ptrKey, DTCValue *val) +{ + if (tdef->key_format() == 0) + { + val->bin.ptr = (char *)ptrKey + 1; + val->bin.len = *(unsigned char *)ptrKey; + } + else + { + for (int i = 0; i < tdef->key_fields(); val++, ptrKey += tdef->field_size(i), i++) + { + const int size = tdef->field_size(i); +#if __BYTE_ORDER == __LITTLE_ENDIAN + val->s64 = tdef->field_type(i) == DField::Unsigned ? 0 : ptrKey[size - 1] & 0x80 ? -1 : 0; + memcpy((char *)&val->s64, ptrKey, size); +#elif __BYTE_ORDER == __BIG_ENDIAN + val->s64 = tdef->field_type(i) == DField::Unsigned ? 0 : ptrKey[0] & 0x80 ? -1 : 0; + memcpy((char *)&val->s64 + (sizeof(uint64_t) - size), ptrKey, size); +#else +#error unkown endian +#endif + } + } +} + +int TaskPackedKey::build_packed_key(const DTCTableDefinition *tdef, const DTCValue *pstKeyValues, unsigned int uiBufSize, char packKey[]) +{ + if (tdef->key_format() == 0) + { // single string key + if (pstKeyValues->str.len > (int)tdef->field_size(0)) + return -1; + if ((int)uiBufSize < pstKeyValues->str.len + 1) + return -2; + if (tdef->field_type(0) == DField::String) + { + packKey[0] = pstKeyValues->str.len; + for (int i = 0; i < pstKeyValues->str.len; i++) + packKey[i + 1] = INTERNAL_TO_LOWER(pstKeyValues->str.ptr[i]); + } + else + { + packKey[0] = pstKeyValues->str.len; + memcpy(packKey + 1, pstKeyValues->str.ptr, pstKeyValues->str.len); + } + } + else if (tdef->key_fields() == 1) + { // single int key + if (uiBufSize < (unsigned int)(tdef->key_format())) + return -3; + + if (store_value(packKey, pstKeyValues->u64, + tdef->field_type(0), + tdef->field_size(0)) < 0) + { + return -4; + } + } + else + { // multi-int key + if ((int)uiBufSize < tdef->key_format()) + return -5; + + for (int i = 0; i < tdef->key_fields(); i++) + { + if (store_value(packKey + tdef->field_offset(i), + (pstKeyValues + i)->u64, + tdef->field_type(i), + tdef->field_size(i)) < 0) + { + return -6; + } + } + } + + return 0; +} + +unsigned long TaskPackedKey::calculate_barrier_key(const DTCTableDefinition *tdef, const char *pkey) +{ + unsigned const char *p = (unsigned const char *)pkey; + int len = tdef->key_format(); + if (len == 0) + len = *p++; + + unsigned long h = 0; + unsigned char c; + while (len-- > 0) + { + c = *p++; + h = h * 11111 + (c << 4) + (c >> 4); + } + return h; +} diff --git a/src/search_local/index_storage/common/task_pkey.h b/src/search_local/index_storage/common/task_pkey.h new file mode 100644 index 0000000..83b33ba --- /dev/null +++ b/src/search_local/index_storage/common/task_pkey.h @@ -0,0 +1,84 @@ +/* + * ===================================================================================== + * + * Filename: task_pkey.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_TASK_PKEY_H__ +#define __H_DTC_TASK_PKEY_H__ + +#include + +class DTCTableDefinition; +union DTCValue; + +class TaskPackedKey +{ +public: + static int packed_key_size(const char *p, int keyformat) { return keyformat ?: 1 + *(uint8_t *)p; } + static void unpack_key(const DTCTableDefinition *, const char *, DTCValue *); + static int store_value(char *p, uint64_t val, int type, int size) + { + static uint64_t intoff[16] = { + 0x0000000000000080ULL, // 1s + 0x0000000000008000ULL, // 2s + 0x0000000000800000ULL, // 3s + 0x0000000080000000ULL, // 4s + 0x0000008000000000ULL, // 5s + 0x0000800000000000ULL, // 6s + 0x0080000000000000ULL, // 7s + 0x0000000000000000ULL, // 8s + 0x0000000000000000ULL, // 1u + 0x0000000000000000ULL, // 2u + 0x0000000000000000ULL, // 3u + 0x0000000000000000ULL, // 4u + 0x0000000000000000ULL, // 5u + 0x0000000000000000ULL, // 6u + 0x0000000000000000ULL, // 7u + 0x0000000000000000ULL, // 8u + }; + + static uint64_t intmask[8] = { + 0xFFFFFFFFFFFFFF00ULL, // 1 + 0xFFFFFFFFFFFF0000ULL, // 2 + 0xFFFFFFFFFF000000ULL, // 3 + 0xFFFFFFFF00000000ULL, // 4 + 0xFFFFFF0000000000ULL, // 5 + 0xFFFF000000000000ULL, // 6 + 0xFF00000000000000ULL, // 7 + 0x0000000000000000ULL, // 8 + }; + + if (((val + intoff[type * 8 + size - 9]) & intmask[size - 1]) != 0) + return -1; // overflow; +#if __BYTE_ORDER == __LITTLE_ENDIAN + if (size == 4) + *(uint32_t *)p = val; + else if (size == 8) + *(uint64_t *)p = val; + else + memcpy(p, &val, size); +#elif __BYTE_ORDER == __BIG_ENDIAN + memcpy(p, (char *)&val + (sizeof(uint64_t) - size), size); +#else +#error unkown endian +#endif + return 0; + } + + static int build_packed_key(const DTCTableDefinition *, const DTCValue *, unsigned int bufsz, char packKey[]); + static unsigned long calculate_barrier_key(const DTCTableDefinition *tdef, const char *pkey); +}; + +#endif diff --git a/src/search_local/index_storage/common/task_request.cc b/src/search_local/index_storage/common/task_request.cc new file mode 100644 index 0000000..29d4c34 --- /dev/null +++ b/src/search_local/index_storage/common/task_request.cc @@ -0,0 +1,532 @@ +/* + * ===================================================================================== + * + * Filename: task_request.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "mem_check.h" +#include "multi_request.h" +#include "task_request.h" +#include "task_pkey.h" +#include "poll_thread.h" +#include "log.h" +#include "compiler.h" +#include "key_list.h" +#include "proxy_multi_request.h" +#include "proxy_client.h" + +TaskRequest::~TaskRequest() +{ + free_packed_key(); + + if (agentMultiReq) + { + delete agentMultiReq; + agentMultiReq = NULL; + } + + if (recvBuff) + { + free(recvBuff); + recvBuff = NULL; + } +} + +void TaskRequest::Clean() +{ + blacklist_size = 0; + timestamp = 0; + barHash = 0; + expire_time = 0; + keyList = NULL; + batchKey = NULL; + DTCTask::Clean(); + TaskOwnerInfo::Clean(); +} + +int TaskRequest::build_packed_key(void) +{ + + if (unlikely(key_format() == 0)) + return build_single_string_key(); + if (likely(key_fields() == 1)) + return build_single_int_key(); + else + return build_multi_int_key(); +} + +int TaskRequest::build_single_string_key(void) +{ + DTCValue empty(""); + const DTCValue *key = request_key(); + if (key == NULL) + key = ∅ + + if (key->str.len > field_size(0)) + { + set_error(-EC_KEY_NEEDED, "key verify", "string key size overflow"); + return -1; + } + + FREE_IF(packedKey); + packedKey = (char *)MALLOC(key->str.len + 1); + if (packedKey == NULL) + { + set_error(-EC_SERVER_ERROR, "key verify", "no enough memory"); + return -1; + } + + int i; + if (field_type(0) == DField::String) + { + *packedKey = key->str.len; + for (i = 0; i < key->str.len; i++) + packedKey[i + 1] = INTERNAL_TO_LOWER(key->str.ptr[i]); + log_debug("key type[string]"); + } + else + { + *packedKey = key->str.len; + memcpy(packedKey + 1, key->str.ptr, key->str.len); + log_debug("key type[binary]"); + } + return 0; +} + +int TaskRequest::build_single_int_key(void) +{ + if (unlikely(key_format() > 8)) + { + FREE_IF(packedKey); + packedKey = (char *)calloc(1, key_format()); + if (packedKey == NULL) + { + set_error(-EC_SERVER_ERROR, "key verify", "no enough memory"); + return -1; + } + } + else + { + packedKey = packedKeyBuf; + memset(packedKey, 0, key_format()); + } + + const DTCValue *key = request_key(); + if (likely(key != NULL)) + { + if (unlikely(TaskPackedKey::store_value(packedKey, key->u64, field_type(0), field_size(0)) < 0)) + { + set_error(-EC_KEY_NEEDED, "key verify", "int key value overflow"); + return -1; + } + } + else if (!(key_auto_increment() && request_code() == DRequest::Insert)) + { + set_error(-EC_KEY_NEEDED, "key verify", "No Key Sepcified"); + return -1; + } + return 0; +} + +int TaskRequest::build_multi_key_values() +{ + uint32_t keymask = 0; + FREE_IF(multiKey); + multiKey = (DTCValue *)calloc(sizeof(DTCValue), key_fields()); + if (multiKey == NULL) + { + set_error(-ENOMEM, "key verify", "NO ENOUGH MEMORY"); + return -1; + } + + if (key != NULL) + { + multiKey[0] = *key; + keymask |= 1; + } + + const DTCFieldValue *kop = request_code() == DRequest::Insert || request_code() == DRequest::Replace ? updateInfo : conditionInfo; + + if (kop == NULL) + { + set_error(-EC_KEY_NEEDED, "key verify", "Missing Key Component"); + return -1; + } + for (int i = 0; i < kop->num_fields(); i++) + { + const int id = kop->field_id(i); + /* NOT KEY */ + if (id >= key_fields()) + continue; + + if ((keymask & (1 << id)) != 0) + { + set_error(-EC_BAD_MULTIKEY, "key verify", "duplicate key field"); + return -1; + } + + keymask |= 1 << id; + const int vt = kop->field_type(i); + DTCValue *val = kop->field_value(i); + // the operation must be EQ/Set, already checked in task_base.cc + if (field_size(id) < 8 && val->s64 < 0 && + ((field_type(id) == DField::Unsigned && vt == DField::Signed) || + (field_type(id) == DField::Signed && vt == DField::Unsigned))) + { + set_error(-EC_KEY_OVERFLOW, "key verify", "int key value overflow"); + return -1; + } + multiKey[id] = *val; + } + + const int ai = auto_increment_field_id(); + if (ai >= 0 && ai < key_fields() && request_code() == DRequest::Insert) + keymask |= 1U << ai; + //keymask |= 1U; + + if ((keymask + 1) != (1U << key_fields())) + { + set_error(-EC_KEY_NEEDED, "key verify", "Missing Key Component"); + return -1; + } + return 0; +} + +int TaskRequest::build_multi_int_key(void) +{ + /* Build first key field */ + if (build_single_int_key() < 0) + return -1; + + log_debug("TaskRequest::build_multi_int_key: flag_multi_key_val: %d", flag_multi_key_val()); + if (flag_multi_key_val() == 0) + { + // NOTICE, in batch request, build_packed_key only called by set_batch_cursor + // which already set the corresponding multiKey member + if (build_multi_key_values() < 0) + return -1; + } + + for (int i = 1; i < key_fields(); i++) + { + if (TaskPackedKey::store_value(packedKey + field_offset(i), multiKey[i].u64, + field_type(i), field_size(i)) < 0) + { + set_error(-EC_KEY_OVERFLOW, "key verify", "int key value overflow"); + return -1; + } + } + return 0; +} + +int TaskRequest::update_packed_key(uint64_t val) +{ + int id = auto_increment_field_id(); + if (id >= 0 && id < key_fields()) + { + if (multiKey) + multiKey[id].u64 = val; + if (TaskPackedKey::store_value(packedKey + field_offset(id), val, field_type(id), field_size(id)) < 0) + { + set_error(-EC_KEY_NEEDED, "key verify", "int key value overflow"); + return -1; + } + } + return 0; +} + +void TaskRequest::calculate_barrier_key(void) +{ + if (packedKey) + { + barHash = TaskPackedKey::calculate_barrier_key(table_definition(), packedKey); + } +} + +int TaskRequest::get_batch_size(void) const +{ + return batchKey != NULL ? batchKey->total_count() : 0; +} + +void TaskRequest::dump_update_info(const char *prefix) const +{ + /* for debug */ + return; + + char buff[1024]; + buff[0] = 0; + char *s = buff; + + const DTCFieldValue *p = request_operation(); + if (!p) + return; + + for (int i = 0; i < p->num_fields(); i++) + { + //only dump setbits operations + if (p->field_operation(i) == DField::SetBits) + { + uint64_t f_v = p->field_value(i)->u64; + + const int len = 8; + unsigned int off = f_v >> 32; + unsigned int size = off >> 24; + off &= 0xFFFFFF; + unsigned int value = f_v & 0xFFFFFFFF; + + if (off >= 8 * len || size == 0) + break; + if (size > 32) + size = 32; + if (size > 8 * len - off) + size = 8 * len - off; + + s += snprintf(s, sizeof(buff) - (s - buff), " [%d %d]", off, value); + } + } + + if (s != buff) + log_debug("[%s]%u%s", prefix, int_key(), buff); + + return; +} + +#define ERR_RET(ret_msg, fmt, args...) \ + do \ + { \ + set_error(err, "decoder", ret_msg); \ + log_debug(fmt, ##args); \ + return -1; \ + } while (0) +int TaskRequest::prepare_process(void) +{ + int err; + + if (1) + { + /* timeout present */ + int client_timeout = requestInfo.tag_present(1) == 0 ? default_expire_time() : requestInfo.get_expire_time(versionInfo.CTLibIntVer()); + + log_debug("client api set timeout %d ms", client_timeout); + struct timeval now; + gettimeofday(&now, NULL); + + responseTimer = (int)(now.tv_sec * 1000000ULL + now.tv_usec); + expire_time = now.tv_sec * 1000ULL + now.tv_usec / 1000 + client_timeout; + timestamp = now.tv_sec; + } + + if (0) + { +#if 0 + // internal API didn't call PreparePrcess() !!! + // move checker to task_api.cc +#endif + } + else if (flag_multi_key_val()) + { + // batch key present + if (!versionInfo.tag_present(10) || !versionInfo.tag_present(11) || !versionInfo.tag_present(12) || !versionInfo.tag_present(13)) + { + err = -EC_KEY_NEEDED; + ERR_RET("require key field info", "require key field info: %d", err); + } + if ((int)(versionInfo.get_tag(10)->u64) != key_fields()) + { + err = -EC_KEY_NEEDED; + ERR_RET("key field count incorrect", "key field count incorrect. request: %d, table: %d", (int)(versionInfo.get_tag(10)->u64), key_fields()); + } + if (versionInfo.get_tag(11)->u64 < 1) + { + err = -EC_KEY_NEEDED; + ERR_RET("require key value", "require key value"); + } + Array keyTypeList(versionInfo.get_tag(12)->bin); + Array keyNameList(versionInfo.get_tag(13)->bin); + uint8_t keyType; + DTCBinary keyName; + for (unsigned int i = 0; i < versionInfo.get_tag(10)->u64; i++) + { + if (keyTypeList.Get(keyType) != 0 || keyNameList.Get(keyName) != 0) + { + err = -EC_KEY_NEEDED; + ERR_RET("require key field info", "require key field info: %d", err); + } + int fid = field_id(keyName.ptr); + if (fid < 0 || fid >= key_fields()) + { + err = -EC_BAD_FIELD_NAME; + ERR_RET("bad key field name", "bad key field name: %s", keyName.ptr); + } + int tabKeyType = field_type(fid); + if (keyType >= DField::TotalType || !validktype[keyType][tabKeyType]) + { + err = -EC_BAD_KEY_TYPE; + ERR_RET("key type incorrect", "key[%s] type[%d] != table-def[%d]", keyName.ptr, keyType, field_type(field_id(keyName.ptr))); + } + } + } + else if (request_code() == DRequest::SvrAdmin) + { + // admin requests + // Migrate命令需要在barrier里排队，所以需要校验是否有key以及计算hash + if (DRequest::ServerAdminCmd::Migrate == requestInfo.admin_code()) + { + const DTCFieldValue *condition = request_condition(); + if (condition == 0 || condition->num_fields() <= 0 || condition->field_value(0) == 0) + { + err = -EC_KEY_NEEDED; + ERR_RET("migrate commond", "need set migrate key"); + } + else if (condition->num_fields() > 1) + { + err = -EC_SERVER_ERROR; + ERR_RET("migrate commond", "not support bulk migrate now"); + } + const DTCValue *key = condition->field_value(0); + + if (key->str.len > 8) + { + if (packedKey != packedKeyBuf && packedKey != 0) + { + free(packedKey); + packedKey = NULL; + } + packedKey = (char *)calloc(1, key->str.len); + if (packedKey == NULL) + { + set_error(-EC_SERVER_ERROR, "key verify", "no enough memory"); + return -1; + } + } + else + { + packedKey = packedKeyBuf; + memset(packedKey, 0, key_format()); + } + + if (packedKey == NULL) + { + set_error(-EC_SERVER_ERROR, "key verify", "no enough memory"); + return -1; + } + memcpy(packedKey, key->bin.ptr, key->bin.len); + + calculate_barrier_key(); + } + } + else + { + // single key request + err = build_packed_key(); + if (err < 0) + return err; + + calculate_barrier_key(); + } + + return 0; +} + +/* for agent request */ +int TaskRequest::decode_agent_request() +{ + if (agentMultiReq) + { + delete agentMultiReq; + agentMultiReq = NULL; + } + + agentMultiReq = new AgentMultiRequest(this); + if (agentMultiReq == NULL) + { + log_crit("no mem new AgentMultiRequest"); + return -1; + } + + pass_recved_result_to_agent_muti_req(); + + if (agentMultiReq->decode_agent_request() < 0) + { + log_error("agent multi request decode error"); + return -1; + } + + return 0; +} + +void TaskRequest::pass_recved_result_to_agent_muti_req() +{ + agentMultiReq->save_recved_result(recvBuff, recvLen, recvPacketCnt); + recvBuff = NULL; + recvLen = recvPacketCnt = 0; +} + +bool TaskRequest::is_agent_request_completed() +{ + return agentMultiReq->is_completed(); +} + +void TaskRequest::done_one_agent_sub_request() +{ + AgentMultiRequest *ownerReq = OwnerInfo(); + if (ownerReq) + ownerReq->complete_task(owner_index()); + else + delete this; +} + +void TaskRequest::link_to_owner_client(ListObject &head) +{ + if (ownerClient && agentMultiReq) + agentMultiReq->ListAdd(head); +} + +void TaskRequest::set_owner_client(ClientAgent *client) +{ + ownerClient = client; +} + +ClientAgent *TaskRequest::owner_client() +{ + return ownerClient; +} + +void TaskRequest::clear_owner_client() +{ + ownerClient = NULL; +} + +int TaskRequest::agent_sub_task_count() +{ + return agentMultiReq->packet_count(); +} + +bool TaskRequest::is_curr_sub_task_processed(int index) +{ + return agentMultiReq->is_curr_task_processed(index); +} + +TaskRequest *TaskRequest::curr_agent_sub_task(int index) +{ + return agentMultiReq->curr_task(index); +} + +void TaskRequest::copy_reply_for_agent_sub_task() +{ + agentMultiReq->copy_reply_for_sub_task(); +} diff --git a/src/search_local/index_storage/common/task_request.h b/src/search_local/index_storage/common/task_request.h new file mode 100644 index 0000000..004b617 --- /dev/null +++ b/src/search_local/index_storage/common/task_request.h @@ -0,0 +1,263 @@ +/* + * ===================================================================================== + * + * Filename: task_request.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_REQUEST_REAL_H__ +#define __H_DTC_REQUEST_REAL_H__ + +#include "request_base_all.h" +#include "task_base.h" +#include "stop_watch.h" +#include "hotback_task.h" +class DecoderUnit; +class MultiRequest; +class NCKeyValueList; +class NCRequest; +class AgentMultiRequest; +class ClientAgent; + +class TaskOwnerInfo +{ +private: + const struct sockaddr *clientaddr; + void *volatile ownerInfo; + int ownerIndex; + +public: + TaskOwnerInfo(void) : clientaddr(NULL), ownerInfo(NULL), ownerIndex(0) {} + virtual ~TaskOwnerInfo(void) {} + + void set_owner_info(void *info, int idx, const struct sockaddr *addr) + { + ownerInfo = info; + ownerIndex = idx; + clientaddr = addr; + } + + inline void Clean() { ownerInfo = NULL; } + + void clear_owner_info(void) { ownerInfo = NULL; } + + template + T *OwnerInfo(void) const { return (T *)ownerInfo; } + + const struct sockaddr *OwnerAddress(void) const { return clientaddr; } + int owner_index(void) const { return ownerIndex; } +}; + +class TaskRequest : public DTCTask, public TaskReplyList, public TaskOwnerInfo +{ +public: + TaskRequest(DTCTableDefinition *t = NULL) : DTCTask(t, TaskRoleServer), + blacklist_size(0), + timestamp(0), + barHash(0), + packedKey(NULL), + expire_time(0), + multiKey(NULL), + keyList(NULL), + batchKey(NULL), + agentMultiReq(NULL), + ownerClient(NULL), + recvBuff(NULL), + recvLen(0), + recvPacketCnt(0), + resourceId(0), + resourceOwner(NULL), + resourceSeq(0){}; + + virtual ~TaskRequest(); + + inline TaskRequest(const TaskRequest &rq) + { + TaskRequest(); + Copy(rq); + } + + int Copy(const TaskRequest &rq); + int Copy(const TaskRequest &rq, const DTCValue *newkey); + int Copy(const RowValue &); + int Copy(NCRequest &, const DTCValue *); + +public: + void Clean(); + // msecond: absolute ms time + uint64_t default_expire_time(void) { return 5000 /*default:5000ms*/; } + const uint64_t get_expire_time(void) const { return expire_time; } + + int is_expired(uint64_t now) const + { + // client gone, always expired + if (OwnerInfo() == NULL) + return 1; + // flush cmd never time out + if (requestType == TaskTypeCommit) + return 0; + return expire_time <= now; + } + uint32_t Timestamp(void) const { return timestamp; } + void renew_timestamp(void) { timestamp = time(NULL); } + + const char *packed_key(void) const { return packedKey; } + unsigned long barrier_key(void) const { return barHash; } + DTCValue *multi_key_array(void) { return multiKey; } + +public: + int build_packed_key(void); + void calculate_barrier_key(void); + + int prepare_process(void); + int update_packed_key(uint64_t); + void push_black_list_size(const unsigned size) { blacklist_size = size; } + unsigned pop_black_list_size(void) + { + register unsigned ret = blacklist_size; + blacklist_size = 0; + return ret; + } + + void dump_update_info(const char *prefix) const; + void update_key(RowValue &r) + { + if (multiKey) + r.copy_value(multiKey, 0, key_fields()); + else + r[0] = *request_key(); + } + void update_key(RowValue *r) + { + if (r) + update_key(*r); + } + int update_row(RowValue &row) + { + row.update_timestamp(timestamp, + requestCode != DRequest::Update || + (updateInfo && updateInfo->has_type_commit())); + return DTCTask::update_row(row); + } + + void set_remote_addr(const char *addr) + { + strncpy(remoteAddr, addr, sizeof(remoteAddr)); + remoteAddr[sizeof(remoteAddr) - 1] = 0; + } + + const char *remote_addr() { return remoteAddr; } + HotBackTask &get_hot_back_task() + { + return hotbacktask; + } + +private: + /* following filed should be clean: + * blacklist_size + * timestamp + * barHash + * expire_time + * + * */ + /* 加入黑名单的大小 */ + unsigned blacklist_size; + uint32_t timestamp; + + unsigned long barHash; + char *packedKey; + char packedKeyBuf[8]; + uint64_t expire_time; /* ms */ /* derived from packet */ + DTCValue *multiKey; + char remoteAddr[32]; + +private: + int build_multi_key_values(void); + int build_single_string_key(void); + int build_single_int_key(void); + int build_multi_int_key(void); + + void free_packed_key(void) + { + if (packedKey && packedKey != packedKeyBuf) + FREE_CLEAR(packedKey); + FREE_IF(multiKey); + } + /* for batch request*/ +private: + const NCKeyValueList *keyList; + /* need clean when task begin in use(deleted when batch request finished) */ + MultiRequest *batchKey; + + /* for agent request */ +private: + AgentMultiRequest *agentMultiReq; + ClientAgent *ownerClient; + char *recvBuff; + int recvLen; + int recvPacketCnt; + +public: + unsigned int key_val_count() const { return versionInfo.get_tag(11) ? versionInfo.get_tag(11)->s64 : 1; } + const Array *key_type_list() const { return (Array *)&(versionInfo.get_tag(12)->bin); } + const Array *key_name_list() const { return (Array *)&(versionInfo.get_tag(13)->bin); } + const Array *key_val_list() const { return (Array *)&(requestInfo.Key()->bin); } + const NCKeyValueList *internal_key_val_list() const { return keyList; } + + int is_batch_request(void) const { return batchKey != NULL; } + int get_batch_size(void) const; + void set_batch_key_list(MultiRequest *bk) { batchKey = bk; } + MultiRequest *get_batch_key_list(void) { return batchKey; } + int set_batch_cursor(int i); + void done_batch_cursor(int i); + +public: + /* for agent request */ + void set_owner_client(ClientAgent *client); + ClientAgent *owner_client(); + void clear_owner_client(); + + int decode_agent_request(); + inline void save_recved_result(char *buff, int len, int pktcnt) + { + recvBuff = buff; + recvLen = len; + recvPacketCnt = pktcnt; + } + bool is_agent_request_completed(); + void done_one_agent_sub_request(); + + void link_to_owner_client(ListObject &head); + + int agent_sub_task_count(); + void copy_reply_for_agent_sub_task(); + TaskRequest *curr_agent_sub_task(int index); + + bool is_curr_sub_task_processed(int index); + +private: + void pass_recved_result_to_agent_muti_req(); + +public: // timing + stopwatch_usec_t responseTimer; + void response_timer_start(void) { responseTimer.start(); } + unsigned int resourceId; + DecoderUnit *resourceOwner; + uint32_t resourceSeq; + +private: + /*use as async hotback task*/ + HotBackTask hotbacktask; +}; + +#endif diff --git a/src/search_local/index_storage/common/task_server.cc b/src/search_local/index_storage/common/task_server.cc new file mode 100644 index 0000000..b72d8d1 --- /dev/null +++ b/src/search_local/index_storage/common/task_server.cc @@ -0,0 +1,169 @@ +/* + * ===================================================================================== + * + * Filename: task_server.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "task_base.h" +#include "decode.h" +#include "protocol.h" +#include "buffer_error.h" +#include "log.h" + +int DTCTask::append_result(ResultSet *rs) +{ + if (rs == NULL) + { + set_total_rows(0); + return 0; + } + rs->rewind(); + if (all_rows() && (count_only() || !in_range(rs->total_rows(), 0))) + { + set_total_rows(rs->total_rows()); + } + else + { + for (int i = 0; i < rs->total_rows(); i++) + { + const RowValue *r = rs->fetch_row(); + if (r == NULL) + return rs->error_num(); + if (compare_row(*r)) + { + int rv = resultWriter->append_row(*r); + if (rv < 0) + return rv; + if (all_rows() && result_full()) + { + set_total_rows(rs->total_rows()); + break; + } + } + } + } + return 0; +} + +/* all use prepareResult interface */ +int DTCTask::pass_all_result(ResultSet *rs) +{ + prepare_result(0, 0); + + rs->rewind(); + if (count_only()) + { + set_total_rows(rs->total_rows()); + } + else + { + for (int i = 0; i < rs->total_rows(); i++) + { + int rv; + const RowValue *r = rs->fetch_row(); + if (r == NULL) + { + rv = rs->error_num(); + set_error(rv, "fetch_row()", NULL); + return rv; + } + rv = resultWriter->append_row(*r); + if (rv < 0) + { + set_error(rv, "append_row()", NULL); + return rv; + } + } + } + resultWriter->set_total_rows((unsigned int)(resultInfo.total_rows())); + + return 0; +} + +int DTCTask::merge_result(const DTCTask &task) +{ + /*首先根据子包击中情况统计父包的名种情况*/ + uint32_t uChildTaskHitFlag = task.resultInfo.hit_flag(); + if (HIT_SUCCESS == uChildTaskHitFlag) + { + resultInfo.incr_tech_hit_num(); + ResultPacket *pResultPacket = task.result_code() >= 0 ? task.get_result_packet() : NULL; + if (pResultPacket && (pResultPacket->numRows || pResultPacket->totalRows)) + { + resultInfo.incr_bussiness_hit_num(); + } + } + + int ret = task.resultInfo.affected_rows(); + + if (ret > 0) + { + resultInfo.set_affected_rows(ret + resultInfo.affected_rows()); + } + + if (task.resultWriter == NULL) + return 0; + + if (resultWriter == NULL) + { + if ((ret = prepare_result_no_limit()) != 0) + { + return ret; + } + } + return resultWriter->merge_no_limit(task.resultWriter); +} + +/* the only entry to create resultWriter */ +int DTCTask::prepare_result(int st, int ct) +{ + int err; + + if (resultWriterReseted && resultWriter) + return 0; + + if (resultWriter) + { + err = resultWriter->Set(fieldList, st, ct); + if (err < 0) + { + set_error(-EC_TASKPOOL, "ResultPacket()", NULL); + return err; + } + } + else + { + try + { + resultWriter = new ResultPacket(fieldList, st, ct); + } + catch (int err) + { + set_error(err, "ResultPacket()", NULL); + return err; + } + } + + resultWriterReseted = 1; + + return 0; +} diff --git a/src/search_local/index_storage/common/thread.cc b/src/search_local/index_storage/common/thread.cc new file mode 100644 index 0000000..4a5b11d --- /dev/null +++ b/src/search_local/index_storage/common/thread.cc @@ -0,0 +1,286 @@ +/* + * ===================================================================================== + * + * Filename: thread.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "thread.h" +#include "mem_check.h" +#include "log.h" +#include + +__attribute__((__weak__)) volatile extern int stop; +Thread::thread_map_t Thread::_thread_map; +pthread_mutex_t Thread::_thread_map_lock = PTHREAD_MUTEX_INITIALIZER; + +Thread *Thread::FindThreadByName(const char *name) +{ + Thread::LOCK_THREAD_MAP(); + thread_map_t::const_iterator i = _thread_map.find(name); + Thread *ret = i == _thread_map.end() ? NULL : i->second; + Thread::UNLOCK_THREAD_MAP(); + return ret; +} + +__attribute__((__weak__)) void DaemonCrashed(int); + +#if HAS_TLS +static __thread Thread *currentThread __TLS_MODEL; + +extern "C" void crash_hook(int signo) +{ + if (currentThread != NULL) + currentThread->crash_hook(signo); +} +#endif + +void *Thread::Entry(void *thread) +{ + Thread *my = (Thread *)thread; + my->prepare_internal(); + void *ret = my->Process(); + my->Cleanup(); + return ret; +} + +Thread::Thread(const char *name, int type) +{ + tid = 0; + pid = 0; + tasktype = type; + taskname = STRDUP(name); + stacksize = 0; + cpumask = 0; + + stopped = 0; + stopPtr = &stopped; + + pthread_mutex_init(&runlock, NULL); + + Thread::LOCK_THREAD_MAP(); + _thread_map[taskname] = this; + Thread::UNLOCK_THREAD_MAP(); +} + +Thread::~Thread() +{ + pthread_mutex_destroy(&runlock); + FREE_IF(taskname); +} + +void Thread::set_stack_size(int size) +{ + if (size < 0) // default size + size = 0; + else if (size > 64 << 20) // max 64M + size = 64 << 20; + + // round to 64K + if ((size & 0xFFFF) != 0) + size = (size & ~0xFFFF) + 0x10000; + stacksize = size; +} + +AutoConfig *Thread::g_autoconf = NULL; + +void Thread::auto_config_stack_size(void) +{ + unsigned long long size = g_autoconf->get_size_val("ThreadStackSize", taskname, stacksize, 'M'); + + if (size > (1 << 30)) + size = 1 << 30; + + set_stack_size((int)size); + log_debug("autoconf thread %s ThreadStackSize %d", taskname, stacksize); +} + +void Thread::auto_config_cpu_mask(void) +{ +#ifdef CPU_ZERO + const char *val = g_autoconf->get_str_val("ThreadCPUMask", taskname); + if (val != NULL) + { + cpumask = strtoll(val, NULL, 16); + } + log_debug("autoconf thread %s ThreadCPUMask %llx", taskname, (unsigned long long)cpumask); +#endif +} + +void Thread::auto_config(void) +{ + if (g_autoconf) + { + switch (tasktype) + { + case ThreadTypeSync: + case ThreadTypeAsync: + auto_config_stack_size(); + break; + default: + break; + } + auto_config_cpu_mask(); + } +} + +int Thread::initialize_thread(void) +{ + int ret = Initialize(); + if (ret < 0) + return -1; + switch (tasktype) + { + case 0: // main + tid = pthread_self(); + auto_config(); +#ifdef CPU_ZERO + if (cpumask != 0) + { + uint64_t temp[2] = {cpumask, 0}; + sched_setaffinity(pid, sizeof(temp), (cpu_set_t *)&temp); + } +#endif + break; + + case -1: // world class + tid = 0; + break; + + case 1: // async + case 2: // sync + auto_config(); + pthread_mutex_lock(&runlock); + pthread_attr_t attr; + pthread_attr_init(&attr); + if (stacksize) + { + pthread_attr_setstacksize(&attr, stacksize); + } + pthread_create(&tid, &attr, Entry, this); + break; + } + return 0; +} + +void Thread::running_thread() +{ + switch (tasktype) + { + case -1: + // error + return; + case 0: + stopPtr = &stop; + prepare_internal(); + Process(); + Cleanup(); + break; + case 1: + case 2: + pthread_mutex_unlock(&runlock); + break; + } +} + +void Thread::prepare_internal(void) +{ + pid = _gettid_(); + _set_log_thread_name_(taskname); + log_info("thread %s[%d] started", taskname, pid); + _set_remote_log_fd_(); + sigset_t sset; + sigemptyset(&sset); + sigaddset(&sset, SIGTERM); + pthread_sigmask(SIG_BLOCK, &sset, &sset); +#ifdef CPU_ZERO + if (cpumask != 0) + { + uint64_t temp[2] = {cpumask, 0}; + sched_setaffinity(pid, sizeof(temp), (cpu_set_t *)&temp); + } +#endif + Prepare(); + pthread_mutex_lock(&runlock); + +#if HAS_TLS + currentThread = this; +#endif +} + +void Thread::interrupt(void) +{ + if (this == NULL || stopped || !tid) + return; + + int ret; + stopped = 1; + + if (tasktype > 0) + { + pthread_kill(tid, SIGINT); + pthread_mutex_unlock(&runlock); + + if ((ret = pthread_join(tid, 0)) != 0) + { + log_warning("Thread [%s] join failed %d.", Name(), ret); + } + else + { + log_info("Thread [%s] stopped.", Name()); + } + } +} + +int Thread::Initialize(void) +{ + return 0; +} + +void Thread::Prepare(void) +{ +} + +void *Thread::Process(void) +{ + while (!Stopping()) + pause(); + return NULL; +} + +void Thread::Cleanup(void) +{ +} + +void Thread::crash_hook(int signo) +{ + // mark mine is stopped + this->stopped = 1; + // global stopping + if (&DaemonCrashed != 0) + { + DaemonCrashed(signo); + } + log_crit("Ouch......, I crashed, hang and stopping server"); + pthread_exit(NULL); + while (1) + pause(); +} + +Thread Thread::TheWorldThread("world", Thread::ThreadTypeWorld); diff --git a/src/search_local/index_storage/common/thread.h b/src/search_local/index_storage/common/thread.h new file mode 100644 index 0000000..c2bb91a --- /dev/null +++ b/src/search_local/index_storage/common/thread.h @@ -0,0 +1,106 @@ +/* + * ===================================================================================== + * + * Filename: thread.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __GENERICTHREAD_H__ +#define __GENERICTHREAD_H__ + +#include +#include +#include +#include +#include + +#include "timestamp.h" +#include "poller.h" +#include "timer_list.h" +#include "config.h" + +class Thread +{ +public: + enum + { + ThreadTypeWorld = -1, + ThreadTypeProcess = 0, + ThreadTypeAsync = 1, + ThreadTypeSync = 2 + }; + +public: + static AutoConfig *g_autoconf; + static void set_auto_config_instance(AutoConfig *ac) { g_autoconf = ac; }; + +public: + Thread(const char *name, int type = 0); + virtual ~Thread(); + + int initialize_thread(); + void running_thread(); + const char *Name(void) { return taskname; } + int Pid(void) const { return stopped ? 0 : pid; } + void set_stack_size(int); + int Stopping(void) { return *stopPtr; } + virtual void interrupt(void); + virtual void crash_hook(int); + + Thread *the_world(void) { return &TheWorldThread; }; + +protected: + char *taskname; + pthread_t tid; + pthread_mutex_t runlock; + int stopped; + volatile int *stopPtr; + int tasktype; // 0-->main, 1--> epoll, 2-->sync, -1, world + int stacksize; + uint64_t cpumask; + int pid; + +protected: + virtual int Initialize(void); + virtual void Prepare(void); + virtual void *Process(void); + virtual void Cleanup(void); + +protected: + struct cmp + { + bool operator()(const char *const &a, const char *const &b) const + { + return strcmp(a, b) < 0; + } + }; + typedef std::map thread_map_t; + static thread_map_t _thread_map; + static pthread_mutex_t _thread_map_lock; + + static void LOCK_THREAD_MAP() { pthread_mutex_lock(&_thread_map_lock); } + static void UNLOCK_THREAD_MAP() { pthread_mutex_unlock(&_thread_map_lock); } + +public: + static Thread *FindThreadByName(const char *name); + +private: + static Thread TheWorldThread; + static void *Entry(void *thread); + void prepare_internal(void); + void auto_config(void); + void auto_config_stack_size(void); + void auto_config_cpu_mask(void); +}; + +#endif diff --git a/src/search_local/index_storage/common/thread_cpu_stat.cc b/src/search_local/index_storage/common/thread_cpu_stat.cc new file mode 100644 index 0000000..7948f9d --- /dev/null +++ b/src/search_local/index_storage/common/thread_cpu_stat.cc @@ -0,0 +1,282 @@ +/* + * ===================================================================================== + * + * Filename: thread_cpu_stat.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include //for HZ +#include +#include +#include + +#include "poll_thread.h" +#include "thread_cpu_stat.h" + +int thread_cpu_stat::init() +{ + add_cpu_stat_object("worker", WORKER_THREAD_CPU_STAT); + add_cpu_stat_object("cache", CACHE_THREAD_CPU_STAT); + add_cpu_stat_object("source", DATA_SOURCE_CPU_STAT); + add_cpu_stat_object("inc0", INC_THREAD_CPU_STAT_0); + add_cpu_stat_object("inc1", INC_THREAD_CPU_STAT_1); + add_cpu_stat_object("inc2", INC_THREAD_CPU_STAT_2); + add_cpu_stat_object("inc3", INC_THREAD_CPU_STAT_3); + add_cpu_stat_object("inc4", INC_THREAD_CPU_STAT_4); + add_cpu_stat_object("inc5", INC_THREAD_CPU_STAT_4); + add_cpu_stat_object("inc6", INC_THREAD_CPU_STAT_6); + add_cpu_stat_object("inc7", INC_THREAD_CPU_STAT_7); + add_cpu_stat_object("inc8", INC_THREAD_CPU_STAT_8); + add_cpu_stat_object("inc9", INC_THREAD_CPU_STAT_9); + + return 0; +} + +//function: 建立cpu统计结构体，添加初始化一些基本项 +//input: 无 +//ouput： return -1: 初始化失败 +// return 0：初始化成功 +int thread_cpu_stat::add_cpu_stat_object(const char *thread_name, int stat_idx) +{ + Thread *thread = Thread::FindThreadByName(thread_name); + if (thread == NULL || thread->Pid() <= 0) + { + return -1; + } + + one_thread_cpu_stat_t *new_cpu_stat = new one_thread_cpu_stat_t; + if (!new_cpu_stat) + { + log_warning("thread create new cpu stat error"); + return -1; + } + + new_cpu_stat->_next = _stat_list_head; + _stat_list_head = new_cpu_stat; + + return new_cpu_stat->init(thread_name, stat_idx, thread->Pid()); +} + +//function: 初始化一些基本项 +//input: 无 +//ouput： return -1: 初始化失败 +// return 0：初始化成功 +int one_thread_cpu_stat::init(const char *thread_name, int stat_idx, int pid) +{ + + _pid = pid; + _thread_name = thread_name; + _last_time = 0; + _this_time = 0; + _last_cpu_time = 0; + _this_cpu_time = 0; + _pcpu = statmgr.get_item_u32(stat_idx); + _pcpu = 100; + _err_times = 0; + _status = CPU_STAT_INVALID; + + int mainpid = getpid(); + char pro_stat_file[256]; + snprintf(pro_stat_file, 255, "/proc/%d/task/%d/stat", mainpid, _pid); + _fd = open(pro_stat_file, O_RDONLY); + if (_fd < 0) + { + log_warning("open proc stat of thread %d failed", _pid); + } + else + { + int ret = read_cpu_time(); + if (ret < 0) + { + _status = CPU_STAT_NOT_INIT; + log_warning("thread [%d] cpu stat init error", _pid); + } + else + { + _last_cpu_time = _this_cpu_time; + _last_time = _this_time; + _status = CPU_STAT_INIT; + } + } + + return 0; +} + +static inline void skip_word(char **p) +{ + *p = strchr(*p, ' '); + *p = *p + 1; +} + +int one_thread_cpu_stat::read_cpu_time(void) +{ + int ret = lseek(_fd, 0, SEEK_SET); + if (ret < 0) + { + log_warning("lseek proc file to begain failed"); + return -1; + } + + char stat_buffer[1024]; + ret = read(_fd, stat_buffer, 1023); + if (ret < 0) + { + log_warning("proc file read errro"); + return -1; + } + + char *p = stat_buffer; + skip_word(&p); /*skip pid*/ + skip_word(&p); /*skip cmd name*/ + skip_word(&p); /*skip status*/ + skip_word(&p); /*skip ppid*/ + skip_word(&p); /*skip pgrp*/ + skip_word(&p); /*skip session*/ + skip_word(&p); /*skip tty*/ + skip_word(&p); /*skip tty pgrp*/ + skip_word(&p); /*skip flags*/ + skip_word(&p); /*skip min flt*/ + skip_word(&p); /*skip cmin flt*/ + skip_word(&p); /*skip maj flt*/ + skip_word(&p); /*skip cmaj flt*/ + + unsigned long long utime = 0; + unsigned long long stime = 0; + + char *q = strchr(p, ' '); + *q = 0; + utime = atoll(p); + *q = ' '; + + skip_word(&p); /*skip utime*/ + + q = strchr(p, ' '); + *q = 0; + stime = atoll(p); + *q = ' '; + + _this_cpu_time = utime + stime; + + struct timeval now; + gettimeofday(&now, NULL); + _this_time = now.tv_sec + now.tv_usec * 1e-6; + + return 0; +} + +void one_thread_cpu_stat::cal_pcpu(void) +{ + double time_diff = _this_time - _last_time; + time_diff *= HZ; + + if (time_diff > 0) + { + _pcpu = (int)(10000 * ((_this_cpu_time - _last_cpu_time) / time_diff)); + } + //else, keep previous cpu percentage + + _last_cpu_time = _this_cpu_time; + _last_time = _this_time; + return; +} + +#define READ_CPU_TIME_ERROR log_warning("thread %s[%d] read cpu time error", _thread_name, _pid) + +void one_thread_cpu_stat::do_stat() +{ + int ret = 0; + switch (_status) + { + case CPU_STAT_INVALID: + break; + + case CPU_STAT_NOT_INIT: + ret = read_cpu_time(); + if (ret < 0) + { + READ_CPU_TIME_ERROR; + } + else + { + _last_cpu_time = _this_cpu_time; + _last_time = _this_time; + _status = CPU_STAT_INIT; + } + break; + + case CPU_STAT_INIT: + ret = read_cpu_time(); + if (ret < 0) + { + READ_CPU_TIME_ERROR; + } + else + { + cal_pcpu(); + _err_times = 0; + _status = CPU_STAT_FINE; + } + break; + + case CPU_STAT_FINE: + ret = read_cpu_time(); + if (ret < 0) + { + READ_CPU_TIME_ERROR; + _err_times++; + if (_err_times >= THREAD_CPU_STAT_MAX_ERR) + { + log_warning("max err, transfer to ill status"); + _pcpu = 10000; + _status = CPU_STAT_ILL; + } + } + else + { + cal_pcpu(); + _err_times = 0; + } + break; + case CPU_STAT_ILL: + ret = read_cpu_time(); + if (ret < 0) + { + READ_CPU_TIME_ERROR; + } + else + { + cal_pcpu(); + _err_times = 0; + _status = CPU_STAT_FINE; + } + break; + default: + log_warning("unkown cpu stat status"); + break; + } //end of this thread statistic + + return; +} + +void thread_cpu_stat::do_stat() +{ + one_thread_cpu_stat_t *cpu_stat = _stat_list_head; + while (cpu_stat) + { + cpu_stat->do_stat(); + cpu_stat = cpu_stat->_next; + } //end of all thread statistic + + return; +} \ No newline at end of file diff --git a/src/search_local/index_storage/common/thread_cpu_stat.h b/src/search_local/index_storage/common/thread_cpu_stat.h new file mode 100644 index 0000000..62cb4a8 --- /dev/null +++ b/src/search_local/index_storage/common/thread_cpu_stat.h @@ -0,0 +1,74 @@ +/* + * ===================================================================================== + * + * Filename: thread_cpu_stat.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef THREAD_CPU_STAT_H +#define THREAD_CPU_STAT_H + +#include "log.h" +#include "poll_thread.h" +#include "stat_dtc.h" +#include +#define THREAD_CPU_STAT_MAX_ERR 6 + +#define CPU_STAT_INVALID 0 +#define CPU_STAT_NOT_INIT 1 +#define CPU_STAT_INIT 2 +#define CPU_STAT_FINE 3 +#define CPU_STAT_ILL 4 + +typedef struct one_thread_cpu_stat +{ + int _fd; //proc fd thread hold + int _pid; //thread pid + const char *_thread_name; + double _last_time; + double _this_time; + unsigned long long _last_cpu_time; //previous cpu time of this thread + unsigned long long _this_cpu_time; //this cpu time of this thread + StatItemU32 _pcpu; //cpu percentage + unsigned int _err_times; //max error calculate time + unsigned int _status; //status + struct one_thread_cpu_stat *_next; + +public: + int init(const char *thread_name, int thread_idx, int pid); + int read_cpu_time(void); + void cal_pcpu(void); + void do_stat(); +} one_thread_cpu_stat_t; + +class thread_cpu_stat +{ +private: + one_thread_cpu_stat_t *_stat_list_head; + +public: + thread_cpu_stat() + { + _stat_list_head = NULL; + } + ~thread_cpu_stat() + { + } + int init(); + void do_stat(); + +private: + int add_cpu_stat_object(const char *thread_name, int thread_idx); +}; + +#endif diff --git a/src/search_local/index_storage/common/timer_list.cc b/src/search_local/index_storage/common/timer_list.cc new file mode 100644 index 0000000..789d6e8 --- /dev/null +++ b/src/search_local/index_storage/common/timer_list.cc @@ -0,0 +1,224 @@ +/* + * ===================================================================================== + * + * Filename: timer_list.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "timer_list.h" +#include "log.h" +#if TIMESTAMP_PRECISION < 1000 +#error TIMESTAMP_PRECISION must >= 1000 +#endif + +#ifndef likely +#if __GCC_MAJOR >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif +#endif + +TimerObject::~TimerObject(void) +{ +} + +void TimerObject::timer_notify(void) +{ + log_debug("TimerObject::timer_notify start"); + delete this; +} + +void TimerObject::attach_timer(class TimerList *lst) +{ + if (lst->timeout > 0) + objexp = lst->get_time_unit_now_time() + lst->timeout * (TIMESTAMP_PRECISION / 1000); + list_move_tail(lst->tlist); +} + +int TimerList::check_expired(int64_t now) +{ + int n = 0; + if (now == 0) + { + now = get_time_unit_now_time(); + } + while (!tlist.ListEmpty()) + { + TimerObject *tobj = tlist.NextOwner(); + if (tobj->objexp > now) + break; + tobj->list_del(); + tobj->timer_notify(); + n++; + } + return n; +} + +int64_t TimerList::get_time_unit_now_time() +{ + if (NULL == timerUnitOwner) + { + return GET_TIMESTAMP(); + } + return timerUnitOwner->get_now_time(); +} + +void TimerUnit::update_now_time(int max_wait, int interrupted) +{ + int64_t adjustTime = 0; + int64_t deadLineTime = 0; + const int MAX_DELAY_MS = 1000000; /*前向拨动最多量*/ + m_SystemTime = GET_TIMESTAMP(); + + if (unlikely(max_wait < 0)) + { + m_TimeOffSet = 0; + m_NowTime = m_SystemTime; + return; + } + + adjustTime = m_SystemTime + m_TimeOffSet; + /*时间被向后拨动了*/ + if (adjustTime < m_NowTime) + { + adjustTime = interrupted ? (m_NowTime) : (m_NowTime + max_wait * (TIMESTAMP_PRECISION / 1000)); + m_TimeOffSet = adjustTime - m_SystemTime; + m_NowTime = adjustTime; + return; + } + + deadLineTime = m_NowTime + max_wait * (TIMESTAMP_PRECISION / 1000) + MAX_DELAY_MS; + if (likely(adjustTime < deadLineTime)) + { + m_NowTime = adjustTime; + return; + } + /*时间被向前拨动了*/ + else + { + adjustTime = interrupted ? (m_NowTime) : (m_NowTime + max_wait * (TIMESTAMP_PRECISION / 1000)); + m_TimeOffSet = adjustTime - m_SystemTime; + m_NowTime = adjustTime; + return; + } +} + +TimerList *TimerUnit::get_timer_list_by_m_seconds(int to) +{ + TimerList *tl; + + for (tl = next; tl; tl = tl->next) + { + if (tl->timeout == to) + return tl; + } + tl = new TimerList(to, this); + tl->next = next; + next = tl; + return tl; +} + +TimerUnit::TimerUnit(void) : pending(0), next(NULL), m_TimeOffSet(0) +{ + m_SystemTime = GET_TIMESTAMP(); + m_NowTime = m_SystemTime; +}; + +TimerUnit::~TimerUnit(void) +{ + while (next) + { + TimerList *tl = next; + next = tl->next; + delete tl; + } +}; + +int TimerUnit::expire_micro_seconds(int msec, int msec0) +{ + int64_t exp; + TimerList *tl; + int64_t timestamp = get_now_time(); + exp = timestamp + msec * (TIMESTAMP_PRECISION / 1000); + + for (tl = next; tl; tl = tl->next) + { + if (tl->tlist.ListEmpty()) + continue; + TimerObject *o = tl->tlist.NextOwner(); + + if (o->objexp < exp) + exp = o->objexp; + } + exp -= timestamp; + if (exp <= 0) + return 0; + msec = exp / (TIMESTAMP_PRECISION / 1000); + return msec >= msec0 ? msec : msec0; +} + +int TimerUnit::check_ready(void) +{ + int n = 0; + while (!pending.tlist.ListEmpty()) + { + TimerObject *tobj = pending.tlist.NextOwner(); + tobj->list_del(); + tobj->timer_notify(); + n++; + } + return n; +} + +int TimerUnit::check_expired(int64_t now) +{ + if (now == 0) + now = get_now_time(); + + int n = check_ready(); + ; + TimerList *tl; + for (tl = next; tl; tl = tl->next) + { + n += tl->check_expired(now); + } + return n; +} + +void ReadyObject::ready_notify(uint64_t now) +{ + delete this; +} + +int ReadyUnit::check_ready(uint64_t now) +{ + int n = 0; + ListObject *tmp; + + tmp = pending; + pending = processing; + processing = tmp; + + while (!processing->ListEmpty()) + { + ReadyObject *robj = processing->NextOwner(); + robj->list_del(); + robj->ready_notify(now); + n++; + } + return n; +} diff --git a/src/search_local/index_storage/common/timer_list.h b/src/search_local/index_storage/common/timer_list.h new file mode 100644 index 0000000..b8e7275 --- /dev/null +++ b/src/search_local/index_storage/common/timer_list.h @@ -0,0 +1,136 @@ +/* + * ===================================================================================== + * + * Filename: timer_list.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __TIMERLIST_H__ +#define __TIMERLIST_H__ + +#include "list.h" +#include "timestamp.h" + +class TimerObject; +class TimerUnit; + +class TimerList +{ +private: + ListObject tlist; + int timeout; + TimerList *next; + TimerUnit *timerUnitOwner; + +public: + friend class TimerUnit; + friend class TimerObject; + TimerList(int t) : timeout(t), next(NULL) + { + } + TimerList(int t, TimerUnit *timerUnitOwner) : timeout(t), next(NULL), timerUnitOwner(timerUnitOwner) + { + } + int64_t get_time_unit_now_time(); + ~TimerList(void) { tlist.FreeList(); } + int check_expired(int64_t now = 0); +}; + +class TimerUnit +{ +private: + TimerList pending; + TimerList *next; + + int64_t m_SystemTime; /*系统时间*/ + int64_t m_NowTime; /*应用层时间*/ + int64_t m_TimeOffSet; /*时间拨动后的修正量*/ + +public: + friend class TimerObject; + TimerUnit(void); + ~TimerUnit(void); + int64_t get_now_time() + { + return m_NowTime; + } + void update_now_time(int max_wait = 0, int interrupted = 0); + TimerList *get_timer_list_by_m_seconds(int); + TimerList *get_timer_list(int t) { return get_timer_list_by_m_seconds(t * 1000); } + int expire_micro_seconds(int, int = 0); // arg: max/min msec + int check_expired(int64_t now = 0); + int check_ready(void); +}; + +class TimerObject : private ListObject +{ +private: + int64_t objexp; + +public: + friend class TimerList; + friend class TimerUnit; + TimerObject() {} + virtual ~TimerObject(void); + virtual void timer_notify(void); + void disable_timer(void) { ResetList(); } + void attach_timer(class TimerList *o); + void attach_ready_timer(class TimerUnit *o) { list_move_tail(o->pending.tlist); } +}; + +template +class TimerMember : public TimerObject +{ +private: + T *owner; + virtual void timer_notify(void) { owner->timer_notify(); } + +public: + TimerMember(T *o) : owner(o) {} + virtual ~TimerMember() {} +}; + +class ReadyObject; +class ReadyUnit +{ +private: + ListObject pending1; + ListObject pending2; + ListObject *pending; + ListObject *processing; + +public: + friend class ReadyObject; + ReadyUnit() + { + pending = &pending1; + processing = &pending2; + } + virtual ~ReadyUnit() {} + + int check_ready(uint64_t now); +}; + +class ReadyObject : private ListObject +{ +public: + friend class ReadyUnit; + ReadyObject() {} + virtual ~ReadyObject() {} + + virtual void ready_notify(uint64_t now); + void attach_ready(ReadyUnit *o) { list_move_tail(o->pending); } + void disable_ready() { ResetList(); } +}; + +#endif diff --git a/src/search_local/index_storage/common/timestamp.h b/src/search_local/index_storage/common/timestamp.h new file mode 100644 index 0000000..9607c02 --- /dev/null +++ b/src/search_local/index_storage/common/timestamp.h @@ -0,0 +1,34 @@ +/* + * ===================================================================================== + * + * Filename: timestamp.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_TIMESTAMP_H__ +#define __DTC_TIMESTAMP_H__ + +#include +#include +#include + +static inline int64_t GET_TIMESTAMP(void) +{ + struct timeval tv; + gettimeofday(&tv, NULL); + return (int64_t)tv.tv_sec * 1000000 + tv.tv_usec; +} + +#define TIMESTAMP_PRECISION 1000000ULL + +#endif diff --git a/src/search_local/index_storage/common/unix_socket.h b/src/search_local/index_storage/common/unix_socket.h new file mode 100644 index 0000000..779ac1f --- /dev/null +++ b/src/search_local/index_storage/common/unix_socket.h @@ -0,0 +1,36 @@ +/* + * ===================================================================================== + * + * Filename: unix_socket.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +static inline int is_unix_socket_path(const char *path) +{ + return path[0] == '/' || path[0] == '@'; +} + +static inline int init_unix_socket_address(struct sockaddr_un *addr, const char *path) +{ + + bzero(addr, sizeof(struct sockaddr_un)); + addr->sun_family = AF_LOCAL; + strncpy(addr->sun_path, path, sizeof(addr->sun_path) - 1); + socklen_t addrlen = SUN_LEN(addr); + if (path[0] == '@') + addr->sun_path[0] = '\0'; + return addrlen; +} diff --git a/src/search_local/index_storage/common/value.c b/src/search_local/index_storage/common/value.c new file mode 100644 index 0000000..b9d4cf9 --- /dev/null +++ b/src/search_local/index_storage/common/value.c @@ -0,0 +1,296 @@ +/* + * ===================================================================================== + * + * Filename: value.c + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +const unsigned char internal_tolower_table_[256] = { + 0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E, + 0x0F, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0x1A, + 0x1B, + 0x1C, + 0x1D, + 0x1E, + 0x1F, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x25, + 0x26, + 0x27, + 0x28, + 0x29, + 0x2A, + 0x2B, + 0x2C, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x32, + 0x33, + 0x34, + 0x35, + 0x36, + 0x37, + 0x38, + 0x39, + 0x3A, + 0x3B, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x40, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6A, + 0x6B, + 0x6C, + 0x6D, + 0x6E, + 0x6F, + 0x70, + 0x71, + 0x72, + 0x73, + 0x74, + 0x75, + 0x76, + 0x77, + 0x78, + 0x79, + 0x7A, + 0x5B, + 0x5C, + 0x5D, + 0x5E, + 0x5F, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6A, + 0x6B, + 0x6C, + 0x6D, + 0x6E, + 0x6F, + 0x70, + 0x71, + 0x72, + 0x73, + 0x74, + 0x75, + 0x76, + 0x77, + 0x78, + 0x79, + 0x7A, + 0x7B, + 0x7C, + 0x7D, + 0x7E, + 0x7F, + 0x80, + 0x81, + 0x82, + 0x83, + 0x84, + 0x85, + 0x86, + 0x87, + 0x88, + 0x89, + 0x8A, + 0x8B, + 0x8C, + 0x8D, + 0x8E, + 0x8F, + 0x90, + 0x91, + 0x92, + 0x93, + 0x94, + 0x95, + 0x96, + 0x97, + 0x98, + 0x99, + 0x9A, + 0x9B, + 0x9C, + 0x9D, + 0x9E, + 0x9F, + 0xA0, + 0xA1, + 0xA2, + 0xA3, + 0xA4, + 0xA5, + 0xA6, + 0xA7, + 0xA8, + 0xA9, + 0xAA, + 0xAB, + 0xAC, + 0xAD, + 0xAE, + 0xAF, + 0xB0, + 0xB1, + 0xB2, + 0xB3, + 0xB4, + 0xB5, + 0xB6, + 0xB7, + 0xB8, + 0xB9, + 0xBA, + 0xBB, + 0xBC, + 0xBD, + 0xBE, + 0xBF, + 0xC0, + 0xC1, + 0xC2, + 0xC3, + 0xC4, + 0xC5, + 0xC6, + 0xC7, + 0xC8, + 0xC9, + 0xCA, + 0xCB, + 0xCC, + 0xCD, + 0xCE, + 0xCF, + 0xD0, + 0xD1, + 0xD2, + 0xD3, + 0xD4, + 0xD5, + 0xD6, + 0xD7, + 0xD8, + 0xD9, + 0xDA, + 0xDB, + 0xDC, + 0xDD, + 0xDE, + 0xDF, + 0xE0, + 0xE1, + 0xE2, + 0xE3, + 0xE4, + 0xE5, + 0xE6, + 0xE7, + 0xE8, + 0xE9, + 0xEA, + 0xEB, + 0xEC, + 0xED, + 0xEE, + 0xEF, + 0xF0, + 0xF1, + 0xF2, + 0xF3, + 0xF4, + 0xF5, + 0xF6, + 0xF7, + 0xF8, + 0xF9, + 0xFA, + 0xFB, + 0xFC, + 0xFD, + 0xFE, + 0xFF, +}; + +#define TOLOWER(x) internal_tolower_table_[x] + +int mystrcmp(const char *s1, const char *s2, int l) +{ + const unsigned char *p1 = (const unsigned char *)s1; + const unsigned char *p2 = (const unsigned char *)s2; + int result = 0; + + if (p1 == p2) + return 0; + + while (l > 0 && (result = TOLOWER(*p1) - TOLOWER(*p2++)) == 0) + if (*p1++ == '\0') + break; + else + l--; + + return result; +} diff --git a/src/search_local/index_storage/common/value.h b/src/search_local/index_storage/common/value.h new file mode 100644 index 0000000..e3585f9 --- /dev/null +++ b/src/search_local/index_storage/common/value.h @@ -0,0 +1,529 @@ +/* + * ===================================================================================== + * + * Filename: value.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __CH_VALUE_H_ +#define __CH_VALUE_H_ + +#include +#include + +struct DTCBinary +{ + // member + int len; + char *ptr; + + // API + inline int is_empty(void) const { return len <= 0; }; + inline uint8_t unsigned_value(int n = 0) const { return *(uint8_t *)(ptr + n); } + inline int8_t signed_value(int n = 0) const { return *(uint8_t *)(ptr + n); } + inline void next_value(int n = 1) + { + ptr += n; + len -= n; + } + inline uint8_t unsigned_value_next(void) + { + len--; + return *(uint8_t *)ptr++; + } + inline int8_t signed_value_next(void) + { + len--; + return *(int8_t *)ptr++; + } + + inline int operator!() const { return len <= 0; } + inline int operator*() const { return *(uint8_t *)ptr; } + inline uint8_t operator[](int n) const { return *(uint8_t *)(ptr + n); } + inline DTCBinary &operator+=(int n) + { + ptr += n; + len -= n; + return *this; + } + inline DTCBinary &operator++() + { + ptr++; + len--; + return *this; + } + inline DTCBinary operator++(int) + { + DTCBinary r = {len, ptr}; + ptr++; + len--; + return r; + } + + inline int operator<(int n) const { return len < n; } + inline int operator<=(int n) const { return len <= n; } + inline int operator>(int n) const { return len > n; } + inline int operator>=(int n) const { return len >= n; } + + /* set ptr and len */ + inline DTCBinary &Set(const char *v) + { + ptr = (char *)v; + len = v ? strlen(v) : 0; + return *this; + } + inline DTCBinary &Set(const char *v, int l) + { + ptr = (char *)v; + len = l; + return *this; + } + inline DTCBinary &SetZ(const char *v) + { + ptr = (char *)v; + len = v ? strlen(v) + 1 : 0; + return *this; + } + inline DTCBinary &operator=(const char *v) { return Set(v); } + + /* set and copy */ + inline static DTCBinary Make(const char *v, int l) + { + DTCBinary a; + a.Set(v, l); + return a; + } + inline static DTCBinary Make(const char *v) + { + DTCBinary a; + a.Set(v); + return a; + } + inline static DTCBinary MakeZ(const char *v) + { + DTCBinary a; + a.SetZ(v); + return a; + } +}; + +struct Array : public DTCBinary +{ + Array(int l, char *p) + { + len = l; + ptr = p; + } + Array(const DTCBinary &bin) + { + len = bin.len; + ptr = bin.ptr; + } + +#define ADD_V(TYPE) \ + void Add(TYPE val) \ + { \ + *(TYPE *)(ptr + len) = val; \ + len += sizeof(TYPE); \ + } + ADD_V(int8_t); + ADD_V(uint8_t); + ADD_V(int32_t); + ADD_V(uint32_t); + ADD_V(int64_t); + ADD_V(uint64_t); +#undef ADD_V + + void Add(const char *val) + { + uint32_t slen = strlen(val); + *(uint32_t *)(ptr + len) = slen + 1; + memcpy(ptr + len + sizeof(uint32_t), val, slen + 1); + len += sizeof(uint32_t) + slen + 1; + } + + void Add(const char *val, int l) + { + *(uint32_t *)(ptr + len) = l; + memcpy(ptr + len + sizeof(uint32_t), val, l); + len += sizeof(uint32_t) + l; + } + +#define GET_V(TYPE) \ + int Get(TYPE &val, int size = 0) \ + { \ + if ((unsigned int)len < sizeof(TYPE)) \ + return (-1); \ + val = *(TYPE *)ptr; \ + ptr += sizeof(TYPE); \ + len -= sizeof(TYPE); \ + return (0); \ + } + + GET_V(int8_t); + GET_V(uint8_t); + GET_V(int32_t); + GET_V(uint32_t); + GET_V(int64_t); + GET_V(uint64_t); +#undef GET_V + + int Get(DTCBinary &bin) + { + if ((unsigned int)len < sizeof(uint32_t)) + return (-1); + bin.len = *(uint32_t *)ptr; + bin.ptr = ptr + sizeof(uint32_t); + if ((unsigned int)len < sizeof(uint32_t) + bin.len) + return (-2); + ptr += sizeof(uint32_t) + bin.len; + len -= sizeof(uint32_t) + bin.len; + return (0); + } +}; + +typedef union DTCValue +{ + // member + int64_t s64; + uint64_t u64; + double flt; + struct DTCBinary bin; + struct DTCBinary str; + + // API + DTCValue() {} +#if !__x86_64__ + DTCValue(long v) : s64(v) + { + } + DTCValue(unsigned long v) : u64(v) {} +#else + DTCValue(long long v) : s64(v) + { + } + DTCValue(unsigned long long v) : u64(v) {} +#endif + DTCValue(int32_t v) : s64(v) + { + } + DTCValue(uint32_t v) : u64(v) {} + DTCValue(int64_t v) : s64(v) {} + DTCValue(uint64_t v) : u64(v) {} + DTCValue(float v) : flt(v) {} + DTCValue(double v) : flt(v) {} + DTCValue(const char *v) { str = DTCBinary::Make(v); } + DTCValue(const char *v, int l) { str = DTCBinary::Make(v, l); } + ~DTCValue() {} +#if !__x86_64__ + static DTCValue Make(const long v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const unsigned long v) + { + DTCValue a(v); + return a; + } +#else + static DTCValue Make(const long long v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const unsigned long long v) + { + DTCValue a(v); + return a; + } +#endif + static DTCValue Make(const int32_t v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const uint32_t v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const int64_t v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const uint64_t v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const float v) + { + DTCValue a(v); + return a; + } + static DTCValue Make(const double v) + { + DTCValue a(v); + return a; + } + static const DTCValue Make(const char *v) + { + const DTCValue a(v); + return a; + } + static const DTCValue Make(const char *v, int l) + { + const DTCValue a(v, l); + return a; + } +#if !__x86_64__ + DTCValue &Set(const long v) + { + s64 = v; + return *this; + } + DTCValue &Set(const unsigned long v) + { + u64 = v; + return *this; + } +#else + DTCValue &Set(const long long v) + { + s64 = v; + return *this; + } + DTCValue &Set(const unsigned long long v) + { + u64 = v; + return *this; + } +#endif + DTCValue &Set(const int32_t v) + { + s64 = v; + return *this; + } + DTCValue &Set(const uint32_t v) + { + u64 = v; + return *this; + } + DTCValue &Set(const int64_t v) + { + s64 = v; + return *this; + } + DTCValue &Set(const uint64_t v) + { + u64 = v; + return *this; + } + DTCValue &Set(const float v) + { + flt = v; + return *this; + } + DTCValue &Set(const double v) + { + flt = v; + return *this; + } + DTCValue &Set(const char *v) + { + str.Set(v); + return *this; + } + DTCValue &SetZ(const char *v) + { + str.SetZ(v); + return *this; + } + DTCValue &Set(const char *v, int l) + { + str.Set(v, l); + return *this; + } + +#if !__x86_64__ + DTCValue &operator=(const long v) + { + s64 = v; + return *this; + } + DTCValue &operator=(const unsigned long v) + { + u64 = v; + return *this; + } +#else + DTCValue &operator=(const long long v) + { + s64 = v; + return *this; + } + DTCValue &operator=(const unsigned long long v) + { + u64 = v; + return *this; + } +#endif + DTCValue &operator=(const int32_t v) + { + s64 = v; + return *this; + } + DTCValue &operator=(const uint32_t v) + { + u64 = v; + return *this; + } + DTCValue &operator=(const int64_t v) + { + s64 = v; + return *this; + } + DTCValue &operator=(const uint64_t v) + { + u64 = v; + return *this; + } + DTCValue &operator=(const float v) + { + flt = v; + return *this; + } + DTCValue &operator=(const double v) + { + flt = v; + return *this; + } + DTCValue &operator=(const char *v) + { + str.Set(v); + return *this; + } +} DTCValue; + +extern "C" +{ + int mystrcmp(const char *s1, const char *s2, int l); + static inline char INTERNAL_TO_LOWER(char c) + { + extern unsigned char internal_tolower_table_[]; + return internal_tolower_table_[(unsigned char)c]; + } +}; +static inline int is_string_z(const DTCBinary &a) +{ + return a.len > 0 && a.ptr[a.len - 1] == '\0'; +} // NULL ENDED +static inline int is_string_z(const DTCValue &a) +{ + return is_string_z(a.str); +} + +static inline int string_equal(const DTCBinary &a, const char *b) +{ + return !mystrcmp(a.ptr, b, a.len); +} +static inline int string_equal(const char *a, const DTCBinary &b) +{ + return string_equal(b, a); +} +static inline int string_equal(const DTCValue &a, const char *b) +{ + return string_equal(a.str, b); +} +static inline int string_equal(const char *a, const DTCValue &b) +{ + return string_equal(a, b.str); +} + +static inline int string_equal(const DTCBinary &a, const DTCBinary &b) +{ + return a.len == b.len && !mystrcmp(a.ptr, b.ptr, a.len); +} +static inline int string_equal(const DTCValue &a, const DTCBinary &b) +{ + return string_equal(a.str, b); +} +static inline int string_equal(const DTCBinary &a, const DTCValue &b) +{ + return string_equal(a, b.str); +} +static inline int string_equal(const DTCValue &a, const DTCValue &b) +{ + return string_equal(a.str, b.str); +} + +static inline int binary_equal(const DTCValue &a, const DTCValue &b) +{ + return a.bin.len == b.bin.len && !memcmp(a.bin.ptr, b.bin.ptr, a.bin.len); +} + +#include + +static inline int FIELD_ISSET(int id, const uint8_t *mask) +{ + return FD_ISSET(id, (const fd_set *)mask); +} + +static inline void FIELD_SET(int id, uint8_t *mask) +{ + FD_SET(id, (fd_set *)mask); +} +static inline void FIELD_CLR(int id, uint8_t *mask) +{ + FD_CLR(id, (fd_set *)mask); +} + +#define FIELD_ZERO(x) memset(x, 0, sizeof(x)) + +struct BufferChain +{ + inline void Count(int &n, int &l) + { + n = l = 0; + BufferChain *bc = this; + while (bc) + { + n++; + l += bc->usedBytes; + bc = bc->nextBuffer; + } + } + /* reset all data's used bytes, total bytes not changed */ + inline void ChainClean() + { + BufferChain *bc = this; + while (bc) + { + bc->usedBytes = 0; + bc = bc->nextBuffer; + } + } + inline void Clean() + { + nextBuffer = NULL; + usedBytes = 0; + } + BufferChain *nextBuffer; + int totalBytes; + int usedBytes; + char data[0]; +}; + +#endif diff --git a/src/search_local/index_storage/common/version.c b/src/search_local/index_storage/common/version.c new file mode 100644 index 0000000..4d17480 --- /dev/null +++ b/src/search_local/index_storage/common/version.c @@ -0,0 +1,67 @@ +/* + * ===================================================================================== + * + * Filename: version.c + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include "version.h" + +const char compdatestr[] = __DATE__; +const char comptimestr[] = __TIME__; +const char version[] = DTC_VERSION; +const char version_detail[] = DTC_VERSION_DETAIL; +long compile_time; + +__attribute__((constructor)) void init_comptime(void) +{ + struct tm tm; + switch (compdatestr[0]) + { + case 'A': + tm.tm_mon = compdatestr[1] == 'p' ? 3 /*Apr*/ : 7 /*Aug*/; + break; + case 'D': + tm.tm_mon = 11; + break; //Dec + case 'F': + tm.tm_mon = 1; + break; //Feb + case 'J': + tm.tm_mon = compdatestr[1] == 'a' ? 0 /*Jan*/ : compdatestr[2] == 'n' ? 5 /*Jun*/ : 6 /*Jul*/; + break; + case 'M': + tm.tm_mon = compdatestr[2] == 'r' ? 2 /*Mar*/ : 4 /*May*/; + break; + case 'N': + tm.tm_mon = 10; /*Nov*/ + break; + case 'S': + tm.tm_mon = 8; /*Sep*/ + break; + case 'O': + tm.tm_mon = 9; /*Oct*/ + break; + default: + return; + } + + tm.tm_mday = strtol(compdatestr + 4, 0, 10); + tm.tm_year = strtol(compdatestr + 7, 0, 10) - 1900; + tm.tm_hour = strtol(comptimestr + 0, 0, 10); + tm.tm_min = strtol(comptimestr + 3, 0, 10); + tm.tm_sec = strtol(comptimestr + 6, 0, 10); + compile_time = mktime(&tm); +} diff --git a/src/search_local/index_storage/common/version.h b/src/search_local/index_storage/common/version.h new file mode 100644 index 0000000..0c6815d --- /dev/null +++ b/src/search_local/index_storage/common/version.h @@ -0,0 +1,37 @@ +/* + * ===================================================================================== + * + * Filename: version.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __DTC_VERSION_H +#define __DTC_VERSION_H + +#define DTC_MAJOR_VERSION 4 +#define DTC_MINOR_VERSION 7 +#define DTC_BETA_VERSION 1 + +/*major.minor.beta*/ +#define DTC_VERSION "4.7.1" +/* the following show line should be line 11 as line number is used in Makefile */ +#define DTC_GIT_VERSION "7b21244" +#define DTC_VERSION_DETAIL DTC_VERSION "." DTC_GIT_VERSION + +extern const char compdatestr[]; +extern const char comptimestr[]; +extern const char version[]; +extern const char version_detail[]; +extern long compile_time; + +#endif diff --git a/src/search_local/index_storage/common/wait_queue.h b/src/search_local/index_storage/common/wait_queue.h new file mode 100644 index 0000000..b9bd27b --- /dev/null +++ b/src/search_local/index_storage/common/wait_queue.h @@ -0,0 +1,122 @@ +/* + * ===================================================================================== + * + * Filename: wait_queue.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: qiulu, choulu@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __WAIT_MTQUEUE_H__ +#define __WAIT_MTQUEUE_H__ + +#include +#include "lqueue.h" + +// typename T must be simple data type with 1,2,4,8,16 bytes +template +class threading_wait_queue +{ +private: + typename LinkQueue::allocator alloc; + LinkQueue queue; + pthread_mutex_t lock; + pthread_cond_t wait; + volatile int stopping; + T stopval; + +private: + // lock management + inline void Lock(void) { pthread_mutex_lock(&lock); } + inline void Unlock(void) { pthread_mutex_unlock(&lock); } + + // queue management + inline void Wait(void) { pthread_cond_wait(&wait, &lock); } + inline void Wake(void) { pthread_cond_signal(&wait); } + inline void WakeAll(void) { pthread_cond_broadcast(&wait); } + +public: + inline threading_wait_queue() : queue(&alloc) + { + pthread_mutex_init(&lock, NULL); + pthread_cond_init(&wait, NULL); + stopping = 0; + } + inline ~threading_wait_queue() + { + pthread_cond_destroy(&wait); + pthread_mutex_destroy(&lock); + } + + inline int Stopping(void) { return stopping; } + inline void Stop(T val = 0) + { + stopping = 1; + stopval = val; + WakeAll(); + } + inline void Clear(void) + { + Lock(); + while (queue.Count() > 0) + { + queue.Pop(); + } + Unlock(); + } + inline T Pop(void) + { + T p; + Lock(); + while (!Stopping() && queue.Count() <= 0) + { + Wait(); + } + p = Stopping() ? stopval : queue.Pop(); + Unlock(); + return p; + } + inline int Push(T p) + { + int ret; + + Lock(); + ret = queue.Push(p); + Unlock(); + Wake(); + return ret; + } + inline int Unshift(T p) + { + int ret; + + Lock(); + ret = queue.Push(p); + Unlock(); + Wake(); + return ret; + } + inline int Count(void) + { + int qsz; + + Lock(); + qsz = queue.Count(); + Unlock(); + return qsz; + } + inline int queue_empty(void) + { + return Count() == 0; + } +}; + +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/Makefile b/src/search_local/index_storage/rocksdb_helper/Makefile new file mode 100644 index 0000000..a2bd38d --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/Makefile @@ -0,0 +1,52 @@ +#LIBNAME = dtcMonitor + +TARGETEXE = rocksdb_helper + +CC = g++ + +.SUFFIXES: .cc .h + +FLAGS = -g -std=c++11 -DROCKSDB_COMPILER=1 -DOMN_PLATFORM_UNIX $(Optimize) -Wl,--no-undefined -Xlinker -zmuldefs +#FLAGS = -O3 -DOMN_PLATFORM_UNIX $(Optimize) + +INCDIRS = -I.. -I../common -I../watchdog -I../cache -I../stat -I../../../3rdlib/rocksdb/include -I../../../3rdlib/rocksdb/logging -I../../../3rdlib/attr_api + +LIBDIR = -L../common -L../cache -L../../../3rdlib/rocksdb/lib +#EXLIBDIR = ../../../Lib + +OBJDIR = . + +CREATEOBJDIR = $(shell if [ -d $(OBJDIR) ]; then echo ""; else mkdir $(OBJDIR); fi) + +OUTPUTDIR = . + +LIBS = -lnsl -lpthread -lstdc++ -ldl -lz -lrocksdb -L../bin + +DLIBS = ../cache/libdtcd.a \ + ../common/libcommon.a \ + ../stat/libstat.a \ + ../watchdog/libwatchdog.a \ + ../../../3rdlib/attr_api/libattr_api_64.a \ + +# +# objects and targets +# +#OBJECTS = $(patsubst %.cc,$(OBJDIR)/%.o,$(wildcard *.cc)) +OBJECTS = $(wildcard *.cc) +OBJECTS:= $(filter-out comm_main.cc db_process_gauss.cc gauss_db_conn.cc, $(OBJECTS)) +OBJECTS:= $(patsubst %.cc,$(OBJDIR)/%.o,$(OBJECTS)) + +# +# Rules for normal comile and link +# +all:: $(TARGETEXE) + +$(TARGETEXE): $(OBJECTS) + $(CC) -rdynamic -Wl,--no-as-needed $(FLAGS) -o $(OBJDIR)/$(TARGETEXE) $^ $(LIBDIR) $(DLIBS) $(LIBS) $(DLIBS) +$(OBJECTS): $(OBJDIR)/%.o: %.cc + $(CREATEOBJDIR) + $(CC) -c -static $(FLAGS) $< $(INCDIRS) -o $@ + +clean: + rm -rf $(OBJDIR)/*.o $(OBJDIR)/$(TARGETEXE) + rm -rf ./tester/rocksdata/* diff --git a/src/search_local/index_storage/rocksdb_helper/adaptive_prefix_extractor.h b/src/search_local/index_storage/rocksdb_helper/adaptive_prefix_extractor.h new file mode 100644 index 0000000..8f99977 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/adaptive_prefix_extractor.h @@ -0,0 +1,119 @@ +/* + * ===================================================================================== + * + * Filename: adaptive_prefix_extractor.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ADAPTIVE_PREFIX_EXTRACTOR_H__ +#define __ADAPTIVE_PREFIX_EXTRACTOR_H__ + +#include "rocksdb/slice_transform.h" +#include "key_format.h" +#include "log.h" + +#include + +// prefix extractor for prefix search +class AdaptivePrefixExtractor : public SliceTransform +{ +private: + // static int mFixedKeyLen; // always encode to be 9 byte length + // static bool mFixedKey; + int mkey_type; // dtc key key type + // bool mKeyCaseSensitive; + +public: + AdaptivePrefixExtractor(int keyType) + { + // mFixedKey = (keyType != DField::String && keyType != DField::Binary); + mkey_type = keyType; + // mKeyCaseSensitive = (keyType != DField::String ? true : false); + } + + virtual const char *Name() const { return "AdaptivePrefixExtractor"; } + + // extract the prefix key base on customer define, and this function is used + // when create the bloom filter + virtual Slice Transform(const Slice &key) const + { + static bool printLog = true; + if (printLog) + { + log_error("use prefix extractor!!!!!!"); + printLog = false; + } + + // extract the key field as the prefix slice + int keyLen = key_format::get_field_len(key.data_, mkey_type); + return Slice(key.data_, (size_t)keyLen); +#if 0 + int prefixLen; + if ( mFixedKey ) + { + prefixLen = mFixedKeyLen; + } + else + { + // need to decode the key and extract the prefix + // key_format::decode_primary_key(key.ToString(), mkey_type, prefixPrimaryKey); + prefixLen = key_format::get_field_len(key.data(), mkey_type); + } + + std::string prefix(key.data(), prefixLen); + if ( !mKeyCaseSensitive ) + { + // translate the prefix string to lower + // std::transform(prefixPrimaryKey.begin(), prefixPrimaryKey.end(), prefixPrimaryKey.begin(), ::tolower); + std::transform(prefix.begin(), prefix.end(), prefix.begin(), ::tolower); + } + + // whether need to append the 4 bytes placeholder ??? + // prefix.append(); + // if ( mHasVirtualField ) fullKey.append(key_format::encode_bytes((uint64_t)0)); + + return prefix; + // return prefixPrimaryKey; +#endif + } + + // 1.before call 'Transform' function to do logic extract, rocksdb firstly call + // 'InDomain' to determine whether the key compatible with the logic specified + // in the Transform method. + // 2.inserted key always contains the prefix, so just return true + virtual bool InDomain(const Slice &key) const { return key.size_ > 0; } + + // This is currently not used and remains here for backward compatibility. + virtual bool InRange(const Slice & /*dst*/) const { return false; } + + virtual bool FullLengthEnabled(size_t *len) const + { + log_error("unexpected call !!!!!"); + + // unfixed dynamic key always return false to disable this function + return false; + } + + // this function is only used by customer user to check whether the 'prefixKey' + // can use the current extractor, always turn true + virtual bool SameResultWhenAppended(const Slice &prefixKey) const + { + return true; + } +}; + +// '1' for append delimiter '|', in future, remove the redundant '|' in key_format encode style +// int AdaptivePrefixExtractor::mFixedKeyLen = sizeof(uint64_t); +// bool AdaptivePrefixExtractor::mFixedKey = false; + +#endif // __ADAPTIVE_PREFIX_EXTRACTOR_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/comm_main.cc b/src/search_local/index_storage/rocksdb_helper/comm_main.cc new file mode 100644 index 0000000..5d1d787 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/comm_main.cc @@ -0,0 +1,392 @@ +/* + * ===================================================================================== + * + * Filename: comm_main.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "comm_process.h" +#include +#include +#include +#include +#include +#include + +const char service_file[] = "./helper-service.so"; +const char create_handle_name[] = "create_process"; +const char progname[] = "custom-helper"; +const char usage_argv[] = "machId addr [port]"; +char cacheFile[256] = CACHE_CONF_NAME; +char tableFile[256] = TABLE_CONF_NAME; + +static CreateHandle CreateHelper = NULL; +static CommHelper *helperProc; +static unsigned int procTimeout; + +class HelperMain +{ +public: + HelperMain(CommHelper *helper) : h(helper){}; + + void attach(DTCTask *task) { h->attach((void *)task); } + void init_title(int group, int role) { h->init_title(group, role); } + void set_title(const char *status) { h->SetTitle(status); } + const char *Name() { return h->Name(); } + int pre_init(int gid, int r) + { + if (dbConfig->machineCnt <= gid) + { + log_error("parse config error, machineCnt[%d] <= GroupID[%d]", dbConfig->machineCnt, gid); + return (-1); + } + h->GroupID = gid; + h->Role = r; + h->_dbconfig = dbConfig; + h->_tdef = gTableDef[0]; + h->_config = gConfig; + h->_server_string = dbConfig->mach[gid].role[r].addr; + h->logapi.init_target(h->_server_string); + return 0; + } + +private: + CommHelper *h; +}; + +static int load_service(const char *dll_file) +{ + void *dll; + + dll = dlopen(dll_file, RTLD_NOW | RTLD_GLOBAL); + if (dll == (void *)NULL) + { + log_crit("dlopen(%s) error: %s", dll_file, dlerror()); + return -1; + } + + CreateHelper = (CreateHandle)dlsym(dll, create_handle_name); + if (CreateHelper == NULL) + { + log_crit("function[%s] not found", create_handle_name); + return -2; + } + + return 0; +} + +static int sync_decode(DTCTask *task, int netfd, CommHelper *helperProc) +{ + SimpleReceiver receiver(netfd); + int code; + do + { + code = task->Decode(receiver); + if (code == DecodeFatalError) + { + if (errno != 0) + log_notice("decode fatal error, fd=%d, %m", netfd); + return -1; + } + if (code == DecodeDataError) + { + if (task->result_code() == 0 || task->result_code() == -EC_EXTRA_SECTION_DATA) // -EC_EXTRA_SECTION_DATA verify package + return 0; + log_notice("decode error, fd=%d, %d", netfd, task->result_code()); + return -1; + } + HelperMain(helperProc).set_title("Receiving..."); + } while (!stop && code != DecodeDone); + + if (task->result_code() < 0) + { + log_notice("register result, fd=%d, %d", netfd, task->result_code()); + return -1; + } + return 0; +} + +static int sync_send(Packet *reply, int netfd) +{ + int code; + do + { + code = reply->Send(netfd); + if (code == SendResultError) + { + log_notice("send error, fd=%d, %m", netfd); + return -1; + } + } while (!stop && code != SendResultDone); + + return 0; +} + +static void alarm_handler(int signo) +{ + if (background == 0 && getppid() == 1) + exit(0); + alarm(10); +} + +static int accept_connection(int fd) +{ + HelperMain(helperProc).set_title("listener"); + signal(SIGALRM, alarm_handler); + while (!stop) + { + alarm(10); + int newfd; + if ((newfd = accept(fd, NULL, 0)) >= 0) + { + alarm(0); + return newfd; + } + if (newfd < 0 && errno == EINVAL) + { + if (getppid() == (pid_t)1) + { // Ѿ˳ + log_error("dtc father process not exist. helper[%d] exit now.", getpid()); + exit(0); + } + usleep(10000); + } + } + exit(0); +} + +static void proc_timeout_handler(int signo) +{ + log_error("comm-helper process timeout(more than %u seconds), helper[pid: %d] exit now.", procTimeout, getpid()); + exit(-1); +} + +struct THelperProcParameter +{ + int netfd; + int gid; + int role; +}; + +static int helper_proc_run(struct THelperProcParameter *args) +{ + // close listen fd + close(0); + open("/dev/null", O_RDONLY); + + HelperMain(helperProc).set_title("Initializing..."); + + if (procTimeout > 0) + signal(SIGALRM, proc_timeout_handler); + + alarm(procTimeout); + if (helperProc->Init() != 0) + { + log_error("%s", "helper process init failed"); + exit(-1); + } + + alarm(0); + + unsigned int timeout; + + while (!stop) + { + HelperMain(helperProc).set_title("Waiting..."); + DTCTask *task = new DTCTask(gTableDef[0]); + if (sync_decode(task, args->netfd, helperProc) < 0) + { + delete task; + break; + } + + if (task->result_code() == 0) + { + switch (task->request_code()) + { + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + timeout = 2 * procTimeout; + default: + timeout = procTimeout; + } + HelperMain(helperProc).attach(task); + alarm(timeout); + helperProc->Execute(); + alarm(0); + } + + HelperMain(helperProc).set_title("Sending..."); + Packet *reply = new Packet; + reply->encode_result(task); + + delete task; + if (sync_send(reply, args->netfd) < 0) + { + delete reply; + break; + } + delete reply; + } + close(args->netfd); + HelperMain(helperProc).set_title("Exiting..."); + + delete helperProc; + daemon_cleanup(); +#if MEMCHECK + log_info("%s v%s: stopped", progname, version); + dump_non_delete(); + log_debug("memory allocated %lu virtual %lu", count_alloc_size(), count_virtual_size()); +#endif + exit(0); + return 0; +} + +int main(int argc, char **argv) +{ + init_proc_title(argc, argv); + if (dtc_daemon_init(argc, argv) < 0) + return -1; + + argc -= optind; + argv += optind; + + struct THelperProcParameter helperArgs = {0, 0, 0}; + char *addr = NULL; + + if (argc > 0) + { + char *p; + helperArgs.gid = strtol(argv[0], &p, 0); + if (*p == '\0' || *p == MACHINEROLESTRING[0]) + helperArgs.role = 0; + else if (*p == MACHINEROLESTRING[1]) + helperArgs.role = 1; + else + { + log_error("Bad machine id: %s", argv[0]); + return -1; + } + } + + if (argc != 2 && argc != 3) + { + show_usage(); + return -1; + } + + int backlog = gConfig->get_int_val("cache", "MaxListenCount", 256); + int helperTimeout = gConfig->get_int_val("cache", "HelperTimeout", 30); + if (helperTimeout > 1) + procTimeout = helperTimeout - 1; + else + procTimeout = 0; + addr = argv[1]; + + // load dll file + const char *file = getenv("HELPER_SERVICE_FILE"); + if (file == NULL || file[0] == 0) + file = service_file; + if (load_service(file) != 0) + return -1; + + helperProc = CreateHelper(); + if (helperProc == NULL) + { + log_crit("create helper error"); + return -1; + } + if (HelperMain(helperProc).pre_init(helperArgs.gid, helperArgs.role) < 0) + { + log_error("%s", "helper prepare init failed"); + exit(-1); + } + if (helperProc->global_init() != 0) + { + log_crit("helper gobal-init error"); + return -1; + } + + int fd = -1; + if (!strcmp(addr, "-")) + fd = 0; + else + { + if (strcasecmp(gConfig->get_str_val("cache", "CacheShmKey") ?: "", "none") != 0) + { + log_warning("standalone %s need CacheShmKey set to NONE", progname); + return -1; + } + + SocketAddress sockaddr; + const char *err = sockaddr.set_address(addr, argc == 2 ? NULL : argv[2]); + if (err) + { + log_warning("host %s port %s: %s", addr, argc == 2 ? "NULL" : argv[2], err); + return -1; + } + if (sockaddr.socket_type() != SOCK_STREAM) + { + log_warning("standalone %s don't support UDP protocol", progname); + return -1; + } + fd = sock_bind(&sockaddr, backlog); + if (fd < 0) + return -1; + } + + daemon_start(); + +#if HAS_LOGAPI + helperProc->logapi.Init( + gConfig->get_int_val("LogApi", "MessageId", 0), + gConfig->get_int_val("LogApi", "CallerId", 0), + gConfig->get_int_val("LogApi", "TargetId", 0), + gConfig->get_int_val("LogApi", "InterfaceId", 0)); +#endif + + HelperMain(helperProc).init_title(helperArgs.gid, helperArgs.role); + if (procTimeout > 1) + helperProc->set_proc_timeout(procTimeout - 1); + while (!stop) + { + helperArgs.netfd = accept_connection(fd); + + watch_dog_fork(HelperMain(helperProc).Name(), (int (*)(void *))helper_proc_run, (void *)&helperArgs); + + close(helperArgs.netfd); + } + + if (fd > 0 && addr && addr[0] == '/') + unlink(addr); + return 0; +} diff --git a/src/search_local/index_storage/rocksdb_helper/comm_process.cc b/src/search_local/index_storage/rocksdb_helper/comm_process.cc new file mode 100644 index 0000000..7982589 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/comm_process.cc @@ -0,0 +1,320 @@ +/* + * ===================================================================================== + * + * Filename: comm_process.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm_process.h" +#include +#include +#include +#include + +#include "proc_title.h" + +#define CAST(type, var) type *var = (type *)addr + +CommHelper::CommHelper() : addr(NULL), check(0) +{ + Timeout = 0; +} + +CommHelper::~CommHelper() +{ + addr = NULL; +} + +CommHelper::CommHelper(CommHelper &proc) +{ +} + +void CommHelper::init_title(int group, int role) +{ + snprintf(_name, sizeof(_name), + "helper%d%c", + group, + MACHINEROLESTRING[role]); + + snprintf(_title, sizeof(_title), "%s: ", _name); + _titlePrefixSize = strlen(_title); + _title[sizeof(_title) - 1] = '\0'; +} + +void CommHelper::set_title(const char *status) +{ + strncpy(_title + _titlePrefixSize, status, sizeof(_title) - 1 - _titlePrefixSize); + set_proc_title(_title); +} + +int CommHelper::global_init(void) +{ + return 0; +} + +int CommHelper::Init(void) +{ + return 0; +} + +int CommHelper::Execute() +{ + int ret = -1; + + log_debug("request code: %d", request_code()); + + switch (request_code()) + { + /* Nop/Purge/Flush⼸Դ˵ҪӦҲ */ + case DRequest::Nop: + case DRequest::Purge: + case DRequest::Flush: + return 0; + + case DRequest::Get: + logapi.Start(); + ret = process_get(); + logapi.Done(__FILE__, __LINE__, "SELECT", ret, ret); + break; + + case DRequest::Insert: + logapi.Start(); + ret = process_insert(); + logapi.Done(__FILE__, __LINE__, "INSERT", ret, ret); + break; + + case DRequest::Update: + logapi.Start(); + ret = process_update(); + logapi.Done(__FILE__, __LINE__, "UPDATE", ret, ret); + break; + + case DRequest::Delete: + logapi.Start(); + ret = process_delete(); + logapi.Done(__FILE__, __LINE__, "DELETE", ret, ret); + break; + + case DRequest::Replace: + logapi.Start(); + ret = process_replace(); + logapi.Done(__FILE__, __LINE__, "REPLACE", ret, ret); + break; + + default: + log_error("unknow request code"); + set_error(-EC_BAD_COMMAND, __FUNCTION__, "[Helper]invalid request-code"); + return -1; + }; + + return ret; +} + +void CommHelper::Attach(void *p) +{ + addr = p; +} + +DTCTableDefinition *CommHelper::Table(void) const +{ + CAST(DTCTask, task); + return task->table_definition(); +} + +const PacketHeader *CommHelper::Header() const +{ +#if 0 + CAST(DTCTask, task); + return &(task->headerInfo); +#else + // NO header anymore + return NULL; +#endif +} + +const DTCVersionInfo *CommHelper::version_info() const +{ + CAST(DTCTask, task); + return &(task->versionInfo); +} + +int CommHelper::request_code() const +{ + CAST(DTCTask, task); + return task->request_code(); +} + +int CommHelper::has_request_key() const +{ + CAST(DTCTask, task); + return task->has_request_key(); +} + +const DTCValue *CommHelper::request_key() const +{ + CAST(DTCTask, task); + return task->request_key(); +} + +unsigned int CommHelper::int_key() const +{ + CAST(DTCTask, task); + return task->int_key(); +} + +const DTCFieldValue *CommHelper::request_condition() const +{ + CAST(DTCTask, task); + return task->request_condition(); +} + +const DTCFieldValue *CommHelper::request_operation() const +{ + CAST(DTCTask, task); + return task->request_operation(); +} + +const DTCFieldSet *CommHelper::request_fields() const +{ + CAST(DTCTask, task); + return task->request_fields(); +} + +void CommHelper::set_error(int err, const char *from, const char *msg) +{ + CAST(DTCTask, task); + task->set_error(err, from, msg); +} + +int CommHelper::count_only(void) const +{ + CAST(DTCTask, task); + return task->count_only(); +} + +int CommHelper::all_rows(void) const +{ + CAST(DTCTask, task); + return task->all_rows(); +} + +int CommHelper::update_row(RowValue &row) +{ + CAST(DTCTask, task); + return task->update_row(row); +} + +int CommHelper::compare_row(const RowValue &row, int iCmpFirstNField) const +{ + CAST(DTCTask, task); + return task->compare_row(row, iCmpFirstNField); +} + +int CommHelper::prepare_result(void) +{ + CAST(DTCTask, task); + return task->prepare_result(); +} + +void CommHelper::update_key(RowValue *r) +{ + CAST(DTCTask, task); + task->update_key(r); +} + +void CommHelper::update_key(RowValue &r) +{ + update_key(&r); +} + +int CommHelper::set_total_rows(unsigned int nr) +{ + CAST(DTCTask, task); + return task->set_total_rows(nr); +} + +int CommHelper::set_affected_rows(unsigned int nr) +{ + CAST(DTCTask, task); + task->resultInfo.set_affected_rows(nr); + return 0; +} + +int CommHelper::append_row(const RowValue &r) +{ + return append_row(&r); +} + +int CommHelper::append_row(const RowValue *r) +{ + CAST(DTCTask, task); + int ret = task->append_row(r); + if (ret > 0) + ret = 0; + return ret; +} + +/* ѯַ */ +const char *CommHelper::get_server_address(void) const +{ + return _server_string; +} + +/* ѯļ */ +int CommHelper::get_int_val(const char *sec, const char *key, int def) +{ + return _config->get_int_val(sec, key, def); +} + +unsigned long long CommHelper::get_size_val(const char *sec, const char *key, unsigned long long def, char unit) +{ + return _config->get_size_val(sec, key, def, unit); +} + +int CommHelper::get_idx_val(const char *v1, const char *v2, const char *const *v3, int v4) +{ + return _config->get_idx_val(v1, v2, v3, v4); +} + +const char *CommHelper::get_str_val(const char *sec, const char *key) +{ + return _config->get_str_val(sec, key); +} + +bool CommHelper::has_section(const char *sec) +{ + return _config->has_section(sec); +} + +bool CommHelper::has_key(const char *sec, const char *key) +{ + return _config->has_key(sec, key); +} + +/* ѯ */ +int CommHelper::field_type(int n) const { return _tdef->field_type(n); } +int CommHelper::field_size(int n) const { return _tdef->field_size(n); } +int CommHelper::field_id(const char *n) const { return _tdef->field_id(n); } +const char *CommHelper::field_name(int id) const { return _tdef->field_name(id); } +int CommHelper::num_fields(void) const { return _tdef->num_fields(); } +int CommHelper::key_fields(void) const { return _tdef->key_fields(); } diff --git a/src/search_local/index_storage/rocksdb_helper/comm_process.h b/src/search_local/index_storage/rocksdb_helper/comm_process.h new file mode 100644 index 0000000..214294e --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/comm_process.h @@ -0,0 +1,194 @@ +/* + * ===================================================================================== + * + * Filename: comm_process.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __COMM_PROCESS_H__ +#define __COMM_PROCESS_H__ + +#include +#include +#include +#include "helper_log_api.h" + +DTC_BEGIN_NAMESPACE + +class CommHelper; +class HelperMain; +class DTCTableDefinition; +class DbConfig; +class RowValue; +class PacketHeader; +class DTCVersionInfo; +class DTCConfig; +union DTCValue; +class DTCFieldValue; +class DTCFieldSet; + +typedef CommHelper *(*CreateHandle)(void); + +/* helperĻ +ʹҪ̳࣬ʵglobal_init()Init()Execute()process_get()virtualӿڡ +*/ +class CommHelper +{ +protected: + int GroupID; + int Role; + int Timeout; + +public: + // 캯Ҫserverӣڹ캯ӡҪInitExecuteִС + CommHelper(); + virtual ~CommHelper(); + + // Ҫʵֵ3 + /* helperһ顣fork֮ǰõ */ + virtual int global_init(void); + + /* fork֮õ */ + virtual int Init(void); + + /* ÿһ */ + virtual int Execute(); + + /* һijʱʱ λΪ*/ + void set_proc_timeout(int timeout) { Timeout = timeout; } + + friend class HelperMain; + +protected: + /* GetĽӿں */ + virtual int process_get() = 0; + + /* Insertӿں */ + virtual int process_insert() = 0; + + /* Deleteӿں */ + virtual int process_delete() = 0; + + /* Updateӿں */ + virtual int process_update() = 0; + + /* Replaceӿں */ + virtual int process_replace() = 0; + + /* ѯַ */ + const char *get_server_address(void) const; + + /* ѯļ */ + int get_int_val(const char *sec, const char *key, int def = 0); + unsigned long long get_size_val(const char *sec, const char *key, unsigned long long def = 0, char unit = 0); + int get_idx_val(const char *, const char *, const char *const *, int = 0); + const char *get_str_val(const char *sec, const char *key); + bool has_section(const char *sec); + bool has_key(const char *sec, const char *key); + + /* ѯ */ + DTCTableDefinition *Table(void) const; + int field_type(int n) const; + int field_size(int n) const; + int field_id(const char *n) const; + const char *field_name(int id) const; + int num_fields(void) const; + int key_fields(void) const; + + /* ȡͷϢ */ + const PacketHeader *Header() const; + + /* ȡϢ */ + const DTCVersionInfo *version_info() const; + + /* */ + int request_code() const; + + /* Ƿkeyֵkeyinsertûkeyֵ */ + int has_request_key() const; + + /* keyֵ */ + const DTCValue *request_key() const; + + /* keyֵ */ + unsigned int int_key() const; + + /* where */ + const DTCFieldValue *request_condition() const; + + /* updateinsertreplaceĸϢ */ + const DTCFieldValue *request_operation() const; + + /* getselectֶб */ + const DTCFieldSet *request_fields() const; + + /* ôϢ롢طϸϢ */ + void set_error(int err, const char *from, const char *msg); + + /* Ƿֻselect count(*)Ҫؽ */ + int count_only(void) const; + + /* Ƿûwhere */ + int all_rows(void) const; + + /* еĸrowͨrowһһӿڸ£Ȼ±ݲ㡣Լrequest_operation£Ҫӿڣ*/ + int update_row(RowValue &row); + + /* ȽеrowǷָֻȽǰnֶ */ + int compare_row(const RowValue &row, int iCmpFirstNField = 256) const; + + /* һǰȵӿڳʼ */ + int prepare_result(void); + + /* keyֵƵr */ + void update_key(RowValue *r); + + /* keyֵƵr */ + void update_key(RowValue &r); + + /* select * from table limit m,ntotal-rowsһȽ */ + int set_total_rows(unsigned int nr); + + /* ʵʸµ */ + int set_affected_rows(unsigned int nr); + + /* һ */ + int append_row(const RowValue &r); + int append_row(const RowValue *r); + +private: + void *addr; + long check; + char _name[16]; + char _title[80]; + int _titlePrefixSize; + const char *_server_string; + const DbConfig *_dbconfig; + DTCConfig *_config; + DTCTableDefinition *_tdef; + +private: + CommHelper(CommHelper &); + void Attach(void *); + + void init_title(int group, int role); + void set_title(const char *status); + const char *Name(void) const { return _name; } + +public: + HelperLogApi logapi; +}; + +DTC_END_NAMESPACE + +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/db_main.cc b/src/search_local/index_storage/rocksdb_helper/db_main.cc new file mode 100644 index 0000000..b1d25e9 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/db_main.cc @@ -0,0 +1,667 @@ +/* + * ===================================================================================== + * + * Filename: db_main.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#if 1 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "table_def_manager.h" +#include +#include +#include +#include +#include +#include "dtcutils.h" + +#include "rocksdb_conn.h" +#include "db_process_rocks.h" +#include "rocksdb_direct_process.h" + +extern void _set_remote_log_config_(const char *addr, int port, int businessid); +const char progname[] = "rocksdb_helper"; +const char usage_argv[] = "machId addr [port]"; +char cacheFile[256] = CACHE_CONF_NAME; +char tableFile[256] = TABLE_CONF_NAME; + +static HelperProcessBase *helperProc; +static unsigned int procTimeout; + +static RocksDBConn *gRocksdbConn; +std::string gRocksdbPath = "../rocksdb_data"; +std::string gRocksDirectAccessPath = "/tmp/domain_socket/"; +static RocksdbDirectProcess *gRocksdbDirectProcess; + +int targetNewHash; +int hashChanging; + +static int sync_decode(DTCTask *task, int netfd, HelperProcessBase *helperProc) +{ + SimpleReceiver receiver(netfd); + int code; + do + { + code = task->Decode(receiver); + if (code == DecodeFatalError) + { + if (errno != 0) + log_notice("decode fatal error, fd=%d, %m", netfd); + log_info("decode error!!!!!"); + return -1; + } + if (code == DecodeDataError) + { + if (task->result_code() == 0 || task->result_code() == -EC_EXTRA_SECTION_DATA) // -EC_EXTRA_SECTION_DATA verify package + return 0; + log_notice("decode error, fd=%d, %d", netfd, task->result_code()); + return -1; + } + helperProc->set_title("Receiving..."); + } while (!stop && code != DecodeDone); + + if (task->result_code() < 0) + { + log_notice("register result, fd=%d, %d", netfd, task->result_code()); + return -1; + } + return 0; +} + +static int sync_send(Packet *reply, int netfd) +{ + int code; + do + { + code = reply->Send(netfd); + if (code == SendResultError) + { + log_notice("send error, fd=%d, %m", netfd); + return -1; + } + } while (!stop && code != SendResultDone); + + return 0; +} + +static void alarm_handler(int signo) +{ + if (background == 0 && getppid() == 1) + exit(0); + alarm(10); +} + +static int accept_connection(int fd) +{ + helperProc->set_title("listener"); + signal(SIGALRM, alarm_handler); + while (!stop) + { + alarm(10); + int newfd; + if ((newfd = accept(fd, NULL, 0)) >= 0) + { + alarm(0); + return newfd; + } + if (newfd < 0 && errno == EINVAL) + { + if (getppid() == (pid_t)1) + { // Ѿ˳ + log_error("dtc father process not exist. helper[%d] exit now.", getpid()); + exit(0); + } + log_info("parent process close the connection!"); + usleep(10000); + } + } + exit(0); +} + +static void proc_timeout_handler(int signo) +{ + log_error("mysql process timeout(more than %u seconds), helper[pid: %d] exit now.", procTimeout, getpid()); + exit(-1); +} + +#ifdef __DEBUG__ +static void inline simulate_helper_delay(void) +{ + char *env = getenv("ENABLE_SIMULATE_DTC_HELPER_DELAY_SECOND"); + if (env && env[0] != 0) + { + unsigned delay_sec = atoi(env); + if (delay_sec > 5) + delay_sec = 5; + + log_debug("simulate dtc helper delay second[%d s]", delay_sec); + sleep(delay_sec); + } + return; +} +#endif + +struct THelperProcParameter +{ + int netfd; + int gid; + int role; +}; + +static int helper_proc_run(struct THelperProcParameter *args) +{ + // close listen fd + close(0); + open("/dev/null", O_RDONLY); + + helperProc->set_title("Initializing..."); + + if (procTimeout > 0) + signal(SIGALRM, proc_timeout_handler); + + alarm(procTimeout); + if (helperProc->Init(args->gid, dbConfig, TableDefinitionManager::Instance()->get_cur_table_def(), args->role) != 0) + { + log_error("%s", "helper process init failed"); + exit(-1); + } + + helperProc->init_ping_timeout(); + alarm(0); + + _set_remote_log_config_(gConfig->get_str_val("cache", "RemoteLogAddr"), + gConfig->get_int_val("cache", "RemoteLogPort", 0), + dtc::utils::get_bid()); + + _set_remote_log_fd_(); + + hashChanging = gConfig->get_int_val("cache", "HashChanging", 0); + targetNewHash = gConfig->get_int_val("cache", "TargetNewHash", 0); + + unsigned int timeout; + + while (!stop) + { + helperProc->set_title("Waiting..."); + DTCTask *task = new DTCTask(TableDefinitionManager::Instance()->get_cur_table_def()); + if (sync_decode(task, args->netfd, helperProc) < 0) + { + log_info("sync decode failed!"); + delete task; + break; + } + + if (task->result_code() == 0) + { + switch (task->request_code()) + { + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + case DRequest::ReloadConfig: + case DRequest::Replicate: + case DRequest::LocalMigrate: + timeout = 2 * procTimeout; + default: + timeout = procTimeout; + } + alarm(timeout); +#ifdef __DEBUG__ + simulate_helper_delay(); +#endif + helperProc->process_task(task); + alarm(0); + } + + helperProc->set_title("Sending..."); + Packet *reply = new Packet; + reply->encode_result(task); + + if (sync_send(reply, args->netfd) < 0) + { + delete reply; + delete task; + break; + log_info("sync send failed!"); + } + delete reply; + delete task; + } + close(args->netfd); + helperProc->set_title("Exiting..."); + + delete helperProc; + daemon_cleanup(); +#if MEMCHECK + log_info("%s v%s: stopped", progname, version); + dump_non_delete(); + log_debug("memory allocated %lu virtual %lu", count_alloc_size(), count_virtual_size()); +#endif + log_info("helper exit!"); + + exit(0); + return 0; +} + +static int helper_proc_run_rocks(struct THelperProcParameter args) +{ + log_info("xx77xx11 test multiple thread model! threadId:%d, fd:%d", std::this_thread::get_id(), args.netfd); + + open("/dev/null", O_RDONLY); + + helperProc->set_title("Initializing..."); + + if (procTimeout > 0) + signal(SIGALRM, proc_timeout_handler); + + alarm(procTimeout); + + // helperProc->init_ping_timeout(); + alarm(0); + + _set_remote_log_config_(gConfig->get_str_val("cache", "RemoteLogAddr"), + gConfig->get_int_val("cache", "RemoteLogPort", 0), + dtc::utils::get_bid()); + + _set_remote_log_fd_(); + + hashChanging = gConfig->get_int_val("cache", "HashChanging", 0); + targetNewHash = gConfig->get_int_val("cache", "TargetNewHash", 0); + + unsigned int timeout; + + while (!stop) + { + helperProc->set_title("Waiting..."); + DTCTask *task = new DTCTask(TableDefinitionManager::Instance()->get_cur_table_def()); + if (sync_decode(task, args.netfd, helperProc) < 0) + { + log_info("sync decode failed!"); + delete task; + break; + } + + log_info("recieve request, threadId:%d", std::this_thread::get_id()); + + if (task->result_code() == 0) + { + switch (task->request_code()) + { + case DRequest::Insert: + case DRequest::Update: + case DRequest::Delete: + case DRequest::Replace: + case DRequest::ReloadConfig: + case DRequest::Replicate: + case DRequest::LocalMigrate: + timeout = 2 * procTimeout; + default: + timeout = procTimeout; + } + alarm(timeout); +#ifdef __DEBUG__ + simulate_helper_delay(); +#endif + helperProc->process_task(task); + alarm(0); + } + + helperProc->set_title("Sending..."); + Packet *reply = new Packet; + reply->encode_result(task); + + if (sync_send(reply, args.netfd) < 0) + { + delete reply; + delete task; + break; + log_info("sync send failed!"); + } + delete reply; + delete task; + } + close(args.netfd); + helperProc->set_title("Exiting..."); + + daemon_cleanup(); +#if MEMCHECK + log_info("%s v%s: stopped", progname, version); + dump_non_delete(); + log_debug("memory allocated %lu virtual %lu", count_alloc_size(), count_virtual_size()); +#endif + log_info("helper exit!"); + + return 0; +} + +// check version base on DB type +int check_db_version(void) +{ + // dbConfig->dstype = 1; + switch (dbConfig->dstype) + { + case 0: + default: + case 2: + { + log_debug("no need to check rocksdb!"); + // checker guass db version + break; + } + } + + return -1; +} + +int check_db_table(int gid, int role) +{ + HelperProcessBase *helper; + switch (dbConfig->dstype) + { + case 0: + default: + case 2: + // no table concept in rocksdb, no need to check + log_error("no need to check table in rocksdb storage!"); + return 0; + } + + if (procTimeout > 1) + { + helper->set_proc_timeout(procTimeout - 1); + signal(SIGALRM, proc_timeout_handler); + } + + alarm(procTimeout); + log_debug("begin initialize gauss db"); + if (helper->Init(gid, dbConfig, TableDefinitionManager::Instance()->get_cur_table_def(), role) != 0) + { + log_error("%s", "helper process init failed"); + delete helper; + alarm(0); + return (-1); + } + + if (helper->check_table() != 0) + { + delete helper; + alarm(0); + return (-2); + } + + alarm(0); + delete helper; + + return (0); +} + +int create_rocks_domain_dir() +{ + // create domain socket directory + int ret = access(gRocksDirectAccessPath.c_str(), F_OK); + if (ret != 0) + { + int err = errno; + if (errno == ENOENT) + { + // create log dir + if (mkdir(gRocksDirectAccessPath.c_str(), 0755) != 0) + { + log_error("create rocksdb domain socket dir failed! path:%s, errno:%d", gRocksDirectAccessPath.c_str(), errno); + return -1; + } + } + else + { + log_error("access rocksdb domain socket dir failed!, path:%s, errno:%d", gRocksDirectAccessPath.c_str(), errno); + return -1; + } + } + + return 0; +} + +// process rocksdb direct access +int rocks_direct_access_proc() +{ + log_error("Rocksdb direct access channel open!"); + + std::string socketPath = gRocksDirectAccessPath; + std::string dtcDeployAddr = dbConfig->cfgObj->get_str_val("cache", "BindAddr"); + + SocketAddress sockAddr; + const char *strRet = sockAddr.set_address(dtcDeployAddr.c_str()); + if (strRet) + { + log_error("parse dtc bind addr failed, errmsg:%s", strRet); + return -1; + } + + int dtcDeployPort; + switch (sockAddr.addr->sa_family) + { + case AF_INET: + dtcDeployPort = ntohs(sockAddr.in4->sin_port); + break; + case AF_INET6: + dtcDeployPort = ntohs(sockAddr.in6->sin6_port); + break; + default: + log_error("unsupport addr type! addr:%s, type:%d", dtcDeployAddr.c_str(), sockAddr.addr->sa_family); + return -1; + } + assert(dtcDeployPort > 0); + + socketPath.append("rocks_direct_").append(std::to_string(dtcDeployPort)).append(".sock"); + + gRocksdbDirectProcess = new RocksdbDirectProcess(socketPath, helperProc); + if (!gRocksdbDirectProcess) + { + log_error("create RocksdbDirectProcess failed!"); + return -1; + } + + int ret = gRocksdbDirectProcess->init(); + if (ret != 0) + return -1; + + return gRocksdbDirectProcess->run_process(); +} + +int main(int argc, char **argv) +{ + init_proc_title(argc, argv); + if (dtc_daemon_init(argc, argv) < 0) + return -1; + + check_db_version(); + argc -= optind; + argv += optind; + + struct THelperProcParameter helperArgs = {0, 0, 0}; + char *addr = NULL; + + if (argc > 0) + { + char *p; + helperArgs.gid = strtol(argv[0], &p, 0); + if (*p == '\0' || *p == MACHINEROLESTRING[0]) + helperArgs.role = 0; + else if (*p == MACHINEROLESTRING[1]) + helperArgs.role = 1; + else + { + log_error("Bad machine id: %s", argv[0]); + return -1; + } + } + + if (argc != 2 && argc != 3) + { + show_usage(); + return -1; + } + + int usematch = gConfig->get_int_val("cache", "UseMatchedAsAffectedRows", 1); + int backlog = gConfig->get_int_val("cache", "MaxListenCount", 256); + int helperTimeout = gConfig->get_int_val("cache", "HelperTimeout", 30); + if (helperTimeout > 1) + procTimeout = helperTimeout - 1; + else + procTimeout = 0; + addr = argv[1]; + log_error("helper listen addr:%s", addr); + + if (dbConfig->checkTable && check_db_table(helperArgs.gid, helperArgs.role) != 0) + { + return -1; + } + + int fd = -1; + if (!strcmp(addr, "-")) + fd = 0; + else + { + if (strcasecmp(gConfig->get_str_val("cache", "CacheShmKey") ?: "", "none") != 0) + { + log_warning("standalone %s need CacheShmKey set to NONE", progname); + return -1; + } + + SocketAddress sockaddr; + const char *err = sockaddr.set_address(addr, argc == 2 ? NULL : argv[2]); + if (err) + { + log_warning("host %s port %s: %s", addr, argc == 2 ? "NULL" : argv[2], err); + return -1; + } + if (sockaddr.socket_type() != SOCK_STREAM) + { + log_warning("standalone %s don't support UDP protocol", progname); + return -1; + } + fd = sock_bind(&sockaddr, backlog); + if (fd < 0) + return -1; + } + log_info("helper listen fd:%d", fd); + + log_debug("If you want to simulate db busy," + "you can set \"ENABLE_SIMULATE_DTC_HELPER_DELAY_SECOND=second\" before dtc startup"); + + daemon_start(); + + // create helper instance base on database type + switch (dbConfig->dstype) + { + default: + case 2: + { + // rocksdb + gRocksdbConn = RocksDBConn::Instance(); + helperProc = new RocksdbProcess(gRocksdbConn); + assert(helperProc); + + int ret = helperProc->Init(helperArgs.gid, dbConfig, TableDefinitionManager::Instance()->get_cur_table_def(), helperArgs.role); + if (ret != 0) + { + log_error("%s", "helper process init failed"); + return -1; + } + + gRocksdbConn->set_key_type(TableDefinitionManager::Instance()->get_cur_table_def()->key_type()); + ret = gRocksdbConn->Open(gRocksdbPath); + assert(ret == 0); + + // start direct rocksdb access channel + ret = create_rocks_domain_dir(); + if (ret != 0) + return -1; + + ret = rocks_direct_access_proc(); + if (ret != 0) + return -1; + + break; + } + } + + if (usematch) + helperProc->use_matched_rows(); +#if HAS_LOGAPI + helperProc->logapi.Init( + gConfig->get_int_val("LogApi", "MessageId", 0), + gConfig->get_int_val("LogApi", "CallerId", 0), + gConfig->get_int_val("LogApi", "TargetId", 0), + gConfig->get_int_val("LogApi", "InterfaceId", 0)); +#endif + + helperProc->init_title(helperArgs.gid, helperArgs.role); + if (procTimeout > 1) + helperProc->set_proc_timeout(procTimeout - 1); + while (!stop) + { + helperArgs.netfd = accept_connection(fd); + char buf[16]; + memset(buf, 0, 16); + buf[0] = WATCHDOG_INPUT_OBJECT; + log_info("xx77xx11 procName:%s", helperProc->Name()); + snprintf(buf + 1, 15, "%s", helperProc->Name()); + + log_info("fork child helper! fd:%d", helperArgs.netfd); + + if (dbConfig->dstype != 2) + { + watch_dog_fork(buf, (int (*)(void *))helper_proc_run, (void *)&helperArgs); + close(helperArgs.netfd); + } + else + { + // rocksdb use multiple thread mode + std::thread runner(helper_proc_run_rocks, helperArgs); + runner.detach(); + } + } + + /* close global rocksdb connection */ + if (gRocksdbConn) + { + int ret = gRocksdbConn->Close(); + if (ret != 0) + { + log_error("close rocksdb connection failed, rockspath:%s", gRocksdbPath.c_str()); + } + } + + log_info("helper main process exist!"); + if (fd > 0 && addr && addr[0] == '/') + unlink(addr); + return 0; +} +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/db_process_base.h b/src/search_local/index_storage/rocksdb_helper/db_process_base.h new file mode 100644 index 0000000..88ef24b --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/db_process_base.h @@ -0,0 +1,63 @@ +/* + * ===================================================================================== + * + * Filename: db_process_base.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __HELPER_PROCESS_BASE_H__ +#define __HELPER_PROCESS_BASE_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include "helper_log_api.h" + +class DirectRequestContext; +class DirectResponseContext; + +class HelperProcessBase +{ +public: + HelperLogApi logapi; + +public: + virtual ~HelperProcessBase() {} + + virtual void use_matched_rows(void) = 0; + virtual int Init(int GroupID, const DbConfig *Config, DTCTableDefinition *tdef, int slave) = 0; + virtual void init_ping_timeout(void) = 0; + virtual int check_table() = 0; + + virtual int process_task(DTCTask *Task) = 0; + + virtual void init_title(int m, int t) = 0; + virtual void set_title(const char *status) = 0; + virtual const char *Name(void) = 0; + virtual void set_proc_timeout(unsigned int Seconds) = 0; + + virtual int process_direct_query( + DirectRequestContext *reqCxt, + DirectResponseContext *respCxt) + { + return -1; + } +}; + +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/db_process_rocks.cc b/src/search_local/index_storage/rocksdb_helper/db_process_rocks.cc new file mode 100644 index 0000000..2833e8d --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/db_process_rocks.cc @@ -0,0 +1,3399 @@ + +/* + * ===================================================================================== + * + * Filename: db_process_rocks.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "db_process_rocks.h" +#include +#include + +#include "proc_title.h" +#include "table_def_manager.h" + +#include "buffer_pool.h" +#include +#include "mysql_error.h" +#include +#include + +// #define DEBUG_INFO +#define PRINT_STAT +#define BITS_OF_BYTE 8 /* bits of one byte */ +#define MAX_REPLICATE_LEN (1UL << 20) +static const std::string gReplicatePrefixKey = "__ROCKS_REPLICAE_PREFIX_KEY__"; +CommKeyComparator gInternalComparator; + +RocksdbProcess::RocksdbProcess(RocksDBConn *conn) +{ + mDBConn = conn; + + strncpy(name, "helper", 6); + titlePrefixSize = 0; + procTimeout = 0; + + mTableDef = NULL; + mCompoundKeyFieldNums = -1; + mExtraValueFieldNums = -1; + + mNoBitmapKey = true; + + mPrevMigrateKey = ""; + mCurrentMigrateKey = ""; + mUncommitedMigId = -1; + + mOrderByUnit = NULL; +} + +RocksdbProcess::~RocksdbProcess() +{ +} + +int RocksdbProcess::check_table() +{ + // no table concept in rocksdb + return (0); +} + +void RocksdbProcess::init_ping_timeout(void) +{ + // only for frame adapt + return; +} + +void RocksdbProcess::use_matched_rows(void) +{ + // only for frame adapt, no actual meanings + return; +} + +int RocksdbProcess::Init(int GroupID, const DbConfig *Config, DTCTableDefinition *tdef, int slave) +{ + int ret; + + SelfGroupID = GroupID; + mTableDef = tdef; + + std::vector dtcFieldIndex; + int totalFields = mTableDef->num_fields(); + for (int i = 0; i <= totalFields; i++) + { + //bug fix volatile不在db中 + if (mTableDef->is_volatile(i)) + continue; + dtcFieldIndex.push_back(i); + } + + totalFields = dtcFieldIndex.size(); + if (totalFields <= 0) + { + log_error("field can not be empty!"); + return -1; + } + + mCompoundKeyFieldNums = mTableDef->uniq_fields(); + if (mCompoundKeyFieldNums <= 0) + { + log_error("not found unique constraint in any field!"); + return -1; + } + mExtraValueFieldNums = totalFields - mCompoundKeyFieldNums; + log_info("total fields:%d, uniqKeyNum:%d, valueNum:%d", totalFields, mCompoundKeyFieldNums, mExtraValueFieldNums); + + // create map relationship + uint8_t keyIndex; + uint8_t *uniqFields = mTableDef->uniq_fields_list(); + for (int idx = 0; idx < mCompoundKeyFieldNums; idx++) + { + keyIndex = *(uniqFields + idx); + dtcFieldIndex[keyIndex] = -1; + mFieldIndexMapping.push_back(keyIndex); + } + + if (dtcFieldIndex.size() <= 0) + { + log_error("no value field!"); + return -1; + } + + // classify the unique keys into two types: Integer fixed len and elastic string type, + // no need to do collecting if the key is binary + // int keyType = mTableDef->key_type(); + mKeyfield_types.resize(mCompoundKeyFieldNums); + // mKeyfield_types[0] = keyType; + + { + // shrink string keys or integer keys into the head of the array + int fieldType; + // int moveHeadIdx = -1; + for (size_t idx = 0; idx < mFieldIndexMapping.size(); idx++) + { + fieldType = mTableDef->field_type(mFieldIndexMapping[idx]); + mKeyfield_types[idx] = fieldType; + log_info("fieldId:%d, fieldType:%d", mFieldIndexMapping[idx], fieldType); + switch (fieldType) + { + case DField::Signed: + case DField::Unsigned: + case DField::Float: + case DField::Binary: + break; + + case DField::String: + { + mNoBitmapKey = false; + break; + } + + default: + log_error("unexpected field type! type:%d", fieldType); + return -1; + }; + } + } + + // remove key from vector + auto start = std::remove_if(dtcFieldIndex.begin(), dtcFieldIndex.end(), + [](const int idx) { return idx == -1; }); + dtcFieldIndex.erase(start, dtcFieldIndex.end()); + + // append value maps + mFieldIndexMapping.insert(mFieldIndexMapping.end(), dtcFieldIndex.begin(), dtcFieldIndex.end()); + + { + mReverseFieldIndexMapping.resize(mFieldIndexMapping.size()); + for (size_t idx1 = 0; idx1 < mFieldIndexMapping.size(); idx1++) + { + mReverseFieldIndexMapping[mFieldIndexMapping[idx1]] = idx1; + } + } + + // init replication tag key + ret = get_replicate_end_key(); + + std::stringstream ss; + ss << "rocks helper meta info, keysize:" << mCompoundKeyFieldNums << " valuesize:" + << mExtraValueFieldNums << " rocksdb fields:["; + for (size_t idx = 0; idx < mFieldIndexMapping.size(); idx++) + { + log_info("%d, type:%d", mFieldIndexMapping[idx], idx < mCompoundKeyFieldNums ? mKeyfield_types[idx] : -1); + if (idx == 0) + ss << mFieldIndexMapping[idx]; + else + ss << ", " << mFieldIndexMapping[idx]; + } + ss << "]"; + log_info("%s", ss.str().c_str()); + + return ret; +} + +int RocksdbProcess::get_replicate_end_key() +{ + return 0; + std::string value; + std::string fullKey = gReplicatePrefixKey; + int ret = mDBConn->get_entry(fullKey, value, RocksDBConn::COLUMN_META_DATA); + if (ret < 0 && ret != -RocksDBConn::ERROR_KEY_NOT_FOUND) + { + log_error("query replicate end key failed! ret:%d", ret); + return -1; + } + else + { + mReplEndKey = value; + } + + return 0; +} + +inline int RocksdbProcess::value_add_to_str( + const DTCValue *additionValue, + int ifield_type, + std::string &baseValue) +{ + log_debug("value_add_to_str ifield_type[%d]", ifield_type); + + if (additionValue == NULL) + { + log_error("value can not be null!"); + return -1; + } + + switch (ifield_type) + { + case DField::Signed: + { + long long va = strtoll(baseValue.c_str(), NULL, 10); + va += (long long)additionValue->s64; + baseValue = std::to_string(va); + break; + } + + case DField::Unsigned: + { + unsigned long long va = strtoull(baseValue.c_str(), NULL, 10); + va += (unsigned long long)additionValue->u64; + baseValue = std::to_string(va); + break; + } + + case DField::Float: + { + double va = strtod(baseValue.c_str(), NULL); + va += additionValue->flt; + baseValue = std::to_string(va); + break; + } + + case DField::String: + case DField::Binary: + log_error("string type can not do add operation!"); + break; + + default: + log_error("unexpected field type! type:%d", ifield_type); + return -1; + }; + + return 0; +} + +inline int RocksdbProcess::value2Str( + const DTCValue *Value, + int fieldId, + std::string &strValue) +{ + const DTCValue *defaultValue; + bool valueNull = false; + + if (Value == NULL) + { + log_info("value is null, use user default value!"); + defaultValue = mTableDef->default_value(fieldId); + valueNull = true; + } + + int ifield_type = mTableDef->field_type(fieldId); + { + switch (ifield_type) + { + case DField::Signed: + { + int64_t val; + if (valueNull) + val = defaultValue->s64; + else + val = Value->s64; + strValue = std::move(std::to_string((long long)val)); + break; + } + + case DField::Unsigned: + { + uint64_t val; + if (valueNull) + val = defaultValue->u64; + else + val = Value->u64; + strValue = std::move(std::to_string((unsigned long long)val)); + break; + } + + case DField::Float: + { + double val; + if (valueNull) + val = defaultValue->flt; + else + val = Value->flt; + strValue = std::move(std::to_string(val)); + break; + } + + case DField::String: + case DField::Binary: + { + // value whether be "" or NULL ???? + // in current, regard NULL as empty string, not support NULL attribute here + if (valueNull) + strValue = std::move(std::string(defaultValue->str.ptr, defaultValue->str.len)); + else + { + if (Value->str.is_empty()) + { + log_info("empty str value!"); + strValue = ""; + return 0; + } + + strValue = std::move(std::string(Value->str.ptr, Value->str.len)); + /*if ( mkey_type == DField::String ) + { + // case insensitive + std::transform(strValue.begin(), strValue.end(), strValue.begin(), ::tolower); + }*/ + } + break; + } + default: + log_error("unexpected field type! type:%d", ifield_type); + return -1; + }; + } + + return 0; +} + +inline int RocksdbProcess::setdefault_value( + int field_type, + DTCValue &Value) +{ + switch (field_type) + { + case DField::Signed: + Value.s64 = 0; + break; + + case DField::Unsigned: + Value.u64 = 0; + break; + + case DField::Float: + Value.flt = 0.0; + break; + + case DField::String: + Value.str.len = 0; + Value.str.ptr = 0; + break; + + case DField::Binary: + Value.bin.len = 0; + Value.bin.ptr = 0; + break; + + default: + Value.s64 = 0; + }; + + return (0); +} + +inline int RocksdbProcess::str2Value( + const std::string &Str, + int field_type, + DTCValue &Value) +{ + if (Str == NULL) + { + log_debug("Str is NULL, field_type: %d. Check mysql table definition.", field_type); + setdefault_value(field_type, Value); + return (0); + } + + switch (field_type) + { + case DField::Signed: + errno = 0; + Value.s64 = strtoll(Str.c_str(), NULL, 10); + if (errno != 0) + return (-1); + break; + + case DField::Unsigned: + errno = 0; + Value.u64 = strtoull(Str.c_str(), NULL, 10); + if (errno != 0) + return (-1); + break; + + case DField::Float: + errno = 0; + Value.flt = strtod(Str.c_str(), NULL); + if (errno != 0) + return (-1); + break; + + case DField::String: + Value.str.len = Str.length(); + Value.str.ptr = const_cast(Str.data()); // 不重新new，要等这个value使用完后释放内存(如果Str是动态分配的) + break; + + case DField::Binary: + Value.bin.len = Str.length(); + Value.bin.ptr = const_cast(Str.data()); + break; + + default: + log_error("type[%d] invalid.", field_type); + return -1; + } + + return 0; +} + +int RocksdbProcess::condition_filter( + const std::string &rocksValue, + int fieldid, + int fieldType, + const DTCFieldValue *condition) +{ + if (condition == NULL) + return 0; + + bool matched; + // find out the condition value + for (int idx = 0; idx < condition->num_fields(); idx++) + { + if (mTableDef->is_volatile(idx)) + { + log_error("volatile field, idx:%d", idx); + return -1; + } + + int fId = condition->field_id(idx); + if (fId != fieldid) + continue; + + // DTC support query condition + /* enum { + EQ = 0, + NE = 1, + LT = 2, + LE = 3, + GT = 4, + GE = 5, + TotalComparison + }; */ + int comparator = condition->field_operation(idx); + const DTCValue *condValue = condition->field_value(idx); + + switch (fieldType) + { + case DField::Signed: + // matched = is_matched_template(strtoll(rocksValue.c_str(), NULL, 10), comparator, condValue.s64); + matched = is_matched(strtoll(rocksValue.c_str(), NULL, 10), comparator, condValue->s64); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%lld, com:%d", rocksValue.c_str(), + (long long)condValue->s64, comparator); + return 1; + } + break; + + case DField::Unsigned: + // matched = is_matched_template(strtoull(rocksValue.c_str(), NULL, 10), comparator, condValue.u64); + matched = is_matched(strtoull(rocksValue.c_str(), NULL, 10), comparator, condValue->u64); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%llu, com:%d", rocksValue.c_str(), + (unsigned long long)condValue->u64, comparator); + return 1; + } + break; + + case DField::Float: + // matched = is_matched_template(strtod(rocksValue.c_str(), NULL, 10), comparator, condValue.flt); + matched = is_matched(strtod(rocksValue.c_str(), NULL), comparator, condValue->flt); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%lf, com:%d", rocksValue.c_str(), + condValue->flt, comparator); + return 1; + } + break; + + case DField::String: + matched = is_matched(rocksValue.c_str(), comparator, condValue->str.ptr, (int)rocksValue.length(), condValue->str.len, false); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + std::string(condValue->str.ptr, condValue->str.len).c_str(), comparator); + return 1; + } + case DField::Binary: + // matched = is_matched_template(rocksValue.c_str(), comparator, condValue.str.ptr, (int)rocksValue.length(), condValue.str.len); + matched = is_matched(rocksValue.c_str(), comparator, condValue->bin.ptr, (int)rocksValue.length(), condValue->bin.len, true); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + std::string(condValue->bin.ptr, condValue->bin.len).c_str(), comparator); + return 1; + } + break; + + default: + log_error("field[%d] type[%d] invalid.", fieldid, fieldType); + return -1; + } + } + + return 0; +} + +int RocksdbProcess::condition_filter( + const std::string &rocksValue, + const std::string &condValue, + int fieldType, + int comparator) +{ + bool matched; + switch (fieldType) + { + case DField::Signed: + matched = is_matched(strtoll(rocksValue.c_str(), NULL, 10), comparator, strtoll(condValue.c_str(), NULL, 10)); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + condValue.c_str(), comparator); + return 1; + } + break; + + case DField::Unsigned: + matched = is_matched(strtoull(rocksValue.c_str(), NULL, 10), comparator, strtoull(condValue.c_str(), NULL, 10)); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + condValue.c_str(), comparator); + return 1; + } + break; + + case DField::Float: + matched = is_matched(strtod(rocksValue.c_str(), NULL), comparator, strtod(condValue.c_str(), NULL)); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + condValue.c_str(), comparator); + return 1; + } + break; + + case DField::String: + case DField::Binary: + matched = is_matched(rocksValue.c_str(), comparator, condValue.c_str(), (int)rocksValue.length(), (int)condValue.length(), false); + if (!matched) + { + log_info("not match the condition, lv:%s, rv:%s, com:%d", rocksValue.c_str(), + condValue.c_str(), comparator); + return 1; + } + break; + + default: + log_error("invalid field type[%d].", fieldType); + return -1; + } + + return 0; +} + +template +bool RocksdbProcess::is_matched_template(Args... len) +{ + return is_matched(len...); +} + +template +bool RocksdbProcess::is_matched( + const T lv, + int comparator, + const T rv) +{ + /* enum { + EQ = 0, + NE = 1, + LT = 2, + LE = 3, + GT = 4, + GE = 5, + TotalComparison + }; */ + switch (comparator) + { + case 0: + return lv == rv; + case 1: + return lv != rv; + case 2: + return lv < rv; + case 3: + return lv <= rv; + case 4: + return lv > rv; + case 5: + return lv >= rv; + default: + log_error("unsupport comparator:%d", comparator); + } + + return false; +} +template bool RocksdbProcess::is_matched(const int64_t lv, int comp, const int64_t rv); +template bool RocksdbProcess::is_matched(const uint64_t lv, int comp, const uint64_t rv); +template bool RocksdbProcess::is_matched(const double lv, int comp, const double rv); + +//template<> +bool RocksdbProcess::is_matched( + const char *lv, + int comparator, + const char *rv, + int lLen, + int rLen, + bool caseSensitive) +{ + /* enum { + EQ = 0, + NE = 1, + LT = 2, + LE = 3, + GT = 4, + GE = 5, + TotalComparison + }; */ + int ret; + int minLen = lLen <= rLen ? lLen : rLen; + switch (comparator) + { + case 0: + if (caseSensitive) + return lLen == rLen && !strncmp(lv, rv, minLen); + return lLen == rLen && !strncasecmp(lv, rv, minLen); + case 1: + if (lLen != rLen) + return true; + if (caseSensitive) + return strncmp(lv, rv, minLen); + return strncasecmp(lv, rv, minLen); + case 2: + if (caseSensitive) + ret = strncmp(lv, rv, minLen); + else + ret = strncasecmp(lv, rv, minLen); + return ret < 0 || (ret == 0 && lLen < rLen); + case 3: + if (caseSensitive) + ret = strncmp(lv, rv, minLen); + else + ret = strncasecmp(lv, rv, minLen); + return ret < 0 || (ret == 0 && lLen <= rLen); + case 4: + if (caseSensitive) + ret = strncmp(lv, rv, minLen); + else + ret = strncasecmp(lv, rv, minLen); + return ret > 0 || (ret == 0 && lLen > rLen); + case 5: + if (caseSensitive) + ret = strncmp(lv, rv, minLen); + else + ret = strncasecmp(lv, rv, minLen); + return ret > 0 || (ret == 0 && lLen >= rLen); + default: + log_error("unsupport comparator:%d", comparator); + } + + return false; +} + +int RocksdbProcess::saveRow( + const std::string &compoundKey, + const std::string &compoundValue, + bool countOnly, + int &totalRows, + DTCTask *Task) +{ + if (mCompoundKeyFieldNums + mExtraValueFieldNums <= 0) + { + log_error("no fields in the table! key:%s"); + return (-1); + } + + int ret; + // decode the compoundKey and check whether it is matched + std::vector keys; + key_format::Decode(compoundKey, mKeyfield_types, keys); + if (keys.size() != mCompoundKeyFieldNums) + { + // unmatched row + log_error("unmatched row, fullKey:%s, keyNum:%lu, definitionFieldNum:%d", + compoundKey.c_str(), keys.size(), mCompoundKeyFieldNums); + return -1; + } + + if (countOnly) + { + totalRows++; + return 0; + } + + // decode key bitmap + int bitmapLen = 0; + decodeBitmapKeys(compoundValue, keys, bitmapLen); + + //DBConn.Row[0]是key的值，mTableDef->Field[0]也是key， + //因此从1开始。而结果Row是从0开始的(不包括key) + RowValue *row = new RowValue(mTableDef); + const DTCFieldValue *Condition = Task->request_condition(); + std::string fieldValue; + char *valueHead = const_cast(compoundValue.data()) + bitmapLen; + for (size_t idx = 0; idx < mFieldIndexMapping.size(); idx++) + { + int fieldId = mFieldIndexMapping[idx]; + if (idx < mCompoundKeyFieldNums) + { + fieldValue = keys[idx]; + } + else + { + ret = get_value_by_id(valueHead, fieldId, fieldValue); + if (ret != 0) + { + log_error("parse field value failed! compoundValue:%s", compoundValue.c_str()); + delete row; + return -1; + } + } + log_info("save row, fieldId:%d, val:%s", fieldId, fieldValue.data()); + + // do condition filter + ret = condition_filter(fieldValue, fieldId, mTableDef->field_type(fieldId), Condition); + if (ret < 0) + { + delete row; + log_error("string[%s] conver to value[%d] error: %d", fieldValue.c_str(), mTableDef->field_type(fieldId), ret); + return (-2); + } + else if (ret == 1) + { + // condition is not matched + delete row; + return 0; + } + + // fill the value + ret = str2Value(fieldValue, mTableDef->field_type(fieldId), (*row)[fieldId]); + if (ret < 0) + { + delete row; + log_error("string[%s] conver to value[%d] error: %d", fieldValue.c_str(), mTableDef->field_type(fieldId), ret); + return (-2); + } + } + + // Task->update_key(row); + ret = Task->append_row(row); + + delete row; + + if (ret < 0) + { + log_error("append row to task failed!"); + return (-3); + } + + // totalRows++; + + return (0); +} + +int RocksdbProcess::save_direct_row( + const std::string &prefixKey, + const std::string &compoundKey, + const std::string &compoundValue, + DirectRequestContext *reqCxt, + DirectResponseContext *respCxt, + int &totalRows) +{ + if (mCompoundKeyFieldNums + mExtraValueFieldNums <= 0) + { + log_error("no fields in the table! key:%s", prefixKey.c_str()); + return (-1); + } + + int ret; + // decode the compoundKey and check whether it is matched + std::vector keys; + key_format::Decode(compoundKey, mKeyfield_types, keys); + if (keys.size() != mCompoundKeyFieldNums) + { + // unmatched row + log_error("unmatched row, key:%s, fullKey:%s, keyNum:%lu, definitionFieldNum:%d", + prefixKey.c_str(), compoundKey.c_str(), keys.size(), mCompoundKeyFieldNums); + return -1; + } + + // decode key bitmap + int bitmapLen = 0; + decodeBitmapKeys(compoundValue, keys, bitmapLen); + + std::string realValue = compoundValue.substr(bitmapLen); + + std::vector values; + split_values(realValue, values); + assert(values.size() == mExtraValueFieldNums); + + int fieldId, rocksFId; + std::string fieldValue; + std::vector &condFields = reqCxt->sFieldConds; + for (size_t idx = 0; idx < condFields.size(); idx++) + { + fieldId = condFields[idx].sFieldIndex; + rocksFId = translate_field_idx(fieldId); + if (rocksFId < mCompoundKeyFieldNums) + { + fieldValue = keys[rocksFId]; + } + else + { + fieldValue = values[rocksFId - mCompoundKeyFieldNums]; + } + + // do condition filter + ret = condition_filter(fieldValue, condFields[idx].sCondValue, mTableDef->field_type(fieldId), condFields[idx].sCondOpr); + if (ret < 0) + { + log_error("condition filt failed! key:%s", prefixKey.c_str()); + return -1; + } + else if (ret == 1) + { + // condition is not matched + return 0; + } + } + + // deal with order by syntax + if (mOrderByUnit || reqCxt->sOrderbyFields.size() > 0) + { + if (!mOrderByUnit) + { + // build order by unit + int heapSize = reqCxt->sLimitCond.sLimitStart >= 0 && reqCxt->sLimitCond.sLimitStep > 0 ? reqCxt->sLimitCond.sLimitStart + reqCxt->sLimitCond.sLimitStep : 50; + mOrderByUnit = new RocksdbOrderByUnit(mTableDef, heapSize, + mReverseFieldIndexMapping, reqCxt->sOrderbyFields); + assert(mOrderByUnit); + } + + struct OrderByUnitElement element; + element.mRocksKeys.swap(keys); + element.mRocksValues.swap(values); + mOrderByUnit->add_row(element); + return 0; + } + + // limit condition control + ret = 0; + if (reqCxt->sLimitCond.sLimitStart >= 0 && reqCxt->sLimitCond.sLimitStep > 0) + { + if (totalRows < reqCxt->sLimitCond.sLimitStart) + { + // not reach to the range of limitation + totalRows++; + return 0; + } + + // leaving from the range of limitaion + if (respCxt->sRowValues.size() == reqCxt->sLimitCond.sLimitStep - 1) + ret = 1; + } + + // build row + build_direct_row(keys, values, respCxt); + totalRows++; + + return ret; +} + +void RocksdbProcess::build_direct_row( + const std::vector &keys, + const std::vector &values, + DirectResponseContext *respCxt) +{ + int rocksFId; + std::string row, fieldValue; + for (size_t idx1 = 0; idx1 < mReverseFieldIndexMapping.size(); idx1++) + { + rocksFId = mReverseFieldIndexMapping[idx1]; + if (rocksFId < mCompoundKeyFieldNums) + { + fieldValue = keys[rocksFId]; + } + else + { + fieldValue = values[rocksFId - mCompoundKeyFieldNums]; + } + + int dataLen = fieldValue.length(); + row.append(std::string((char *)&dataLen, 4)).append(fieldValue); + } + + respCxt->sRowValues.push_front(row); + return; +} + +int RocksdbProcess::update_row( + const std::string &prefixKey, + const std::string &compoundKey, + const std::string &compoundValue, + DTCTask *Task, + std::string &newKey, + std::string &newValue) +{ + if (mCompoundKeyFieldNums + mExtraValueFieldNums <= 0) + { + log_error("no fields in the table!"); + return (-1); + } + + int ret; + // decode the compoundKey and check whether it is matched + std::vector keys; + key_format::Decode(compoundKey, mKeyfield_types, keys); + + if (keys.size() != mCompoundKeyFieldNums) + { + // unmatched row + log_error("unmatched row, key:%s, fullKey:%s, keyNum:%lu, definitionFieldNum:%d", + prefixKey.c_str(), compoundKey.c_str(), keys.size(), mCompoundKeyFieldNums); + return -1; + } + + bool upKey = false, upVal = false; + const DTCFieldValue *updateInfo = Task->request_operation(); + whether_update_key(updateInfo, upKey, upVal); + + int keyBitmapLen = 0; + if (upKey) + { + // recover keys for the next update + decodeBitmapKeys(compoundValue, keys, keyBitmapLen); + } + else + { + // no need to create bitmap and compound key again + keyBitmapLen = get_key_bitmap_len(compoundValue); + assert(keyBitmapLen >= 0); + } + + std::string bitmapKey = compoundValue.substr(0, keyBitmapLen); + std::string realValue = compoundValue.substr(keyBitmapLen); + + std::string fieldValue; + const DTCFieldValue *Condition = Task->request_condition(); + char *valueHead = (char *)realValue.data(); + for (size_t idx = 1; idx < mFieldIndexMapping.size(); idx++) + { + int dtcfid = mFieldIndexMapping[idx]; + if (idx < mCompoundKeyFieldNums) + { + fieldValue = keys[idx]; + } + else + { + ret = get_value_by_id(valueHead, dtcfid, fieldValue); + if (ret != 0) + { + log_error("parse field value failed! compoundValue:%s, key:%s", realValue.c_str(), prefixKey.c_str()); + return -1; + } + } + + // do condition filter + ret = condition_filter(fieldValue, dtcfid, mTableDef->field_type(dtcfid), Condition); + if (ret < 0) + { + log_error("string[%s] conver to value[%d] error: %d, %m", fieldValue.c_str(), mTableDef->field_type(dtcfid), ret); + return (-2); + } + else if (ret == 1) + { + // condition is not matched + return 1; + } + } + + // update value + std::vector values; + if (upVal) + { + // translate the plane raw value to individual field + split_values(realValue, values); + } + + for (int i = 0; i < updateInfo->num_fields(); i++) + { + const int fid = updateInfo->field_id(i); + + if (mTableDef->is_volatile(fid)) + continue; + + int rocksFId = translate_field_idx(fid); + assert(rocksFId >= 0 && rocksFId < mCompoundKeyFieldNums + mExtraValueFieldNums); + + // if not update the value field, the rocksfid must be less than 'mCompoundKeyFieldNums', so it would not touch + // the container of 'values' + std::string &va = rocksFId < mCompoundKeyFieldNums ? keys[rocksFId] : values[rocksFId - mCompoundKeyFieldNums]; + + switch (updateInfo->field_operation(i)) + { + case DField::Set: + value2Str(updateInfo->field_value(i), fid, va); + break; + + case DField::Add: + value_add_to_str(updateInfo->field_value(i), updateInfo->field_type(i), va); + break; + + default: + log_error("unsupport operation, opr:%d", updateInfo->field_operation(i)); + return -1; + }; + } + + if (upKey) + { + bitmapKey.clear(); + encode_bitmap_keys(keys, bitmapKey); + + newKey = std::move(key_format::Encode(keys, mKeyfield_types)); + } + else + newKey = compoundKey; + + if (upVal) + shrink_value(values, newValue); + else + newValue = std::move(realValue); + + newValue = std::move(bitmapKey.append(newValue)); + + return 0; +} + +// query value of the key +int RocksdbProcess::process_select(DTCTask *Task) +{ + log_info("come into process select!"); + +#ifdef PRINT_STAT + mSTime = GET_TIMESTAMP(); +#endif + + int ret, i; + int haslimit = !Task->count_only() && (Task->requestInfo.limit_start() || Task->requestInfo.limit_count()); + + // create resultWriter + ret = Task->prepare_result_no_limit(); + if (ret != 0) + { + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "task prepare-result error"); + log_error("task prepare-result error: %d, %m", ret); + return (-2); + } + + // prefix key + //std::string prefixKey; + std::vector prefixKey(1); + ret = value2Str(Task->request_key(), 0, prefixKey[0]); + if (ret != 0 || prefixKey[0].empty()) + { + log_error("dtc primary key can not be empty!"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get dtc primary key failed!"); + return -1; + } + std::vector preType(1); + preType[0] = mKeyfield_types[0]; + + if (mKeyfield_types[0] == DField::String) + std::transform(prefixKey[0].begin(), prefixKey[0].end(), prefixKey[0].begin(), ::tolower); + + + // encode the key to rocksdb format + std::string fullKey = std::move(key_format::Encode(prefixKey, preType)); + if (fullKey.empty()) + { + log_error("encode primary key failed! key:%s", prefixKey[0].c_str()); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "encode primary key failed!"); + return -1; + } + + std::string encodePreKey = fullKey; + + std::string value; + RocksDBConn::RocksItr_t rocksItr; + ret = mDBConn->retrieve_start(fullKey, value, rocksItr, true); + if (ret < 0) + { + log_error("query rocksdb failed! key:%s, ret:%d", prefixKey[0].c_str(), ret); + Task->set_error(ret, __FUNCTION__, "retrieve primary key failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // not found the key + Task->set_total_rows(0); + log_info("no matched key:%s", prefixKey[0].c_str()); + mDBConn->retrieve_end(rocksItr); + return 0; + } + + log_info("begin to iterate key:%s", prefixKey[0].c_str()); + + // iterate the matched prefix key and find out the real one from start to end + int totalRows = 0; + bool countOnly = Task->count_only(); + while (true) + { + ret = key_matched(encodePreKey, fullKey); + if (ret != 0) + { + // prefix key not matched, reach to the end + break; + } + + // save row + ret = saveRow(fullKey, value, countOnly, totalRows, Task); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s", prefixKey[0].c_str()); + } + + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, fullKey, value); + if (ret < 0) + { + log_error("iterate rocksdb failed! key:%s", prefixKey[0].c_str()); + Task->set_error(ret, __FUNCTION__, "iterate rocksdb failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the storage end + break; + } + + // has remaining value in rocksdb + } + + if (Task->count_only()) + { + Task->set_total_rows(totalRows); + } + + //bug fixed确认客户端带Limit限制 + if (haslimit) + { // 获取总行数 + Task->set_total_rows(totalRows, 1); + } + + mDBConn->retrieve_end(rocksItr); + +#ifdef PRINT_STAT + mETime = GET_TIMESTAMP(); + insert_stat(OprType::eQuery, mETime - mSTime); +#endif + + return (0); +} + +int RocksdbProcess::process_insert(DTCTask *Task) +{ + log_info("come into process insert!!!"); + +#ifdef PRINT_STAT + mSTime = GET_TIMESTAMP(); +#endif + + int ret; + + set_title("INSERT..."); + + int totalFields = mTableDef->num_fields(); + + // initialize the fixed empty string vector + std::bitset<256> filledMap; + std::vector keys(mCompoundKeyFieldNums); + std::vector values(mExtraValueFieldNums); + + ret = value2Str(Task->request_key(), 0, keys[0]); + if (ret != 0 || keys[0].empty()) + { + log_error("dtc primary key can not be empty!"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get dtc primary key failed!"); + return -1; + } + else + { + filledMap[0] = 1; + } + log_info("insert key:%s", keys[0].c_str()); + + if (Task->request_operation()) + { + int fid, rocksFId; + const DTCFieldValue *updateInfo = Task->request_operation(); + for (int i = 0; i < updateInfo->num_fields(); ++i) + { + fid = updateInfo->field_id(i); + if (mTableDef->is_volatile(fid)) + continue; + + rocksFId = translate_field_idx(fid); + assert(rocksFId >= 0 && rocksFId < mCompoundKeyFieldNums + mExtraValueFieldNums); + + if (fid == 0) + continue; + + std::string &va = rocksFId < mCompoundKeyFieldNums ? keys[rocksFId] : values[rocksFId - mCompoundKeyFieldNums]; + ret = value2Str(updateInfo->field_value(i), fid, va); + assert(ret == 0); + + filledMap[fid] = 1; + } + } + + // set default value + for (int i = 1; i <= totalFields; ++i) + { + int rocksFId; + if (mTableDef->is_volatile(i)) + continue; + + if (filledMap[i]) + continue; + + rocksFId = translate_field_idx(i); + assert(rocksFId >= 0 && rocksFId < mCompoundKeyFieldNums + mExtraValueFieldNums); + + std::string &va = rocksFId < mCompoundKeyFieldNums ? keys[rocksFId] : values[rocksFId - mCompoundKeyFieldNums]; + ret = value2Str(mTableDef->default_value(i), i, va); + assert(ret == 0); + } + +#ifdef DEBUG_INFO + std::stringstream ss; + ss << "insert row:["; + for (size_t idx = 0; idx < mCompoundKeyFieldNums; idx++) + { + ss << keys[idx]; + if (idx != mCompoundKeyFieldNums - 1) + ss << ","; + } + ss << "]"; + log_error("%s", ss.str().c_str()); +#endif + + // convert string type 'key' into lower case and build case letter bitmap + std::string keyBitmaps; + encode_bitmap_keys(keys, keyBitmaps); + + std::string rocksKey, rocksValue; + rocksKey = std::move(key_format::Encode(keys, mKeyfield_types)); + + // value use plane style to organize, no need to encode + ret = shrink_value(values, rocksValue); + if (ret != 0) + { + std::string rkey; + value2Str(Task->request_key(), 0, rkey); + log_error("shrink value failed, key:%s", rkey.c_str()); + + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "shrink buff failed!"); + return -1; + } + + // add key bitmaps to the rocksdb value field + keyBitmaps.append(rocksValue); + + log_info("save kv, key:%s, value:%s", rocksKey.c_str(), rocksValue.c_str()); + ret = mDBConn->insert_entry(rocksKey, keyBitmaps, true); + if (ret != 0) + { + std::string rkey; + value2Str(Task->request_key(), 0, rkey); + if (ret != -ER_DUP_ENTRY) + { + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "insert key failed!"); + log_error("insert key failed, ret:%d, key:%s", ret, rkey.c_str()); + } + else + { + Task->set_error_dup(ret, __FUNCTION__, "insert entry failed!"); + log_error("insert duplicate key : %s", rkey.c_str()); + } + + return (-1); + } + + Task->resultInfo.set_affected_rows(1); + +#ifdef PRINT_STAT + mETime = GET_TIMESTAMP(); + insert_stat(OprType::eInsert, mETime - mSTime); +#endif + + return (0); +} + +// update the exist rows matched the condition +int RocksdbProcess::process_update(DTCTask *Task) +{ + log_info("come into rocks update"); + +#ifdef PRINT_STAT + mSTime = GET_TIMESTAMP(); +#endif + + int ret, affectRows = 0; + + // prefix key + std::string prefixKey; + ret = value2Str(Task->request_key(), 0, prefixKey); + if (ret != 0) + { + log_error("get dtc primary key failed! key"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get key failed!"); + return -1; + } + log_info("update key:%s", prefixKey.c_str()); + + if (Task->request_operation() == NULL) + { + log_info("request operation info is null!"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "update field not found"); + return (-1); + } + + if (Task->request_operation()->has_type_commit() == 0) + { + // pure volatile fields update, always succeed + log_info("update pure volatile fields!"); + Task->resultInfo.set_affected_rows(affectRows); + return (0); + } + + const DTCFieldValue *updateInfo = Task->request_operation(); + if (updateInfo == NULL) + { + // no need to do update + log_info("request update info is null!"); + Task->resultInfo.set_affected_rows(affectRows); + return (0); + } + + set_title("UPDATE..."); + + bool updateKey = false, updateValue = false; + whether_update_key(updateInfo, updateKey, updateValue); + if (!updateKey && !updateValue) + { + // no need to do update + log_info("no change for the row!"); + Task->resultInfo.set_affected_rows(affectRows); + return (0); + } + + if (mKeyfield_types[0] == DField::String) + std::transform(prefixKey.begin(), prefixKey.end(), prefixKey.begin(), ::tolower); + + // encode the key to rocksdb format + std::string fullKey = std::move(key_format::encode_bytes(prefixKey)); + std::string encodePreKey = fullKey; + + std::string compoundValue; + RocksDBConn::RocksItr_t rocksItr; + ret = mDBConn->retrieve_start(fullKey, compoundValue, rocksItr, true); + if (ret < 0) + { + log_info("retrieve rocksdb failed, key:%s", prefixKey.c_str()); + Task->set_error_dup(ret, __FUNCTION__, "retrieve rocksdb failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // not found the key + log_info("no matched key:%s", prefixKey.c_str()); + Task->resultInfo.set_affected_rows(affectRows); + mDBConn->retrieve_end(rocksItr); + return 0; + } + + // retrieve the range keys one by one + std::set deletedKeys; + std::map newEntries; + if (updateKey) + { + // Will update the row, so we need to keep the whole row in the memory for checking + // the unique constraints + // Due to the limitaion of the memory, we can not hold all rows in the memory sometimes, + // so use the LRU to evit the oldest row + std::set originKeys; // keys located in rocksdb those before doing update + while (true) + { + // find if the key has been update before, if yes, should rollback the previously update + auto itr = newEntries.find(fullKey); + if (itr != newEntries.end()) + { + log_info("duplicated entry, key:%s", fullKey.c_str()); + // report alarm + Task->set_error_dup(-ER_DUP_ENTRY, __FUNCTION__, "duplicate key!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + + ret = key_matched(encodePreKey, fullKey); + if (ret != 0) + { + // prefix key not matched, reach to the end + break; + } + + // update row + std::string newKey, newValue; + ret = update_row(prefixKey, fullKey, compoundValue, Task, newKey, newValue); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s, compoundValue:%s", fullKey.c_str(), compoundValue.c_str()); + } + else if (ret == 1) + { + // key matched, but condition mismatched, keep on retrieve + originKeys.insert(fullKey); + } + else + { + { + ret = rocks_key_matched(fullKey, newKey); + if (ret == 0) + { + log_info("duplicated entry, newkey:%s", newKey.c_str()); + Task->set_error_dup(-ER_DUP_ENTRY, __FUNCTION__, "duplicate key!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + } + + if (originKeys.find(newKey) == originKeys.end() && newEntries.find(newKey) == newEntries.end() && deletedKeys.find(newKey) == deletedKeys.end()) + { + // there are so many duplcate string save in the different containers, this will + // waste too much space, we can optimize it in the future + affectRows++; + deletedKeys.insert(fullKey); + newEntries[newKey] = newValue; + } + else + { + // duplicate key, ignore it + log_info("duplicated entry, newkey:%s", newKey.c_str()); + Task->set_error_dup(-ER_DUP_ENTRY, __FUNCTION__, "duplicate key!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + } + + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, fullKey, compoundValue); + if (ret < 0) + { + log_error("retrieve compoundValue failed, key:%s", prefixKey.c_str()); + Task->set_error_dup(ret, __FUNCTION__, "get next entry failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // no remaining rows in the storage + break; + } + + // has remaining compoundValue in rocksdb + } + } + else + { + // do not break the unique key constraints, no need to hold the entire row in memory + // iterate the matched prefix key and find out the real one from start to end + while (true) + { + ret = key_matched(encodePreKey, fullKey); + if (ret != 0) + { + // prefix key not matched, reach to the end + break; + } + + // update row + std::string newKey, newValue; + ret = update_row(prefixKey, fullKey, compoundValue, Task, newKey, newValue); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s, compoundValue:%s", fullKey.c_str(), compoundValue.c_str()); + } + else if (ret == 1) + { + // key matched, but condition mismatched, keep on retrieve + } + else + { + affectRows++; + newEntries[fullKey] = newValue; + } + + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, fullKey, compoundValue); + if (ret < 0) + { + log_error("retrieve compoundValue failed, key:%s", prefixKey.c_str()); + Task->set_error_dup(ret, __FUNCTION__, "get next entry failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the storage end + break; + } + + // has remaining compoundValue in rocksdb + } + } + +#ifdef DEBUG_INFO + std::vector keys; + std::stringstream ss; + + ss << "delete keys:["; + auto itr = deletedKeys.begin(); + while (itr != deletedKeys.end()) + { + keys.clear(); + key_format::Decode(*itr, mKeyfield_types, keys); + + ss << "("; + for (size_t idx = 0; idx < keys.size(); idx++) + { + ss << keys[idx]; + if (idx != keys.size() - 1) + ss << ","; + } + ss << ") "; + + itr++; + } + ss << "]"; + + ss << "new keys:["; + auto itrM = newEntries.begin(); + while (itrM != newEntries.end()) + { + keys.clear(); + key_format::Decode(itrM->first, mKeyfield_types, keys); + + ss << "("; + for (size_t idx = 0; idx < keys.size(); idx++) + { + ss << keys[idx]; + if (idx != keys.size() - 1) + ss << ","; + } + ss << ") "; + + itrM++; + } + ss << "]"; + + log_error("%s", ss.str().c_str()); +#endif + + // do atomic update + ret = mDBConn->batch_update(deletedKeys, newEntries, true); + if (ret != 0) + { + log_error("batch update rocksdb failed! key:%s", prefixKey.c_str()); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "update failed!"); + return -1; + } + + mDBConn->retrieve_end(rocksItr); + + Task->resultInfo.set_affected_rows(affectRows); + +#ifdef PRINT_STAT + mETime = GET_TIMESTAMP(); + insert_stat(OprType::eUpdate, mETime - mSTime); +#endif + + return (0); +} + +int RocksdbProcess::whether_update_key( + const DTCFieldValue *UpdateInfo, + bool &updateKey, + bool &updateValue) +{ + int fieldId; + updateKey = false; + updateValue = false; + for (int i = 0; i < UpdateInfo->num_fields(); i++) + { + const int fid = UpdateInfo->field_id(i); + + if (mTableDef->is_volatile(fid)) + continue; + + assert(fid < mFieldIndexMapping.size()); + + for (size_t idx = 0; idx < mFieldIndexMapping.size(); idx++) + { + if (fid == mFieldIndexMapping[idx]) + { + if (idx < mCompoundKeyFieldNums) + updateKey = true; + else + updateValue = true; + + break; + } + } + } + + return 0; +} + +int RocksdbProcess::process_delete(DTCTask *Task) +{ + int ret, affectRows = 0; + + // prefix key + std::string prefixKey; + ret = value2Str(Task->request_key(), 0, prefixKey); + if (ret != 0) + { + log_error("get dtc primary key failed! key"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get key failed!"); + return -1; + } + + if (mKeyfield_types[0] == DField::String) + std::transform(prefixKey.begin(), prefixKey.end(), prefixKey.begin(), ::tolower); + + // encode the key to rocksdb format + std::string fullKey = std::move(key_format::encode_bytes(prefixKey)); + std::string encodePreKey = fullKey; + + std::string compoundValue; + RocksDBConn::RocksItr_t rocksItr; + ret = mDBConn->retrieve_start(fullKey, compoundValue, rocksItr, true); + if (ret < 0) + { + log_error("retrieve rocksdb failed! key:%s", fullKey.c_str()); + Task->set_error_dup(ret, __FUNCTION__, "retrieve failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // not found the key + Task->resultInfo.set_affected_rows(affectRows); + mDBConn->retrieve_end(rocksItr); + log_info("no matched key:%s", prefixKey.c_str()); + return 0; + } + + // iterate the matched prefix key and find out the real one from start to end + bool condMatch = true; + int bitmapLen = 0; + DTCFieldValue *condition; + std::set deleteKeys; + while (true) + { + // check whether the key is in the range of the prefix of the 'fullKey' + ret = key_matched(encodePreKey, fullKey); + if (ret != 0) + { + // prefix key not matched, reach to the end + break; + } + + // decode the compoundKey and check whether it is matched + std::string realValue; + std::vector keys; + std::vector values; + key_format::Decode(fullKey, mKeyfield_types, keys); + assert(keys.size() > 0); + + ret = prefixKey.compare(keys[0]); + // if ( ret != 0 ) goto NEXT_ENTRY; + if (ret != 0) + break; + + if (keys.size() != mCompoundKeyFieldNums) + { + // unmatched row + log_error("unmatched row, fullKey:%s, keyNum:%lu, definitionFieldNum:%d", + fullKey.c_str(), keys.size(), mCompoundKeyFieldNums); + ret = 0; + } + + // decode key bitmap + decodeBitmapKeys(compoundValue, keys, bitmapLen); + + realValue = compoundValue.substr(bitmapLen); + split_values(realValue, values); + if (values.size() != mExtraValueFieldNums) + { + log_info("unmatched row, fullKey:%s, value:%s, keyNum:%lu, valueNum:%lu", + fullKey.c_str(), compoundValue.c_str(), keys.size(), values.size()); + ret = 0; + } + + // condition filter + condition = (DTCFieldValue *)Task->request_condition(); + for (size_t idx = 1; idx < mFieldIndexMapping.size(); idx++) + { + int fieldId = mFieldIndexMapping[idx]; + std::string &fieldValue = idx < mCompoundKeyFieldNums ? keys[idx] : values[idx - mCompoundKeyFieldNums]; + + // do condition filter + ret = condition_filter(fieldValue, fieldId, mTableDef->field_type(fieldId), condition); + if (ret < 0) + { + log_error("string[%s] conver to value[%d] error: %d, %m", fieldValue.c_str(), mTableDef->field_type(fieldId), ret); + condMatch = false; + break; + } + else if (ret == 1) + { + // condition is not matched + condMatch = false; + break; + } + } + + if (condMatch) + { +#ifdef DEBUG_INFO + std::stringstream ss; + ss << "delete key:["; + for (size_t idx = 0; idx < mCompoundKeyFieldNums; idx++) + { + ss << keys[idx]; + if (idx != mCompoundKeyFieldNums - 1) + ss << ","; + } + ss << "]"; + log_error("%s", ss.str().c_str()); +#endif + deleteKeys.insert(fullKey); + affectRows++; + } + + NEXT_ENTRY: + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, fullKey, compoundValue); + if (ret < 0) + { + Task->set_error_dup(ret, __FUNCTION__, "get next entry failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the end of the storage + break; + } + } + + // delete key from Rocksdb storage, inside the 'retrieve end' for transaction isolation, this delete will be not seen before release this retrieve + ret = mDBConn->batch_update(deleteKeys, std::map(), true); + if (ret != 0) + { + log_error("batch update rocksdb failed! key:%s", prefixKey.c_str()); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "update failed!"); + return -1; + } + + mDBConn->retrieve_end(rocksItr); + Task->resultInfo.set_affected_rows(affectRows); + + return (0); +} + +int RocksdbProcess::process_task(DTCTask *Task) +{ + if (Task == NULL) + { + log_error("Task is NULL!%s", ""); + return (-1); + } + + mTableDef = TableDefinitionManager::Instance()->get_cur_table_def(); + + switch (Task->request_code()) + { + case DRequest::Nop: + case DRequest::Purge: + case DRequest::Flush: + return 0; + + case DRequest::Get: + return process_select(Task); + + case DRequest::Insert: + return process_insert(Task); + + case DRequest::Update: + return process_update(Task); + + case DRequest::Delete: + return process_delete(Task); + + case DRequest::Replace: + return process_replace(Task); + + case DRequest::ReloadConfig: + return process_reload_config(Task); + + // master-slave replication + case DRequest::Replicate: + return ProcessReplicate(Task); + // cluster scaleable + //case DRequest::Migrate: + // return ProcessMigrate(); + + default: + Task->set_error(-EC_BAD_COMMAND, __FUNCTION__, "invalid request-code"); + return (-1); + } +} + +//add by frankyang 处理更新过的交易日志 +int RocksdbProcess::process_replace(DTCTask *Task) +{ + int ret, affectRows = 0; + + set_title("REPLACE..."); + + // primary key in DTC + std::vector keys, values; + keys.resize(mCompoundKeyFieldNums); + values.resize(mExtraValueFieldNums); + + std::string strKey; + value2Str(Task->request_key(), 0, strKey); + keys[0] = strKey; + + // deal update info + const DTCFieldValue *updateInfo = Task->request_operation(); + if (updateInfo != NULL) + { + for (int idx = 0; idx < updateInfo->num_fields(); idx++) + { + const int fid = updateInfo->field_id(idx); + + if (mTableDef->is_volatile(fid)) + continue; + + std::string fieldValue; + switch (updateInfo->field_operation(idx)) + { + case DField::Set: + { + ret = value2Str(updateInfo->field_value(idx), fid, fieldValue); + if (ret != 0) + { + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get value failed!"); + log_error("translate value failed"); + return (-1); + } + break; + } + + case DField::Add: + { + // add additional value to the defaule value + const DTCValue *addValue = updateInfo->field_value(idx); + const DTCValue *defValue = mTableDef->default_value(idx); + switch (updateInfo->field_type(idx)) + { + case DField::Signed: + fieldValue = std::move(std::to_string((long long)(addValue->s64 + defValue->s64))); + break; + + case DField::Unsigned: + fieldValue = std::move(std::to_string((unsigned long long)(addValue->u64 + defValue->u64))); + break; + + case DField::Float: + fieldValue = std::move(std::to_string(addValue->flt + defValue->flt)); + break; + default: + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "unkonwn field type!"); + log_error("unsupport field data type:%d", updateInfo->field_type(idx)); + return -1; + } + break; + } + + default: + log_error("unsupport field operation:%d", updateInfo->field_operation(idx)); + break; + } + + int rocksidx = translate_field_idx(fid); + assert(rocksidx >= 0 && rocksidx < mCompoundKeyFieldNums + mExtraValueFieldNums); + rocksidx < mCompoundKeyFieldNums ? keys[rocksidx] = std::move(fieldValue) : values[rocksidx - mCompoundKeyFieldNums] = std::move(fieldValue); + } + } + + // deal default value + uint8_t mask[32]; + FIELD_ZERO(mask); + if (updateInfo) + updateInfo->build_field_mask(mask); + + for (int i = 1; i <= mTableDef->num_fields(); i++) + { + if (FIELD_ISSET(i, mask) || mTableDef->is_volatile(i)) + continue; + + std::string fieldValue; + ret = value2Str(mTableDef->default_value(i), i, fieldValue); + if (ret != 0) + { + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get value failed!"); + log_error("translate value failed"); + return (-1); + } + + int rocksidx = translate_field_idx(i); + assert(rocksidx >= 0 && rocksidx < mCompoundKeyFieldNums + mExtraValueFieldNums); + rocksidx < mCompoundKeyFieldNums ? keys[rocksidx] = std::move(fieldValue) : values[rocksidx - mCompoundKeyFieldNums] = std::move(fieldValue); + } + + // convert string type 'key' into lower case and build case letter bitmap + std::string keyBitmaps; + encode_bitmap_keys(keys, keyBitmaps); + + std::string rocksKey, rocksValue; + rocksKey = std::move(key_format::Encode(keys, mKeyfield_types)); + + shrink_value(values, rocksValue); + + // add key bitmaps to the rocksdb value field + keyBitmaps.append(rocksValue); + + ret = mDBConn->replace_entry(rocksKey, keyBitmaps, true); + if (ret == 0) + { + Task->resultInfo.set_affected_rows(1); + return 0; + } + + log_error("replace key failed, key:%s, code:%d", rocksKey.c_str(), ret); + Task->set_error_dup(ret, __FUNCTION__, "replace key failed!"); + return -1; +} + +int RocksdbProcess::ProcessReplicate(DTCTask *Task) +{ + log_info("come into rocksdb replicate!"); + + int ret, totalRows = 0; + + // create resultWriter + ret = Task->prepare_result_no_limit(); + if (ret != 0) + { + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "task prepare-result error"); + log_error("task prepare-result error: %d, %m", ret); + return (-2); + } + + // key for replication start + std::string startKey, prevPrimaryKey, compoundKey, compoundValue; + RocksDBConn::RocksItr_t rocksItr; + + // whether start a newly replication or not + uint32_t start = Task->requestInfo.limit_start(); + uint32_t count = Task->requestInfo.limit_count(); + // if full replicate start from 0 and the start key is empty, means it's a newly replication + bool isBeginRepl = (start == 0); + if (likely(!isBeginRepl)) + { + // replicate with user given key + ret = value2Str(Task->request_key(), 0, startKey); + if (ret != 0 || startKey.empty()) + { + log_error("replicate key can not be empty!"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get replicate key failed!"); + return -1; + } + + // encode the key to rocksdb format + compoundKey = std::move(key_format::encode_bytes(startKey)); + if (compoundKey.empty()) + { + log_error("encode primary key failed! key:%s", startKey.c_str()); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "encode replicate key failed!"); + return -1; + } + + prevPrimaryKey = compoundKey; + + ret = mDBConn->search_lower_bound(compoundKey, compoundValue, rocksItr); + } + else + { +#if 0 + // get the last key for replication finished tag + ret = mDBConn->get_last_entry(compoundKey, compoundValue, rocksItr); + if ( ret < 0 ) + { + // replicate error, let the user decide to try again + log_error("get last key failed!"); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "get last replicate key failed!"); + return -1; + } + else if ( ret == 1 ) + { + // empty database + Task->set_total_rows(0); + return 0; + } + + // set the finished key of replicating into meta data column family + // delete the odd migrate-end-key from that may insert by the previous slave + mReplEndKey = compoundKey; + // use replace api to instead of insert, in case there has multi slave replicator, all + // of them should always replicate to the latest one key + ret = mDBConn->replace_entry(gReplicatePrefixKey, mReplEndKey, true, RocksDBConn::COLUMN_META_DATA); + if ( ret != 0 ) + { + log_error("save replicating-finished-key failed! key:%s", mReplEndKey.c_str()); + Task->set_error(-EC_ERROR_BASE, __FUNCTION__, "save replicating finished key failed!"); + return -1; + } +#endif + + // do forward retrieving for reducing duplicate replication + startKey = ""; + prevPrimaryKey = ""; + + ret = mDBConn->get_first_entry(compoundKey, compoundValue, rocksItr); + } + + if (ret < 0) + { + log_error("query rocksdb failed! isBeginRepl:%d, key:%s", isBeginRepl, startKey.c_str()); + Task->set_error_dup(ret, __FUNCTION__, "do replication failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // not found the key + Task->set_total_rows(0); + log_info("do full replication finished! %s", startKey.c_str()); + mDBConn->retrieve_end(rocksItr); + return 0; + } + + // iterate the matched prefix key and find out the real one from start to end + int replLen = 0; + while (true) + { + // 1.skip the user given key + // 2.the same prefix key only get once + if (key_matched(prevPrimaryKey, compoundKey) == 0) + { + // ignore the matched key that has been migrated in the previous call + } + else + { + // save row + ret = saveRow(compoundKey, compoundValue, false, totalRows, Task); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s, value:%s", compoundKey.c_str(), compoundValue.c_str()); + } + + key_format::get_format_key(compoundKey, mKeyfield_types[0], prevPrimaryKey); + } + + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, compoundKey, compoundValue); + if (ret < 0) + { + log_error("iterate rocksdb failed! key:%s", startKey.c_str()); + Task->set_error(ret, __FUNCTION__, "iterate rocksdb failed!"); + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the end + break; + } + + // has remaining value in rocksdb + if (totalRows >= count) + break; + // replLen += (compoundKey,length() + compoundValue.length()); + // if ( relpLen >= MAX_REPLICATE_LEN ) + // { + // // avoid network congestion + // break; + // } + } + + Task->set_total_rows(totalRows); + mDBConn->retrieve_end(rocksItr); + + return (0); +} + +int RocksdbProcess::process_direct_query( + DirectRequestContext *reqCxt, + DirectResponseContext *respCxt) +{ + log_info("come into process direct query!"); + +#ifdef PRINT_STAT + mSTime = GET_TIMESTAMP(); +#endif + + int ret; + + std::vector primaryKeyConds; + ret = analyse_primary_key_conds(reqCxt, primaryKeyConds); + if (ret != 0) + { + log_error("query condition incorrect in query context!"); + respCxt->sRowNums = -EC_ERROR_BASE; + return -1; + } + + // prefix key + std::string prefixKey = primaryKeyConds[0].sCondValue; + if (prefixKey.empty()) + { + log_error("dtc primary key can not be empty!"); + respCxt->sRowNums = -EC_ERROR_BASE; + return -1; + } + + if (mKeyfield_types[0] == DField::String) + std::transform(prefixKey.begin(), prefixKey.end(), prefixKey.begin(), ::tolower); + + // encode the key to rocksdb format + std::string fullKey = std::move(key_format::encode_bytes(prefixKey)); + std::string encodePreKey = fullKey; + + int totalRows = 0; + std::string value; + RocksDBConn::RocksItr_t rocksItr; + + bool forwardDirection = (primaryKeyConds[0].sCondOpr == (uint8_t)CondOpr::eEQ || primaryKeyConds[0].sCondOpr == (uint8_t)CondOpr::eGT || primaryKeyConds[0].sCondOpr == (uint8_t)CondOpr::eGE); + bool backwardEqual = primaryKeyConds[0].sCondOpr == (uint8_t)CondOpr::eLE; + if (backwardEqual) + { + // if the query condtion is < || <=, use seek_for_prev to seek in the total_order_seek mode + // will not use the prefix seek features, eg: + // 1. we have the flowing union key in the rocks (101,xx), (102,xx), (103,xx) + // 2. we use total_order_seek features for ranging query with primary key '102', and this + // lead the rocksdb doesn't use prefix_extractor to match the prefix key, so it use the + // entire key for comparing, and seek_for_prev will stop in the last key that <= the + // target key, so the iterator point to the key '(101, xx)', that's not what we want, + // wo need it point to the '(102,xx)' + + // do primary key equal query first in this section + ret = mDBConn->retrieve_start(fullKey, value, rocksItr, true); + if (ret < 0) + { + log_error("query rocksdb failed! key:%s, ret:%d", prefixKey.c_str(), ret); + respCxt->sRowNums = -EC_ERROR_BASE; + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 0) + { + while (true) + { + ret = key_matched(encodePreKey, fullKey); + if (ret != 0) + { + // prefix key not matched, reach to the end + break; + } + + // save row + ret = save_direct_row(prefixKey, fullKey, value, reqCxt, respCxt, totalRows); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s, value:%s", fullKey.c_str(), value.c_str()); + } + else if (ret == 1) + break; + + // move iterator to the next key + ret = mDBConn->next_entry(rocksItr, fullKey, value); + if (ret < 0) + { + log_error("iterate rocksdb failed! key:%s", prefixKey.c_str()); + respCxt->sRowNums = -EC_ERROR_BASE; + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the storage end + break; + } + + // has remaining value in rocksdb + } + } + } + + // range query in the following section + ret = mDBConn->retrieve_start(fullKey, value, rocksItr, false, forwardDirection); + if (ret < 0) + { + log_error("query rocksdb failed! key:%s, ret:%d", prefixKey.c_str(), ret); + respCxt->sRowNums = -EC_ERROR_BASE; + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // not found the key + log_info("no matched key:%s", prefixKey.c_str()); + respCxt->sRowNums = 0; + mDBConn->retrieve_end(rocksItr); + return 0; + } + + // iterate the matched prefix key and find out the real one from start to end + while (true) + { + ret = range_key_matched(fullKey, primaryKeyConds); + if (ret == -1) + { + // prefix key not matched, reach to the end + break; + } + + if (ret == 0) + { + // save row + ret = save_direct_row(prefixKey, fullKey, value, reqCxt, respCxt, totalRows); + if (ret < 0) + { + // ignore the incorrect key and keep going + log_error("save row failed! key:%s, value:%s", fullKey.c_str(), value.c_str()); + } + else if (ret == 1) + break; + } + + // move iterator to the next key + if (forwardDirection) + { + ret = mDBConn->next_entry(rocksItr, fullKey, value); + } + else + { + ret = mDBConn->prev_entry(rocksItr, fullKey, value); + } + if (ret < 0) + { + log_error("iterate rocksdb failed! key:%s", prefixKey.c_str()); + respCxt->sRowNums = ret; + mDBConn->retrieve_end(rocksItr); + return -1; + } + else if (ret == 1) + { + // reach to the storage end + break; + } + + // has remaining value in rocksdb + } + + // generate response rows in order by container + if (mOrderByUnit) + { + OrderByUnitElement element; + int start = reqCxt->sLimitCond.sLimitStart >= 0 && reqCxt->sLimitCond.sLimitStep > 0 ? reqCxt->sLimitCond.sLimitStart : 0; + while (true) + { + ret = mOrderByUnit->get_row(element); + if (0 == ret) + { + delete mOrderByUnit; + mOrderByUnit = NULL; + break; + } + + if (start != 0) + { + start--; + continue; + } + + build_direct_row(element.mRocksKeys, element.mRocksValues, respCxt); + } + } + + respCxt->sRowNums = respCxt->sRowValues.size(); + mDBConn->retrieve_end(rocksItr); + +#ifdef PRINT_STAT + mETime = GET_TIMESTAMP(); + insert_stat(OprType::eDirectQuery, mETime - mSTime); +#endif + + return (0); +} + +int RocksdbProcess::encode_dtc_key( + const std::string &oKey, + std::string &codedKey) +{ + int keyLen = oKey.length(); + static const int maxLen = 10240; + assert(sizeof(int) + keyLen <= maxLen); + static thread_local char keyBuff[maxLen]; + + char *pos = keyBuff; + *(int *)pos = keyLen; + pos += sizeof(int); + memcpy((void *)pos, (void *)oKey.data(), keyLen); + + codedKey.assign(keyBuff, keyLen + sizeof(int)); + + return 0; +} + +int RocksdbProcess::decode_keys( + const std::string &compoundKey, + std::vector &keys) +{ + int ret; + std::string keyField; + char *head = const_cast(compoundKey.data()); + + // decode dtckey first + int keyLen = *(int *)head; + head += sizeof(int); + keyField.assign(head, keyLen); + head += keyLen; + keys.push_back(std::move(keyField)); + + // decode other key fields + for (int idx = 1; idx < mCompoundKeyFieldNums; idx++) + { + ret = get_value_by_id(head, mFieldIndexMapping[idx], keyField); + assert(ret == 0); + keys.push_back(std::move(keyField)); + } + + return 0; +} + +int RocksdbProcess::encode_rocks_key( + const std::vector &keys, + std::string &rocksKey) +{ + assert(keys.size() == mCompoundKeyFieldNums); + + // evaluate space + static int align = 1 << 12; + int valueLen = 0, fid, fsize; + int totalLen = align; + char *valueBuff = (char *)malloc(totalLen); + + // encode key first + int keyLen = keys[0].length(); + *(int *)valueBuff = keyLen; + valueLen += sizeof(int); + memcpy((void *)valueBuff, (void *)keys[0].data(), keyLen); + valueLen += keyLen; + + for (size_t idx = 1; idx < mCompoundKeyFieldNums; idx++) + { + fid = mFieldIndexMapping[idx]; + switch (mTableDef->field_type(fid)) + { + case DField::Signed: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(int32_t)) + { + if (valueLen + sizeof(int64_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(int64_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int64_t *)(valueBuff + valueLen) = strtoll(keys[idx].c_str(), NULL, 10); + valueLen += sizeof(int64_t); + } + else + { + if (valueLen + sizeof(int32_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(int32_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int32_t *)(valueBuff + valueLen) = strtol(keys[idx].c_str(), NULL, 10); + valueLen += sizeof(int32_t); + } + break; + } + + case DField::Unsigned: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(uint32_t)) + { + if (valueLen + sizeof(uint64_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(uint64_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(uint64_t *)(valueBuff + valueLen) = strtoull(keys[idx].c_str(), NULL, 10); + valueLen += sizeof(uint64_t); + } + else + { + if (valueLen + sizeof(uint32_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(uint32_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(uint32_t *)(valueBuff + valueLen) = strtoul(keys[idx].c_str(), NULL, 10); + valueLen += sizeof(uint32_t); + } + break; + } + + case DField::Float: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(float)) + { + if (valueLen + sizeof(double) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(double) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(double *)(valueBuff + valueLen) = strtod(keys[idx].c_str(), NULL); + valueLen += sizeof(double); + } + else + { + if (valueLen + sizeof(float) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(float) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(float *)(valueBuff + valueLen) = strtof(keys[idx].c_str(), NULL); + valueLen += sizeof(float); + } + break; + } + + case DField::String: + case DField::Binary: + { + int len = keys[idx].length(); + fsize = len + sizeof(int); + { + if (valueLen + fsize > totalLen) + { + // expand buff + totalLen = (valueLen + fsize + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int *)(valueBuff + valueLen) = len; + valueLen += sizeof(int); + if (len > 0) + memcpy((void *)(valueBuff + valueLen), (void *)keys[idx].data(), len); + valueLen += len; + } + break; + } + + default: + log_error("unexpected field type! type:%d", mTableDef->field_type(fid)); + return -1; + }; + } + + rocksKey.assign(valueBuff, valueLen); + free(valueBuff); + + return 0; +} + +// 1. convert string type key into lower case +// 2. create bit map for those been converted keys +void RocksdbProcess::encode_bitmap_keys( + std::vector &keys, + std::string &keyBitmaps) +{ + if (mNoBitmapKey) + return; + + std::vector keyLocationBitmap, keyCaseBitmap; + int8_t localBits = 0; + bool hasBeenConverted = false; + + for (size_t idx = 0; idx < keys.size(); idx++) + { + switch (mKeyfield_types[idx]) + { + default: + hasBeenConverted = false; + break; + case DField::String: + { + // maybe need convert + std::vector currentKeyBitmap; + hasBeenConverted = convert_to_lower(keys[idx], currentKeyBitmap); + if (hasBeenConverted) + { + keyCaseBitmap.insert(keyCaseBitmap.end(), + std::make_move_iterator(currentKeyBitmap.begin()), + std::make_move_iterator(currentKeyBitmap.end())); + } + } + } + + // record key location bitmap + if (hasBeenConverted) + { + int shift = BITS_OF_BYTE - 1 - 1 - idx % (BITS_OF_BYTE - 1); + localBits = (localBits >> shift | 1U) << shift; + } + + // the last boundary bit in this section and has remaining keys, need to set the + // head bit for indicading + if ((idx + 1) % (BITS_OF_BYTE - 1) == 0 || idx == keys.size() - 1) + { + if (idx != keys.size() - 1) + localBits |= 128U; + keyLocationBitmap.push_back((char)localBits); + localBits = 0; + } + } + + // shrink bits to buffer + keyBitmaps.append( + std::string(keyLocationBitmap.begin(), keyLocationBitmap.end())) + .append( + std::string(keyCaseBitmap.begin(), keyCaseBitmap.end())); +} + +void RocksdbProcess::decodeBitmapKeys( + const std::string &rocksValue, + std::vector &keys, + int &bitmapLen) +{ + bitmapLen = 0; + + if (mNoBitmapKey) + return; + + int8_t sectionBits; + std::vector keyLocationBitmap; + + // decode key location bitmap + while (true) + { + sectionBits = rocksValue[bitmapLen]; + keyLocationBitmap.push_back(sectionBits); + bitmapLen++; + + if ((sectionBits & 0x80) == 0) + break; + } + + int shift = 0; + for (size_t idx = 0; idx < keys.size(); idx++) + { + sectionBits = keyLocationBitmap[idx / (BITS_OF_BYTE - 1)]; + shift = BITS_OF_BYTE - 1 - 1 - idx % (BITS_OF_BYTE - 1); + + switch (mKeyfield_types[idx]) + { + default: + assert((sectionBits >> shift & 1U) == 0); + break; + case DField::String: + { + if ((sectionBits >> shift & 1U) == 0) + { + // no need to do convert + } + else + { + // recovery the origin key + recover_to_upper(rocksValue, bitmapLen, keys[idx]); + } + } + } + } +} + +int RocksdbProcess::get_key_bitmap_len(const std::string &rocksValue) +{ + int bitmapLen = 0; + + if (mNoBitmapKey) + return bitmapLen; + + int8_t sectionBits; + std::deque keyLocationBitmap; + + // decode key location bitmap + while (true) + { + sectionBits = rocksValue[bitmapLen]; + keyLocationBitmap.push_back(sectionBits); + bitmapLen++; + + if ((sectionBits & 0x80) == 0) + break; + } + + int shift = 0; + while (keyLocationBitmap.size() != 0) + { + sectionBits = keyLocationBitmap.front(); + for (int8_t idx = 1; idx < BITS_OF_BYTE; idx++) + { + shift = BITS_OF_BYTE - 1 - idx; + + if ((sectionBits >> shift & 1U) == 1) + { + // collect the key bitmap len + int8_t keyBits; + while (true) + { + keyBits = (int8_t)rocksValue[bitmapLen++]; + if ((keyBits & 0x80) == 0) + break; + } + } + } + + keyLocationBitmap.pop_front(); + } + + return bitmapLen; +} + +bool RocksdbProcess::convert_to_lower( + std::string &key, + std::vector &keyCaseBitmap) +{ + bool hasConverted = false; + int8_t caseBits = 0; + char lowerBase = 'a' - 'A'; + for (size_t idx = 0; idx < key.length(); idx++) + { + char &cv = key.at(idx); + if (cv >= 'A' && cv <= 'Z') + { + cv += lowerBase; + + int shift = BITS_OF_BYTE - 1 - 1 - idx % (BITS_OF_BYTE - 1); + caseBits = (caseBits >> shift | 1U) << shift; + + hasConverted = true; + } + + if ((idx + 1) % (BITS_OF_BYTE - 1) == 0 || idx == key.length() - 1) + { + if (idx != key.length() - 1) + caseBits |= 128U; + keyCaseBitmap.push_back((char)caseBits); + caseBits = 0; + } + } + + return hasConverted; +} + +void RocksdbProcess::recover_to_upper( + const std::string &rocksValue, + int &bitmapLen, + std::string &key) +{ + int shift; + int kIdx = 0; + bool hasRemaining = true; + char upperBase = 'a' - 'A'; + int8_t sectionBits; + + do + { + sectionBits = rocksValue[bitmapLen]; + + shift = BITS_OF_BYTE - 1 - 1 - kIdx % (BITS_OF_BYTE - 1); + if (sectionBits >> shift & 1U) + { + // convert to upper mode + char &cc = key[kIdx]; + assert(cc >= 'a' && cc <= 'z'); + cc -= upperBase; + } + + kIdx++; + if (kIdx % (BITS_OF_BYTE - 1) == 0) + { + bitmapLen++; + hasRemaining = (sectionBits & 0x80) != 0; + } + } while (hasRemaining); +} + +int RocksdbProcess::shrink_value( + const std::vector &values, + std::string &rocksValue) +{ + assert(values.size() == mExtraValueFieldNums); + + // evaluate space + static int align = 1 << 12; + int valueLen = 0, fid, fsize; + int totalLen = align; + char *valueBuff = (char *)malloc(totalLen); + for (size_t idx = 0; idx < mExtraValueFieldNums; idx++) + { + fid = mFieldIndexMapping[mCompoundKeyFieldNums + idx]; + switch (mTableDef->field_type(fid)) + { + case DField::Signed: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(int32_t)) + { + if (valueLen + sizeof(int64_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(int64_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int64_t *)(valueBuff + valueLen) = strtoll(values[idx].c_str(), NULL, 10); + valueLen += sizeof(int64_t); + } + else + { + if (valueLen + sizeof(int32_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(int32_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int32_t *)(valueBuff + valueLen) = strtol(values[idx].c_str(), NULL, 10); + valueLen += sizeof(int32_t); + } + break; + } + + case DField::Unsigned: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(uint32_t)) + { + if (valueLen + sizeof(uint64_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(uint64_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(uint64_t *)(valueBuff + valueLen) = strtoull(values[idx].c_str(), NULL, 10); + valueLen += sizeof(uint64_t); + } + else + { + if (valueLen + sizeof(uint32_t) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(uint32_t) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(uint32_t *)(valueBuff + valueLen) = strtoul(values[idx].c_str(), NULL, 10); + valueLen += sizeof(uint32_t); + } + break; + } + + case DField::Float: + { + fsize = mTableDef->field_size(fid); + if (fsize > sizeof(float)) + { + if (valueLen + sizeof(double) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(double) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(double *)(valueBuff + valueLen) = strtod(values[idx].c_str(), NULL); + valueLen += sizeof(double); + } + else + { + if (valueLen + sizeof(float) > totalLen) + { + // expand buff + totalLen = (valueLen + sizeof(float) + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(float *)(valueBuff + valueLen) = strtof(values[idx].c_str(), NULL); + valueLen += sizeof(float); + } + break; + } + + case DField::String: + case DField::Binary: + { + int len = values[idx].length(); + fsize = len + sizeof(int); + { + if (valueLen + fsize > totalLen) + { + // expand buff + totalLen = (valueLen + fsize + align - 1) & -align; + valueBuff = expand_buff(totalLen, valueBuff); + if (!valueBuff) + return -1; + } + + *(int *)(valueBuff + valueLen) = len; + valueLen += sizeof(int); + if (len > 0) + memcpy((void *)(valueBuff + valueLen), (void *)values[idx].data(), len); + valueLen += len; + } + break; + } + + default: + log_error("unexpected field type! type:%d", mTableDef->field_type(fid)); + return -1; + }; + } + + rocksValue.assign(valueBuff, valueLen); + free(valueBuff); + + return 0; +} + +int RocksdbProcess::split_values( + const std::string &compoundValue, + std::vector &values) +{ + int ret; + std::string value; + char *head = const_cast(compoundValue.data()); + for (int idx = 0; idx < mExtraValueFieldNums; idx++) + { + ret = get_value_by_id(head, mFieldIndexMapping[mCompoundKeyFieldNums + idx], value); + assert(ret == 0); + values.push_back(std::move(value)); + } + + return 0; +} + +// translate dtcfid to rocksfid +int RocksdbProcess::translate_field_idx(int dtcfid) +{ + for (size_t idx = 0; idx < mFieldIndexMapping.size(); idx++) + { + if (mFieldIndexMapping[idx] == dtcfid) + return idx; + } + + return -1; +} + +int RocksdbProcess::get_value_by_id( + char *&valueHead, + int fieldId, + std::string &fieldValue) +{ + assert(valueHead); + + // evaluate space + int fsize; + int fieldType = mTableDef->field_type(fieldId); + switch (fieldType) + { + case DField::Signed: + { + fsize = mTableDef->field_size(fieldId); + if (fsize > sizeof(int32_t)) + { + int64_t value = *(int64_t *)(valueHead); + valueHead += sizeof(int64_t); + fieldValue = std::move(std::to_string(value)); + } + else + { + int32_t value = *(int32_t *)(valueHead); + valueHead += sizeof(int32_t); + fieldValue = std::move(std::to_string(value)); + } + break; + } + + case DField::Unsigned: + { + fsize = mTableDef->field_size(fieldId); + if (fsize > sizeof(uint32_t)) + { + uint64_t value = *(uint64_t *)(valueHead); + valueHead += sizeof(uint64_t); + fieldValue = std::move(std::to_string(value)); + } + else + { + uint32_t value = *(uint32_t *)(valueHead); + valueHead += sizeof(uint32_t); + fieldValue = std::move(std::to_string(value)); + } + break; + } + + case DField::Float: + { + fsize = mTableDef->field_size(fieldId); + if (fsize <= sizeof(float)) + { + float value = *(float *)(valueHead); + valueHead += sizeof(float); + fieldValue = std::move(std::to_string(value)); + } + else + { + double value = *(double *)(valueHead); + valueHead += sizeof(double); + fieldValue = std::move(std::to_string(value)); + } + break; + } + + case DField::String: + case DField::Binary: + { + int len; + { + len = *(int *)(valueHead); + valueHead += sizeof(int); + + fieldValue = std::move(std::string(valueHead, len)); + valueHead += len; + } + break; + } + + default: + log_error("unexpected field type! type:%d", fieldType); + return -1; + }; + + return 0; +} + +char *RocksdbProcess::expand_buff(int len, char *oldPtr) +{ + char *newPtr = (char *)realloc((void *)oldPtr, len); + if (!newPtr) + { + log_error("realloc memory failed!"); + free(oldPtr); + } + + return newPtr; +} + +// check two rocksdb key whether equal or not +int RocksdbProcess::rocks_key_matched(const std::string &rocksKey1, const std::string &rocksKey2) +{ + return rocksKey1.compare(rocksKey2); +} + +// check whether the key in the query conditon range matched or not +// 1 : in the range but not matched +// 0: key matched +// -1: in the out of the range +int RocksdbProcess::range_key_matched( + const std::string &rocksKey, + const std::vector &keyConds) +{ + std::string primaryKey; + int fieldType = mKeyfield_types[0]; + key_format::decode_primary_key(rocksKey, fieldType, primaryKey); + + int ret; + for (size_t idx = 0; idx < keyConds.size(); idx++) + { + ret = condition_filter(primaryKey, keyConds[idx].sCondValue, fieldType, keyConds[idx].sCondOpr); + if (ret != 0) + { + // check boundary value + switch (keyConds[idx].sCondOpr) + { + /* enum { + EQ = 0, + NE = 1, + LT = 2, + LE = 3, + GT = 4, + GE = 5, + }; */ + case 0: + case 1: // not support now + case 3: + case 5: + return -1; + case 2: + case 4: + return primaryKey.compare(keyConds[idx].sCondValue) == 0 ? 1 : -1; + default: + log_error("unsupport condition:%d", keyConds[idx].sCondOpr); + } + } + } + + return 0; +} + +int RocksdbProcess::analyse_primary_key_conds( + DirectRequestContext *reqCxt, + std::vector &primaryKeyConds) +{ + std::vector &queryConds = reqCxt->sFieldConds; + auto itr = queryConds.begin(); + while (itr != queryConds.end()) + { + if (itr->sFieldIndex == 0) + { + switch ((CondOpr)itr->sCondOpr) + { + case CondOpr::eEQ: + case CondOpr::eLT: + case CondOpr::eLE: + case CondOpr::eGT: + case CondOpr::eGE: + break; + case CondOpr::eNE: + default: + log_error("unsupport query expression now! condExpr:%d", itr->sCondOpr); + return -1; + } + primaryKeyConds.push_back(*itr); + itr = queryConds.erase(itr); + } + else + { + itr++; + } + } + + if (primaryKeyConds.size() <= 0) + { + log_error("no explicit primary key in query context!"); + return -1; + } + + return 0; +} + +void RocksdbProcess::init_title(int group, int role) +{ + titlePrefixSize = snprintf(name, sizeof(name), "helper%d%c", group, MACHINEROLESTRING[role]); + memcpy(title, name, titlePrefixSize); + title[titlePrefixSize++] = ':'; + title[titlePrefixSize++] = ' '; + title[titlePrefixSize] = '\0'; + title[sizeof(title) - 1] = '\0'; +} + +void RocksdbProcess::set_title(const char *status) +{ + strncpy(title + titlePrefixSize, status, sizeof(title) - 1 - titlePrefixSize); + set_proc_title(title); +} + +int RocksdbProcess::process_reload_config(DTCTask *Task) +{ + const char *keyStr = gConfig->get_str_val("cache", "CacheShmKey"); + int cacheKey = 0; + if (keyStr == NULL) + { + cacheKey = 0; + log_notice("CacheShmKey not set!"); + return -1; + } + else if (!strcasecmp(keyStr, "none")) + { + log_crit("CacheShmKey set to NONE, Cache disabled"); + return -1; + } + else if (isdigit(keyStr[0])) + { + cacheKey = strtol(keyStr, NULL, 0); + } + else + { + log_crit("Invalid CacheShmKey value \"%s\"", keyStr); + return -1; + } + CacheInfo stInfo; + DTCBufferPool bufPool; + memset(&stInfo, 0, sizeof(stInfo)); + stInfo.ipcMemKey = cacheKey; + stInfo.keySize = TableDefinitionManager::Instance()->get_cur_table_def()->key_format(); + stInfo.readOnly = 1; + + if (bufPool.cache_open(&stInfo)) + { + log_error("%s", bufPool.Error()); + Task->set_error(-EC_RELOAD_CONFIG_FAILED, __FUNCTION__, "open cache error!"); + return -1; + } + + bufPool.reload_table(); + log_error("cmd notify work helper reload table, tableIdx : [%d], pid : [%d]", bufPool.shm_table_idx(), getpid()); + return 0; +} + +void RocksdbProcess::insert_stat( + RocksdbProcess::OprType oprType, + int64_t timeElapse) +{ + assert(oprType >= OprType::eInsert && oprType < OprType::eDelete); + int opr = (int)oprType; + + if (timeElapse < 1000) + mOprTimeCost[opr][(int)TimeZone::eStatLevel0]++; + else if (timeElapse < 2000) + mOprTimeCost[opr][(int)TimeZone::eStatLevel1]++; + else if (timeElapse < 3000) + mOprTimeCost[opr][(int)TimeZone::eStatLevel2]++; + else if (timeElapse < 4000) + mOprTimeCost[opr][(int)TimeZone::eStatLevel3]++; + else if (timeElapse < 5000) + mOprTimeCost[opr][(int)TimeZone::eStatLevel4]++; + else + mOprTimeCost[opr][(int)TimeZone::eStatLevel5]++; + + mTotalOpr++; + + if (mTotalOpr % 10000 == 0) + print_stat_info(); + + return; +} + +void RocksdbProcess::print_stat_info() +{ + int totalNum; + + std::stringstream ss; + ss << "time cost per opr:\n"; + ss << "totalOpr:" << mTotalOpr << "\n"; + for (unsigned char idx0 = 0; idx0 <= (unsigned char)OprType::eQuery; idx0++) + { + switch ((OprType)idx0) + { + case OprType::eInsert: + { + totalNum = 0; + + ss << "Insert:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + case OprType::eUpdate: + { + totalNum = 0; + + ss << "Update:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + case OprType::eDirectQuery: + { + totalNum = 0; + + ss << "DirectQuery:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + case OprType::eQuery: + { + totalNum = 0; + + ss << "Query:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + case OprType::eReplace: + { + totalNum = 0; + + ss << "Replace:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + case OprType::eDelete: + { + totalNum = 0; + + ss << "Delete:["; + for (unsigned char idx1 = 0; idx1 < (unsigned char)TimeZone::eStatMax; idx1++) + { + ss << mOprTimeCost[idx0][idx1]; + if (idx1 != (unsigned char)TimeZone::eStatMax - 1) + ss << ", "; + totalNum += mOprTimeCost[idx0][idx1]; + } + ss << "] total:" << totalNum << "\n"; + break; + } + } + } + + log_error("%s", ss.str().c_str()); + + return; +} diff --git a/src/search_local/index_storage/rocksdb_helper/db_process_rocks.h b/src/search_local/index_storage/rocksdb_helper/db_process_rocks.h new file mode 100644 index 0000000..2a6e3d2 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/db_process_rocks.h @@ -0,0 +1,274 @@ + +/* + * ===================================================================================== + * + * Filename: db_process_rocks.h + * + * Description: Rocksdb invoker + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __DB_PROCESS_ROCKS_H__ +#define __DB_PROCESS_ROCKS_H__ + +#include "db_process_base.h" +#include "rocksdb_conn.h" +#include "key_format.h" +#include "rocksdb_key_comparator.h" +#include "rocksdb_direct_context.h" +#include "rocksdb_orderby_unit.h" + +class RocksdbProcess : public HelperProcessBase +{ + private: + int ErrorNo; + RocksDBConn* mDBConn; + + char name[16]; + char title[80]; + int titlePrefixSize; + + DTCTableDefinition* mTableDef; + int mCompoundKeyFieldNums; + int mExtraValueFieldNums; + std::vector mFieldIndexMapping; // DTC field index map to rocksdb, index is + // the fieldid in rocksdb, and the value is the fieldid in DTC + std::vector mReverseFieldIndexMapping; // rocksdb idx map to dtc, value is rocks idx + std::vector mKeyfield_types; + bool mNoBitmapKey; + + int SelfGroupID; + unsigned int procTimeout; + + // denoted finished key for replication + std::string mReplEndKey; + + // for migrating + std::string mPrevMigrateKey; // rocks key without suffix + std::string mCurrentMigrateKey; + int64_t mUncommitedMigId; + + RocksdbOrderByUnit* mOrderByUnit; + + // statistic info + enum class TimeZone : unsigned char + { + eStatLevel0, // < 1ms + eStatLevel1, // [1ms, 2ms) + eStatLevel2, // < [2ms, 3ms) + eStatLevel3, // < [3ms, 4ms) + eStatLevel4, // < [4ms, 5ms) + eStatLevel5, // >= 5ms + eStatMax + }; + + enum class OprType : unsigned char + { + eInsert, + eUpdate, + eDirectQuery, + eQuery, + eReplace, + eDelete + }; + + std::vector > mOprTimeCost = { + {0, 0, 0, 0, 0, 0}, // OprType::eInsert + {0, 0, 0, 0, 0, 0}, // update + {0, 0, 0, 0, 0, 0}, // direct query + {0, 0, 0, 0, 0, 0} // query + }; + int64_t mSTime; + int64_t mETime; + int64_t mTotalOpr; + + public: + RocksdbProcess(RocksDBConn *conn); + virtual ~RocksdbProcess(); + + int Init(int GroupID, const DbConfig* Config, DTCTableDefinition *tdef, int slave); + int check_table(); + + int process_task (DTCTask* Task); + + void init_title(int m, int t); + void set_title(const char *status); + const char *Name(void) { return name; } + void set_proc_timeout(unsigned int Seconds) { procTimeout = Seconds; } + + int process_direct_query( + DirectRequestContext* reqCxt, + DirectResponseContext* respCxt); + + protected: + void init_ping_timeout(void); + void use_matched_rows(void); + + inline int value2Str(const DTCValue* Value, int fieldId, std::string &strValue); + inline int setdefault_value(int field_type, DTCValue& Value); + inline int str2Value(const std::string &str, int field_type, DTCValue& Value); + std::string valueToStr(const DTCValue *value, int fieldType); + + int saveRow( + const std::string &compoundKey, + const std::string &compoundValue, + bool countOnly, + int &totalRows, + DTCTask* Task); + + int save_direct_row( + const std::string &prefixKey, + const std::string &compoundKey, + const std::string &compoundValue, + DirectRequestContext* reqCxt, + DirectResponseContext* respCxt, + int &totalRows); + + void build_direct_row( + const std::vector& keys, + const std::vector& values, + DirectResponseContext* respCxt); + + int update_row( + const std::string &prefixKey, + const std::string &compoundKey, + const std::string &compoundValue, + DTCTask* Task, + std::string &newKey, + std::string &newValue); + + int whether_update_key( + const DTCFieldValue* UpdateInfo, + bool &updateKey, + bool &updateValue); + + int shrink_value( + const std::vector &values, + std::string &rocksValue); + + int split_values( + const std::string &compoundValue, + std::vector &values); + + int translate_field_idx(int dtcfid); + + int get_value_by_id( + char* &valueHead, + int fieldId, + std::string &fieldValue); + + char* expand_buff(int len, char *oldPtr); + + int process_select(DTCTask* Task); + int process_insert(DTCTask* Task); + int process_update(DTCTask* Task); + int process_delete(DTCTask* Task); + int process_replace (DTCTask* Task); + int process_reload_config(DTCTask *Task); + int ProcessReplicate(DTCTask* Task); + + int value_add_to_str( + const DTCValue* additionValue, + int ifield_type, + std::string &baseValue); + + int condition_filter( + const std::string &rocksValue, + int fieldid, + int fieldType, + const DTCFieldValue *condition); + + int condition_filter( + const std::string &rocksValue, + const std::string& condValue, + int fieldType, + int comparator); + + template + bool is_matched_template(Args... len); + + template + bool is_matched( + const T lv, + int comparator, + const T rv); + + //template + bool is_matched( + const char *lv, + int comparator, + const char *rv, + int lLen, + int rLen, + bool caseSensitive); + + private: + int get_replicate_end_key(); + + int encode_dtc_key( + const std::string& oKey, + std::string& codedKey); + + int encode_rocks_key( + const std::vector &keys, + std::string &rocksKey); + + void encode_bitmap_keys( + std::vector& keys, + std::string& keyBitmaps); + + void decodeBitmapKeys( + const std::string& rocksValue, + std::vector& keys, + int& bitmapLen); + + int decode_keys( + const std::string& compoundKey, + std::vector& keys); + + bool convert_to_lower( + std::string& key, + std::vector& keyCaseBitmap); + + void recover_to_upper( + const std::string& rocksValue, + int& bitmapLen, + std::string& key); + + int get_key_bitmap_len(const std::string& rocksValue); + + // check dtc key whether match the prifix of the `rockskey` + inline int key_matched(const std::string& dtcKey, const std::string& rocksKey) + { + int dtcKLen = dtcKey.length(); + int rockKLen = rocksKey.length(); + + if ((dtcKLen == 0 && rockKLen != 0) || dtcKLen > rockKLen) return 1; + + // compare with case sensitive + return rocksKey.compare(0, dtcKLen, dtcKey); + } + + // check two rocksdb key whether equal or not + int rocks_key_matched(const std::string& rocksKey1, const std::string& rocksKey2); + int range_key_matched(const std::string& rocksKey, const std::vector& keyConds); + + int analyse_primary_key_conds(DirectRequestContext* reqCxt, std::vector& primaryKeyConds); + + void insert_stat( + RocksdbProcess::OprType opr, + int64_t timeElapse); + + void print_stat_info(); +}; + +#endif // __DB_PROCESS_ROCKS_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/helper_log_api.h b/src/search_local/index_storage/rocksdb_helper/helper_log_api.h new file mode 100644 index 0000000..df505fa --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/helper_log_api.h @@ -0,0 +1,117 @@ + +/* + * ===================================================================================== + * + * Filename: helper_log_api.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __HELPER_LOG_API_H__ +#define __HELPER_LOG_API_H__ + +#if HAS_LOGAPI + +#include +#include +#include + +#include "msglogapi.h" +#include "logfmt.h" +struct HelperLogApi +{ + HelperLogApi(); + ~HelperLogApi(); + + TMsgLog log; + unsigned int msec; + unsigned int iplocal, iptarget; + int id[4]; + + void Init(int id0, int id1, int id2, int id3); + void init_target(const char *host); + void Start(void); + void Done(const char *fn, int ln, const char *op, int ret, int err); +}; + +inline HelperLogApi::CHelperLogApi(void) : log(6, "dtc") +{ +} + +inline HelperLogApi::~CHelperLogApi(void) +{ +} + +extern "C" +{ + extern unsigned int get_local_ip(); +} +inline void HelperLogApi::Init(int id0, int id1, int id2, int id3) +{ + log_debug("LogApi: id %d caller %d target %d interface %d", id0, id1, id2, id3); + id[0] = id0; + id[1] = id1; + id[2] = id2; + id[3] = id3; + iplocal = get_local_ip(); + log_debug("local ip is %x\n", iplocal); +} + +inline void HelperLogApi::init_target(const char *host) +{ + if (host == NULL || host[0] == '\0' || host[0] == '/' || !strcasecmp(host, "localhost")) + host = "127.0.0.1"; + + struct in_addr a; + a.s_addr = inet_addr(host); + iptarget = *(unsigned int *)&a; + + log_debug("remote ip is %x\n", iptarget); +} + +inline void HelperLogApi::Start(void) +{ + INIT_MSEC(msec); +} + +inline void HelperLogApi::Done(const char *fn, int ln, const char *op, int ret, int err) +{ + CALC_MSEC(msec); + if (id[0] == 0) + return; + log.msgprintf((unsigned int)6, (unsigned long)id[0], + MAINTENANCE_MONITOR_MODULE_INTERFACE_LOG_FORMAT, + id[1], id[2], id[3], // 0: ID + iplocal, iptarget, // 3: IP + 0, 0, // 5: PORT + fn, ln, // 7: source position + "", // 9: file modification time + op, // 10: operation + err, // 11: MySQL ErrNo + ret ? 1 : 0, // 12: status + msec, -1, // 13: timing + 0, 0, 0, 0, "", "", "", "", ""); +} + +#else + +struct HelperLogApi +{ + void Init(int id0, int id1, int id2, int id3) {} + void init_target(const char *host) {} + void Start(void) {} + void Done(const char *fn, int ln, const char *op, int ret, int err) {} +}; + +#endif + +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/key_format.cc b/src/search_local/index_storage/rocksdb_helper/key_format.cc new file mode 100644 index 0000000..79ba9cc --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/key_format.cc @@ -0,0 +1,547 @@ + +/* + * ===================================================================================== + * + * Filename: key_format.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "key_format.h" +#include "protocol.h" +#include "log.h" + +#include + +#define SEGMENT_SIZE 8 + +const std::string SEG_SYMBOL = "|"; + +const char ENCODER_MARKER = 127; +const uint64_t signMask = 0x8000000000000000; + +uint64_t encode_into_cmp_uint(int64_t src) +{ + + return uint64_t(src) ^ signMask; +} + +uint64_t htonll(uint64_t val) +{ + return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); +} + +uint64_t ntohll(uint64_t val) +{ + return (((uint64_t)ntohl(val)) << 32) + ntohl(val >> 32); +} + +std::string key_format::Encode( + const std::map &fieldValues, + const DTCTableDefinition *table_def, + uint64_t &caseSensitiveFreeLen) +{ + caseSensitiveFreeLen = 0; + + const uint8_t *uniq_fields = table_def->uniq_fields_list(); + std::string temp_field; + std::string rockdb_key; + for (uint8_t i = 0; i < table_def->uniq_fields(); i++) + { + std::map::const_iterator key_value = fieldValues.find(uniq_fields[i]); + if (key_value != fieldValues.end()) + { + switch (table_def->field_type(uniq_fields[i])) + { + case DField::Signed: + rockdb_key.append(encode_bytes((int64_t)key_value->second->s64)); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Unsigned: + rockdb_key.append(encode_bytes((uint64_t)key_value->second->u64)); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Float: + rockdb_key.append(encode_bytes((double)key_value->second->flt)); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Binary: + rockdb_key.append(encode_bytes(std::string(key_value->second->bin.ptr, key_value->second->bin.len))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::String: + temp_field = std::move(encode_bytes(std::string(key_value->second->str.ptr, key_value->second->str.len))); + caseSensitiveFreeLen += temp_field.length(); + rockdb_key.append(std::move(temp_field)); + // rockdb_key.append(SEG_SYMBOL); + // caseSensitiveFreeLen += 1; + break; + } + } + } + return rockdb_key; +} + +std::string key_format::Encode( + const std::vector &fieldValues, + const DTCTableDefinition *table_def, + uint64_t &caseSensitiveFreeLen) +{ + caseSensitiveFreeLen = 0; + + const uint8_t *uniq_fields = table_def->uniq_fields_list(); + std::string temp_field; + std::string rockdb_key; + for (uint8_t i = 0; i < table_def->uniq_fields(); i++) + { + switch (table_def->field_type(uniq_fields[i])) + { + case DField::Signed: + + rockdb_key.append(encode_bytes((int64_t)strtoll(fieldValues[i].c_str(), NULL, 10))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Unsigned: + rockdb_key.append(encode_bytes((uint64_t)strtoull(fieldValues[i].c_str(), NULL, 10))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Float: + rockdb_key.append(encode_bytes(strtod(fieldValues[i].c_str(), NULL))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Binary: + rockdb_key.append(encode_bytes(fieldValues[i])); + // rockdb_key.append(SEG_SYMBOL); + case DField::String: + temp_field = std::move(encode_bytes(fieldValues[i])); + caseSensitiveFreeLen += temp_field.length(); + rockdb_key.append(std::move(temp_field)); + // rockdb_key.append(SEG_SYMBOL); + // caseSensitiveFreeLen += 1; + break; + } + } + return rockdb_key; +} + +std::string key_format::Encode( + const std::vector &fieldValues, + const std::vector &fieldTypes) +{ + std::string temp_field; + std::string rockdb_key; + log_info("fieldTypes size:%d", fieldTypes.size()); + for (size_t i = 0; i < fieldTypes.size(); i++) + { + switch (fieldTypes[i]) + { + case DField::Signed: + + rockdb_key.append(encode_bytes((int64_t)strtoll(fieldValues[i].c_str(), NULL, 10))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Unsigned: + rockdb_key.append(encode_bytes((uint64_t)strtoull(fieldValues[i].c_str(), NULL, 10))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Float: + rockdb_key.append(encode_bytes(strtod(fieldValues[i].c_str(), NULL))); + // rockdb_key.append(SEG_SYMBOL); + break; + case DField::Binary: + case DField::String: + rockdb_key.append(encode_bytes(fieldValues[i])); + // rockdb_key.append(SEG_SYMBOL); + break; + } + } + return rockdb_key; +} + +void key_format::Decode( + const std::string &src, + const std::vector &fieldTypes, + std::vector &fieldValues) +{ + fieldValues.clear(); + + size_t pos = 0; + for (size_t i = 0; i < fieldTypes.size(); i++) + { + std::string value; + switch (fieldTypes[i]) + { + case DField::Signed: + int64_t s64; + DecodeBytes(src.substr(pos, 8), s64); + pos += 8; + // value = new DTCValue(s64); + value = std::to_string(s64); + break; + case DField::Unsigned: + uint64_t u64; + DecodeBytes(src.substr(pos, 8), u64); + pos += 8; + // value = new DTCValue(u64); + value = std::to_string(u64); + break; + case DField::Float: + double d64; + DecodeBytes(src.substr(pos, 8), d64); + pos += 8; + // value = new DTCValue(d64); + value = std::to_string(d64); + break; + case DField::Binary: + case DField::String: + size_t begin_pos = pos; + pos += SEGMENT_SIZE; + // std::string str; + // for (; src[ pos - 1] == ENCODER_MARKER && src[pos] != SEG_SYMBOL[0] && pos < src.length() ; pos += SEGMENT_SIZE) { + // } + for (; src[pos - 1] == ENCODER_MARKER; pos += SEGMENT_SIZE) + { + } + // value = new DTCValue(str.c_str(), str.length()); + DecodeBytes(src.substr(begin_pos, pos - begin_pos), value); + // pos++; + // value = std::move(str); + break; + } + // fieldValues[uniq_fields[i]] = value; + fieldValues.push_back(value); + } +} + +void key_format::decode_primary_key( + const std::string &src, + int keyType, + std::string &pKey) +{ + switch (keyType) + { + default: + log_error("unsupport data type! type:%d", keyType); + break; + case DField::Signed: + int64_t s64; + DecodeBytes(src.substr(0, 8), s64); + pKey = std::to_string(s64); + break; + case DField::Unsigned: + uint64_t u64; + DecodeBytes(src.substr(0, 8), u64); + pKey = std::to_string(u64); + break; + case DField::Float: + double d64; + DecodeBytes(src.substr(0, 8), d64); + pKey = std::to_string(d64); + break; + case DField::Binary: + case DField::String: + size_t pos = 0; + pos += SEGMENT_SIZE; + for (; src[pos - 1] == ENCODER_MARKER; pos += SEGMENT_SIZE) + { + } + DecodeBytes(src.substr(0, pos), pKey); + break; + } + + return; +} + +int key_format::get_field_len( + const char *src, + int fieldType) +{ + int ret = -1; + switch (fieldType) + { + default: + log_error("unsupport data type! type:%d", fieldType); + break; + case DField::Signed: + case DField::Unsigned: + case DField::Float: + ret = 8; + break; + case DField::Binary: + case DField::String: + size_t pos = 0; + pos += SEGMENT_SIZE; + for (; src[pos - 1] == ENCODER_MARKER; pos += SEGMENT_SIZE) + { + } + ret = pos; + break; + } + + return ret; +} + +// get the first field in the row with encode format +int key_format::get_format_key( + const std::string &src, + int fieldType, + std::string &key) +{ + int ret = 0; + switch (fieldType) + { + default: + ret = -1; + log_error("unsupport data type! type:%d", fieldType); + break; + case DField::Signed: + case DField::Unsigned: + case DField::Float: + key = src.substr(0, 8); + break; + case DField::Binary: + case DField::String: + size_t pos = SEGMENT_SIZE; + for (; src[pos - 1] == ENCODER_MARKER; pos += SEGMENT_SIZE) + { + } + key = src.substr(0, pos); + break; + } + + return ret; +} + +// compare all the field one by one with its explicit type +int key_format::Compare( + const std::string &ls, + const std::string &rs, + const std::vector &fieldTypes) +{ + int ret, type, lFieldLen, rFieldLen, compLen; + char *lHead = (char *)ls.data(); + char *rHead = (char *)rs.data(); + for (size_t idx = 0; idx < fieldTypes.size(); idx++) + { + type = fieldTypes[idx]; + switch (type) + { + default: + ret = -2; + log_error("unsupport data type! type:%d", type); + break; + case DField::Signed: + case DField::Unsigned: + case DField::Float: + lFieldLen = rFieldLen = 8; + break; + case DField::Binary: + case DField::String: + lFieldLen = get_field_len(lHead, type); + rFieldLen = get_field_len(rHead, type); + break; + } + + compLen = lFieldLen > rFieldLen ? rFieldLen : lFieldLen; + if (type == DField::String) + { + // the case insensitive + int my_strn_case_cmp(const char *, const char *, size_t); + ret = my_strn_case_cmp(lHead, rHead, compLen); + } + else + { + // case sensitive compare + ret = memcmp((void *)lHead, (void *)rHead, compLen); + } + + if (ret != 0) + return ret; + else if (lFieldLen != rFieldLen) + return lFieldLen < rFieldLen ? -1 : 1; + + // equal in the current field + lHead += compLen; + rHead += compLen; + } + + return 0; +} + +void key_format::Decode( + const std::string &src, std::vector &fieldValues, + const DTCTableDefinition *table_def) +{ + fieldValues.clear(); + const uint8_t *uniq_fields = table_def->uniq_fields_list(); + + size_t pos = 0; + for (uint8_t i = 0; i < table_def->uniq_fields(); i++) + { + // DTCValue *value = NULL; + std::string value; + switch (table_def->field_type(uniq_fields[i])) + { + case DField::Signed: + int64_t s64; + DecodeBytes(src.substr(pos, 8), s64); + pos += 8; + // value = new DTCValue(s64); + value = std::to_string(s64); + break; + case DField::Unsigned: + uint64_t u64; + DecodeBytes(src.substr(pos, 8), u64); + pos += 8; + // value = new DTCValue(u64); + value = std::to_string(u64); + break; + case DField::Float: + double d64; + DecodeBytes(src.substr(pos, 8), d64); + pos += 8; + // value = new DTCValue(d64); + value = std::to_string(d64); + break; + case DField::Binary: + case DField::String: + size_t begin_pos = pos; + pos += SEGMENT_SIZE; + for (; src[pos - 1] == ENCODER_MARKER; pos += SEGMENT_SIZE) + { + } + // value = new DTCValue(str.c_str(), str.length()); + DecodeBytes(src.substr(begin_pos, pos - begin_pos), value); + break; + } + // fieldValues[uniq_fields[i]] = value; + fieldValues.push_back(value); + } +} + +std::string key_format::encode_bytes(const std::string &src) +{ + unsigned char padding_bytes; + size_t left_length = src.length(); + size_t pos = 0; + std::stringstream oss_dst; + while (true) + { + unsigned char copy_len = SEGMENT_SIZE - 1 < left_length ? SEGMENT_SIZE - 1 : left_length; + padding_bytes = SEGMENT_SIZE - 1 - copy_len; + oss_dst << src.substr(pos, copy_len); + pos += copy_len; + left_length -= copy_len; + + if (padding_bytes) + { + oss_dst << std::string(padding_bytes, '\0'); + oss_dst << (char)(ENCODER_MARKER - padding_bytes); + break; + } + else + { + oss_dst << ENCODER_MARKER; + } + } + return oss_dst.str(); +} + +std::string key_format::encode_bytes(int64_t src) +{ + uint64_t host_bytes = encode_into_cmp_uint(src); + uint64_t net_bytes = htonll(host_bytes); + char dst_bytes[8]; + memcpy(dst_bytes, &net_bytes, sizeof(uint64_t)); + std::string dst = std::string(8, '\0'); + for (size_t i = 0; i < dst.length(); i++) + { + dst[i] = dst_bytes[i]; + } + return dst; +} + +std::string key_format::encode_bytes(double src) +{ + uint64_t u; + memcpy(&u, &src, sizeof(double)); + if (src >= 0) + { + u |= signMask; + } + else + { + u = ~u; + } + + return encode_bytes(u); +} + +std::string key_format::encode_bytes(uint64_t src) +{ + uint64_t net_bytes = htonll(src); + char dst_bytes[8]; + memcpy(dst_bytes, &net_bytes, sizeof(uint64_t)); + std::string dst = std::string(8, '\0'); + for (size_t i = 0; i < dst.length(); i++) + { + dst[i] = dst_bytes[i]; + } + return dst; +} + +void key_format::DecodeBytes(const std::string &src, int64_t &dst) +{ + uint64_t net_bytes; + memcpy(&net_bytes, src.c_str(), sizeof(uint64_t)); + uint64_t host_bytes = ntohll(net_bytes); + dst = int64_t(host_bytes ^ signMask); +} + +void key_format::DecodeBytes(const std::string &src, std::string &dst) +{ + if (src.length() == 0) + { + dst = ""; + } + std::stringstream oss_dst; + for (size_t i = 0; i < src.length(); i += SEGMENT_SIZE) + { + char padding_bytes = ENCODER_MARKER - src[i + 7]; + oss_dst << src.substr(i, SEGMENT_SIZE - 1 - padding_bytes); + } + dst = oss_dst.str(); +} + +void key_format::DecodeBytes(const std::string &src, uint64_t &dst) +{ + uint64_t net_bytes; + memcpy(&net_bytes, src.c_str(), sizeof(uint64_t)); + dst = ntohll(net_bytes); +} + +void key_format::DecodeBytes(const std::string &src, double &dst) +{ + uint64_t u; + DecodeBytes(src, u); + + if ((u & signMask) > 0) + { + u &= (~signMask); + } + else + { + u = ~u; + } + memcpy(&dst, &u, sizeof(dst)); +} diff --git a/src/search_local/index_storage/rocksdb_helper/key_format.h b/src/search_local/index_storage/rocksdb_helper/key_format.h new file mode 100644 index 0000000..2ffe76a --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/key_format.h @@ -0,0 +1,80 @@ + +/* + * ===================================================================================== + * + * Filename: key_format.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __KEY_FORMAT_H__ +#define __KEY_FORMAT_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "table_def.h" + +class key_format +{ + +public: + static std::string Encode(const std::map &fieldValues, const DTCTableDefinition *table_def, uint64_t &caseSensitiveFreeLen); + static std::string Encode(const std::vector &fieldValues, const DTCTableDefinition *table_def, uint64_t &caseSensitiveFreeLen); + static void Decode(const std::string &src, std::vector &fieldValues, const DTCTableDefinition *table_def); + + static std::string Encode( + const std::vector &fieldValues, + const std::vector &fieldTypes); + + static void Decode( + const std::string &src, + const std::vector &fieldTypes, + std::vector &fieldValues); + + static void decode_primary_key( + const std::string &src, + int keyType, + std::string &pKey); + + static int get_format_key( + const std::string &src, + int fieldType, + std::string &key); + + static int get_field_len( + const char *src, + int fieldType); + + static int Compare( + const std::string &ls, + const std::string &rs, + const std::vector &fieldTypes); + + // private: + static std::string encode_bytes(const std::string &src); + static std::string encode_bytes(int64_t src); + static std::string encode_bytes(uint64_t src); + static std::string encode_bytes(double src); + static void DecodeBytes(const std::string &src, int64_t &dst); + static void DecodeBytes(const std::string &src, std::string &dst); + static void DecodeBytes(const std::string &src, uint64_t &dst); + static void DecodeBytes(const std::string &src, double &dst); +}; + +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/mysql_error.h b/src/search_local/index_storage/rocksdb_helper/mysql_error.h new file mode 100644 index 0000000..b2e3c88 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/mysql_error.h @@ -0,0 +1,292 @@ + +/* + * ===================================================================================== + * + * Filename: mysql_error.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_DTC_MYSQL_ERROR_H__ +#define __H_DTC_MYSQL_ERROR_H__ +enum +{ + ER_HASHCHK = 1000, + ER_NISAMCHK = 1001, + ER_NO = 1002, + ER_YES = 1003, + ER_CANT_CREATE_FILE = 1004, + ER_CANT_CREATE_TABLE = 1005, + ER_CANT_CREATE_DB = 1006, + ER_DB_CREATE_EXISTS = 1007, + ER_DB_DROP_EXISTS = 1008, + ER_DB_DROP_DELETE = 1009, + ER_DB_DROP_RMDIR = 1010, + ER_CANT_DELETE_FILE = 1011, + ER_CANT_FIND_SYSTEM_REC = 1012, + ER_CANT_GET_STAT = 1013, + ER_CANT_GET_WD = 1014, + ER_CANT_LOCK = 1015, + ER_CANT_OPEN_FILE = 1016, + ER_FILE_NOT_FOUND = 1017, + ER_CANT_READ_DIR = 1018, + ER_CANT_SET_WD = 1019, + ER_CHECKREAD = 1020, + ER_DISK_FULL = 1021, + ER_DUP_KEY = 1022, + ER_ERROR_ON_CLOSE = 1023, + ER_ERROR_ON_READ = 1024, + ER_ERROR_ON_RENAME = 1025, + ER_ERROR_ON_WRITE = 1026, + ER_FILE_USED = 1027, + ER_FILSORT_ABORT = 1028, + ER_FORM_NOT_FOUND = 1029, + ER_GET_ERRNO = 1030, + ER_ILLEGAL_HA = 1031, + ER_KEY_NOT_FOUND = 1032, + ER_NOT_FORM_FILE = 1033, + ER_NOT_KEYFILE = 1034, + ER_OLD_KEYFILE = 1035, + ER_OPEN_AS_READONLY = 1036, + ER_OUTOFMEMORY = 1037, + ER_OUT_OF_SORTMEMORY = 1038, + ER_UNEXPECTED_EOF = 1039, + ER_CON_COUNT_ERROR = 1040, + ER_OUT_OF_RESOURCES = 1041, + ER_BAD_HOST_ERROR = 1042, + ER_HANDSHAKE_ERROR = 1043, + ER_DBACCESS_DENIED_ERROR = 1044, + ER_ACCESS_DENIED_ERROR = 1045, + ER_NO_DB_ERROR = 1046, + ER_UNKNOWN_COM_ERROR = 1047, + ER_BAD_NULL_ERROR = 1048, + ER_BAD_DB_ERROR = 1049, + ER_TABLE_EXISTS_ERROR = 1050, + ER_BAD_TABLE_ERROR = 1051, + ER_NON_UNIQ_ERROR = 1052, + ER_SERVER_SHUTDOWN = 1053, + ER_BAD_FIELD_ERROR = 1054, + ER_WRONG_FIELD_WITH_GROUP = 1055, + ER_WRONG_GROUP_FIELD = 1056, + ER_WRONG_SUM_SELECT = 1057, + ER_WRONG_VALUE_COUNT = 1058, + ER_TOO_LONG_IDENT = 1059, + ER_DUP_FIELDNAME = 1060, + ER_DUP_KEYNAME = 1061, + ER_DUP_ENTRY = 1062, + ER_WRONG_FIELD_SPEC = 1063, + ER_PARSE_ERROR = 1064, + ER_EMPTY_QUERY = 1065, + ER_NONUNIQ_TABLE = 1066, + ER_INVALID_DEFAULT = 1067, + ER_MULTIPLE_PRI_KEY = 1068, + ER_TOO_MANY_KEYS = 1069, + ER_TOO_MANY_KEY_PARTS = 1070, + ER_TOO_LONG_KEY = 1071, + ER_KEY_COLUMN_DOES_NOT_EXITS = 1072, + ER_BLOB_USED_AS_KEY = 1073, + ER_TOO_BIG_FIELDLENGTH = 1074, + ER_WRONG_AUTO_KEY = 1075, + ER_READY = 1076, + ER_NORMAL_SHUTDOWN = 1077, + ER_GOT_SIGNAL = 1078, + ER_SHUTDOWN_COMPLETE = 1079, + ER_FORCING_CLOSE = 1080, + ER_IPSOCK_ERROR = 1081, + ER_NO_SUCH_INDEX = 1082, + ER_WRONG_FIELD_TERMINATORS = 1083, + ER_BLOBS_AND_NO_TERMINATED = 1084, + ER_TEXTFILE_NOT_READABLE = 1085, + ER_FILE_EXISTS_ERROR = 1086, + ER_LOAD_INFO = 1087, + ER_ALTER_INFO = 1088, + ER_WRONG_SUB_KEY = 1089, + ER_CANT_REMOVE_ALL_FIELDS = 1090, + ER_CANT_DROP_FIELD_OR_KEY = 1091, + ER_INSERT_INFO = 1092, + ER_INSERT_TABLE_USED = 1093, + ER_NO_SUCH_THREAD = 1094, + ER_KILL_DENIED_ERROR = 1095, + ER_NO_TABLES_USED = 1096, + ER_TOO_BIG_SET = 1097, + ER_NO_UNIQUE_LOGFILE = 1098, + ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099, + ER_TABLE_NOT_LOCKED = 1100, + ER_BLOB_CANT_HAVE_DEFAULT = 1101, + ER_WRONG_DB_NAME = 1102, + ER_WRONG_TABLE_NAME = 1103, + ER_TOO_BIG_SELECT = 1104, + ER_UNKNOWN_ERROR = 1105, + ER_UNKNOWN_PROCEDURE = 1106, + ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107, + ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108, + ER_UNKNOWN_TABLE = 1109, + ER_FIELD_SPECIFIED_TWICE = 1110, + ER_INVALID_GROUP_FUNC_USE = 1111, + ER_UNSUPPORTED_EXTENSION = 1112, + ER_TABLE_MUST_HAVE_COLUMNS = 1113, + ER_RECORD_FILE_FULL = 1114, + ER_UNKNOWN_CHARACTER_SET = 1115, + ER_TOO_MANY_TABLES = 1116, + ER_TOO_MANY_FIELDS = 1117, + ER_TOO_BIG_ROWSIZE = 1118, + ER_STACK_OVERRUN = 1119, + ER_WRONG_OUTER_JOIN = 1120, + ER_NULL_COLUMN_IN_INDEX = 1121, + ER_CANT_FIND_UDF = 1122, + ER_CANT_INITIALIZE_UDF = 1123, + ER_UDF_NO_PATHS = 1124, + ER_UDF_EXISTS = 1125, + ER_CANT_OPEN_LIBRARY = 1126, + ER_CANT_FIND_DL_ENTRY = 1127, + ER_FUNCTION_NOT_DEFINED = 1128, + ER_HOST_IS_BLOCKED = 1129, + ER_HOST_NOT_PRIVILEGED = 1130, + ER_PASSWORD_ANONYMOUS_USER = 1131, + ER_PASSWORD_NOT_ALLOWED = 1132, + ER_PASSWORD_NO_MATCH = 1133, + ER_UPDATE_INFO = 1134, + ER_CANT_CREATE_THREAD = 1135, + ER_WRONG_VALUE_COUNT_ON_ROW = 1136, + ER_CANT_REOPEN_TABLE = 1137, + ER_INVALID_USE_OF_NULL = 1138, + ER_REGEXP_ERROR = 1139, + ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140, + ER_NONEXISTING_GRANT = 1141, + ER_TABLEACCESS_DENIED_ERROR = 1142, + ER_COLUMNACCESS_DENIED_ERROR = 1143, + ER_ILLEGAL_GRANT_FOR_TABLE = 1144, + ER_GRANT_WRONG_HOST_OR_USER = 1145, + ER_NO_SUCH_TABLE = 1146, + ER_NONEXISTING_TABLE_GRANT = 1147, + ER_NOT_ALLOWED_COMMAND = 1148, + ER_SYNTAX_ERROR = 1149, + ER_DELAYED_CANT_CHANGE_LOCK = 1150, + ER_TOO_MANY_DELAYED_THREADS = 1151, + ER_ABORTING_CONNECTION = 1152, + ER_NET_PACKET_TOO_LARGE = 1153, + ER_NET_READ_ERROR_FROM_PIPE = 1154, + ER_NET_FCNTL_ERROR = 1155, + ER_NET_PACKETS_OUT_OF_ORDER = 1156, + ER_NET_UNCOMPRESS_ERROR = 1157, + ER_NET_READ_ERROR = 1158, + ER_NET_READ_INTERRUPTED = 1159, + ER_NET_ERROR_ON_WRITE = 1160, + ER_NET_WRITE_INTERRUPTED = 1161, + ER_TOO_LONG_STRING = 1162, + ER_TABLE_CANT_HANDLE_BLOB = 1163, + ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164, + ER_DELAYED_INSERT_TABLE_LOCKED = 1165, + ER_WRONG_COLUMN_NAME = 1166, + ER_WRONG_KEY_COLUMN = 1167, + ER_WRONG_MRG_TABLE = 1168, + ER_DUP_UNIQUE = 1169, + ER_BLOB_KEY_WITHOUT_LENGTH = 1170, + ER_PRIMARY_CANT_HAVE_NULL = 1171, + ER_TOO_MANY_ROWS = 1172, + ER_REQUIRES_PRIMARY_KEY = 1173, + ER_NO_RAID_COMPILED = 1174, + ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175, + ER_KEY_DOES_NOT_EXITS = 1176, + ER_CHECK_NO_SUCH_TABLE = 1177, + ER_CHECK_NOT_IMPLEMENTED = 1178, + ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179, + ER_ERROR_DURING_COMMIT = 1180, + ER_ERROR_DURING_ROLLBACK = 1181, + ER_ERROR_DURING_FLUSH_LOGS = 1182, + ER_ERROR_DURING_CHECKPOINT = 1183, + ER_NEW_ABORTING_CONNECTION = 1184, + ER_DUMP_NOT_IMPLEMENTED = 1185, + ER_FLUSH_MASTER_BINLOG_CLOSED = 1186, + ER_INDEX_REBUILD = 1187, + ER_MASTER = 1188, + ER_MASTER_NET_READ = 1189, + ER_MASTER_NET_WRITE = 1190, + ER_FT_MATCHING_KEY_NOT_FOUND = 1191, + ER_LOCK_OR_ACTIVE_TRANSACTION = 1192, + ER_UNKNOWN_SYSTEM_VARIABLE = 1193, + ER_CRASHED_ON_USAGE = 1194, + ER_CRASHED_ON_REPAIR = 1195, + ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196, + ER_TRANS_CACHE_FULL = 1197, + ER_SLAVE_MUST_STOP = 1198, + ER_SLAVE_NOT_RUNNING = 1199, + ER_BAD_SLAVE = 1200, + ER_MASTER_INFO = 1201, + ER_SLAVE_THREAD = 1202, + ER_TOO_MANY_USER_CONNECTIONS = 1203, + ER_SET_CONSTANTS_ONLY = 1204, + ER_LOCK_WAIT_TIMEOUT = 1205, + ER_LOCK_TABLE_FULL = 1206, + ER_READ_ONLY_TRANSACTION = 1207, + ER_DROP_DB_WITH_READ_LOCK = 1208, + ER_CREATE_DB_WITH_READ_LOCK = 1209, + ER_WRONG_ARGUMENTS = 1210, + ER_NO_PERMISSION_TO_CREATE_USER = 1211, + ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212, + ER_LOCK_DEADLOCK = 1213, + ER_TABLE_CANT_HANDLE_FULLTEXT = 1214, + ER_CANNOT_ADD_FOREIGN = 1215, + ER_NO_REFERENCED_ROW = 1216, + ER_ROW_IS_REFERENCED = 1217, + ER_CONNECT_TO_MASTER = 1218, + ER_QUERY_ON_MASTER = 1219, + ER_ERROR_WHEN_EXECUTING_COMMAND = 1220, + ER_WRONG_USAGE = 1221, + ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222, + ER_CANT_UPDATE_WITH_READLOCK = 1223, + ER_MIXING_NOT_ALLOWED = 1224, + ER_DUP_ARGUMENT = 1225, + ER_USER_LIMIT_REACHED = 1226, + ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227, + ER_LOCAL_VARIABLE = 1228, + ER_GLOBAL_VARIABLE = 1229, + ER_NO_DEFAULT = 1230, + ER_WRONG_VALUE_FOR_VAR = 1231, + ER_WRONG_TYPE_FOR_VAR = 1232, + ER_VAR_CANT_BE_READ = 1233, + ER_CANT_USE_OPTION_HERE = 1234, + ER_NOT_SUPPORTED_YET = 1235, + ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236, + ER_SLAVE_IGNORED_TABLE = 1237, + ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238, + CR_UNKNOWN_ERROR = 1900, + CR_SOCKET_CREATE_ERROR = 1901, + CR_CONNECTION_ERROR = 1902, + CR_CONN_HOST_ERROR = 1903, + CR_IPSOCK_ERROR = 1904, + CR_UNKNOWN_HOST = 1905, + CR_SERVER_GONE_ERROR = 1906, + CR_VERSION_ERROR = 1907, + CR_OUT_OF_MEMORY = 1908, + CR_WRONG_HOST_INFO = 1909, + CR_LOCALHOST_CONNECTION = 1910, + CR_TCP_CONNECTION = 1911, + CR_SERVER_HANDSHAKE_ERR = 1912, + CR_SERVER_LOST = 1913, + CR_COMMANDS_OUT_OF_SYNC = 1914, + CR_NAMEDPIPE_CONNECTION = 1915, + CR_NAMEDPIPEWAIT_ERROR = 1916, + CR_NAMEDPIPEOPEN_ERROR = 1917, + CR_NAMEDPIPESETSTATE_ERROR = 1918, + CR_CANT_READ_CHARSET = 1919, + CR_NET_PACKET_TOO_LARGE = 1920, + CR_EMBEDDED_CONNECTION = 1921, + CR_PROBE_SLAVE_STATUS = 1922, + CR_PROBE_SLAVE_HOSTS = 1923, + CR_PROBE_SLAVE_CONNECT = 1924, + CR_PROBE_MASTER_CONNECT = 1925, + CR_SSL_CONNECTION_ERROR = 1926, + CR_MALFORMED_PACKET = 1927, + CR_WRONG_LICENSE = 1928, +}; +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_block_filter.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_block_filter.h new file mode 100644 index 0000000..9aeec43 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_block_filter.h @@ -0,0 +1,47 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_block_filter.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#pragma once +#if 0 +// user define bloom filter in rocksdb datablock SST file for performance improving +class HashBloomFilterPolicy : public FilterPolicy +{ +public: + const char* Name() const override { return "HashBloomFilterPolicy"; } + + // create bloom filter base on the hash code of the prefix key + void CreateFilter(const Slice* keys, int n, std::string* dst) const override + { + for (int i = 0; i < n; i++) { + uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); + PutFixed32(dst, h); + } + } + + bool KeyMayMatch(const Slice& key, const Slice& filter) const override + { + uint32_t h = Hash(key.data(), key.size(), 1); + for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) { + if (h == DecodeFixed32(filter.data() + i)) { + return true; + } + } + return false; + } +}; +#endif diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.cc b/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.cc new file mode 100644 index 0000000..4af9b2f --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.cc @@ -0,0 +1,992 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_conn.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "rocksdb_conn.h" +#include "adaptive_prefix_extractor.h" +#include "rocksdb_key_comparator.h" +#include "rocksdb_block_filter.h" +#include "rocksdb/table.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/trace_reader_writer.h" +#include + +#include +#include +#include +#include +#include + +#define USE_PREFIX_EXTRATOR +// #define USE_ROCKS_TRACER +// #define USE_STATISTIC_COUNTER + +RocksDBConn::RocksDBConn() + : mRocksEnv(NULL), + mRocksDb(NULL), + mHasConnected(false), + mDtckey_type(-1), + mFixedKeyLen(0) +{ +} + +int RocksDBConn::get_errno() +{ + if (dberr >= 2000) + return -(dberr - 100); + return -dberr; +} + +int RocksDBConn::get_raw_errno() +{ + return dberr; +} + +const char *RocksDBConn::get_err_msg() +{ + return achErr; +} + +int RocksDBConn::Open(const std::string &dbPath) +{ + if (mHasConnected) + { + log_info("rocksdb has been opened yet! path:%s", mRocksDbPath.c_str()); + return 0; + } + + mRocksDbPath = dbPath; + + Options options; + + // not check the underlying sstfile during DB open + //options.skip_checking_sst_files_sizes_on_db_open = true; + options.skip_stats_update_on_db_open = true; + options.level_compaction_dynamic_level_bytes = true; + + init_db_options(options); + + // create rocksdb logger + // if the log dir not exist, create it + int ret = create_rocks_dir(options.db_log_dir); + if (ret != 0) + return -1; + + Status s = CreateLoggerFromOptions(mRocksDbPath, options, &mRocksdbLogger); + mRocksdbLogger->SetInfoLogLevel(InfoLogLevel::DEBUG_LEVEL); // application may want + // the diffrent log level with rocksdb, set it manually + if (!s.ok() || !mRocksdbLogger) + { + log_error("create rocksdb logger failed, code:%d, subcode:%d, errmsg:%s", + s.code(), s.subcode(), s.getState()); + return -1; + } + options.info_log = mRocksdbLogger; + + ret = create_tracer(); + if (ret != 0) + return -1; + + // set memtable(include immutable memtable) related for flushing + init_flushing_options(options); + + // kernal environment + init_environment_options(options); + + // init table options + init_table_options(options); + + // due to its single-threaded compaction + init_compaction_options(options); + + // deprecated, use default case sensitive compare + ColumnFamilyOptions colOptions; + init_column_family_options(colOptions); + + // enfoce Rocksdb work on prefix search mode, create user defined prefix extrator + // for prefix search + // open db should have 'prefix_extractor' been set +#ifdef USE_PREFIX_EXTRATOR + { + if (!mPrefixExtractor) + mPrefixExtractor.reset(new AdaptivePrefixExtractor(mDtckey_type)); + options.prefix_extractor = mPrefixExtractor; + } +#endif + init_prefix_extractor(colOptions); + + // column family compaction behavior + options.max_subcompactions = 3; // concurrently compacte in Level0 -> level1, + + // create with two column families + // get column family from storage if they exist + std::vector columnFamiliesNames; + s = DB::ListColumnFamilies(options, mRocksDbPath, &columnFamiliesNames); + if (!s.ok() && !s.IsPathNotFound()) + goto REPORT_ERROR; + + if (s.IsPathNotFound() || columnFamiliesNames.size() == 0) + { + // no column families in the storage, need to create it + // enforce open to create the 'default' column family with customer comparator + // options.comparator = mCommKeyComparator; + s = DB::Open(options, mRocksDbPath, &mRocksDb); + if (!s.ok()) + goto REPORT_ERROR; + + columnFamiliesNames.push_back("COLUMN_REAL_DATA"); + columnFamiliesNames.push_back("COLUMN_META_DATA"); + // in this open style, rocksdb will create the default column, no need to create it + // in user space explicitly + // columnFamiliesNames.push_back(kDefaultColumnFamilyName); + s = mRocksDb->CreateColumnFamilies(colOptions, columnFamiliesNames, &mColumnHandles); + if (!s.ok()) + goto REPORT_ERROR; + } + else + { + // check column families + assert(columnFamiliesNames.size() == COLUMN_FAMILIES_NUM); + + // user need to specifically list all column families those aready lying in the rocksdb storage, + // include the 'default' one, and the column family name order must be same with the previous + // user-defined 'ColumnFamiliesIndex' enum + std::vector columnFamilies; + columnFamilies.push_back(ColumnFamilyDescriptor("COLUMN_REAL_DATA", colOptions)); + columnFamilies.push_back(ColumnFamilyDescriptor("COLUMN_META_DATA", colOptions)); + columnFamilies.push_back(ColumnFamilyDescriptor(kDefaultColumnFamilyName, colOptions)); + + // open db with all existed column families + s = DB::Open(options, mRocksDbPath, columnFamilies, &mColumnHandles, &mRocksDb); + if (!s.ok()) + goto REPORT_ERROR; + } + + mHasConnected = true; + return 0; + +REPORT_ERROR: + log_error("open rocksdb failed! path:%s, code:%d, subcode:%d, errmsg:%s", + mRocksDbPath.c_str(), s.code(), s.subcode(), s.getState()); + return -1; +} + +// sync memory value into disk +int RocksDBConn::Close() +{ + if (!mHasConnected) + { + log_info("rocksdb has been closed yet! path:%s", mRocksDbPath.c_str()); + return 0; + } + + Status s = mRocksDb->SyncWAL(); + if (!s.ok()) + { + // there has unreleased snapshot in the system, user should release the unreleased snapshots and try again + log_error("close rocksdb failed! code:%d, subCode:%d, errMsg:%s", s.code(), s.subcode(), s.getState()); + return -1; + } + + // drop column families + // s = mRocksDb->DropColumnFamily(mColumnHandles[1]); + // assert( s.ok() ); + for (auto handler : mColumnHandles) + { + // close the column families + s = mRocksDb->DestroyColumnFamilyHandle(handler); + assert(s.ok()); + } + mColumnHandles.clear(); + + s = mRocksDb->Close(); + assert(s.ok()); + + mRocksDb = NULL; + mHasConnected = false; + + // flush log + if (mRocksdbLogger) + mRocksdbLogger->Flush(); + + return 0; +} + +int RocksDBConn::get_entry( + const std::string &key, + std::string &value, + ColumnFamiliesIndex colIndex) +{ + assert(mRocksDb); + + ReadOptions options = ReadOptions(); + // force entire key matched even in prefix search mode + // ig. if column family is set to be work on prefix_mode(set prefix_extractor), when + // customer call get("foo"), it will return the key that prefix matched which satisfy + // the prefix_extractor rule + options.total_order_seek = true; + + Status s = mRocksDb->Get(options, mColumnHandles[colIndex], key, &value); + if (s.ok()) + return 0; + + // handle error + unsigned char code = s.code(); + return code == rocksdb::Status::kNotFound ? -ERROR_KEY_NOT_FOUND : -code; +} + +/* if there has partial concept, the same key must be in the same partion, otherwise, + * 'insert_entry' operation will be no longer atomic in case of the Rocks support mutil thread*/ +int RocksDBConn::insert_entry( + const std::string &key, + const std::string &value, + bool syncMode, + ColumnFamiliesIndex colIndex) +{ + int ret = key_exist(key, colIndex); + if (ret == 1) + { + // value has been exist in the db + log_info("value already exist in the db, key:%s", key.c_str()); + return -ERROR_DUPLICATE_KEY; + } + else if (ret < 0) + { + // encounter error + return ret; + } + + // key not found + ret = replace_entry(key, value, syncMode, colIndex); + if (ret != 0) + { + log_error("insert key failed, key:%s, value:%s", key.c_str(), value.c_str()); + } + + return ret; +} + +int RocksDBConn::update_entry( + const std::string &key, + const std::string &value, + bool syncMode, + ColumnFamiliesIndex colIndex) +{ + int ret = key_exist(key, colIndex); + if (ret == 0) + { + // can not update non-exist row + log_info("value not exist in the db, key:%s", key.c_str()); + return -ERROR_KEY_NOT_FOUND; + } + else if (ret < 0) + return ret; + + return replace_entry(key, value, syncMode, colIndex); +} + +int RocksDBConn::replace_entry( + const std::string &key, + const std::string &value, + bool syncMode, + ColumnFamiliesIndex colIndex) +{ + WriteOptions opt; + opt.sync = syncMode; + + Status s = mRocksDb->Put(opt, mColumnHandles[colIndex], key, value); + if (s.ok()) + return 0; + + log_error("update value failed, key:%s, value:%s, errMsg:%s, code:%d, subCode:%d", key.c_str(), value.c_str(), s.getState(), s.code(), s.subcode()); + return -s.code(); +} + +int RocksDBConn::delete_entry( + const std::string &key, + bool syncMode, + ColumnFamiliesIndex colIndex) +{ + WriteOptions opt; + opt.sync = syncMode; + + Status s = mRocksDb->Delete(opt, mColumnHandles[colIndex], key); + if (s.ok()) + return 0; + + log_error("remove key failed, key:%s, errMsg:%s, code:%d, subCode:%d", key.c_str(), s.getState(), s.code(), s.subcode()); + return -s.code(); +} + +int RocksDBConn::single_delete_entry( + const std::string &key, + bool syncMode, + ColumnFamiliesIndex colIndex) +{ + WriteOptions opt; + opt.sync = syncMode; + + Status s = mRocksDb->SingleDelete(opt, mColumnHandles[colIndex], key); + if (s.ok()) + return 0; + + log_error("single remove key failed, key:%s, errMsg:%s, code:%d, subCode:%d", key.c_str(), s.getState(), s.code(), s.subcode()); + return -s.code(); +} + +int RocksDBConn::batch_update( + const std::set &deletedKeys, + const std::map &newEntries, + const bool syncMode, + ColumnFamiliesIndex colIndex) +{ + WriteOptions opt; + opt.sync = syncMode; + + WriteBatch batch; + auto sItr = deletedKeys.begin(); + while (sItr != deletedKeys.end()) + { + batch.Delete(mColumnHandles[colIndex], *sItr); + sItr++; + } + + auto mitr = newEntries.begin(); + while (mitr != newEntries.end()) + { + batch.Put(mColumnHandles[colIndex], mitr->first, mitr->second); + mitr++; + } + + TraceOptions traceOpt; + +#ifdef USE_ROCKS_TRACER + assert(mRocksDb->StartTrace(traceOpt, std::move(mTraceWriter)).ok()); +#endif + + Status s = mRocksDb->Write(opt, &batch); + if (s.ok()) + return 0; + +#ifdef USE_ROCKS_TRACER + assert(mRocksDb->EndTrace().ok()); +#endif + + log_error("do atomic write failed!"); + return -s.code(); +} + +// start a transaction, caller should hold the iterator and call 'next_entry' with it the +// next time +// return value: +// -1 : error occur +// 0 : get value success and not reach to the end +// 1: reach to the end, no more key left +int RocksDBConn::retrieve_start( + std::string &startKey, + std::string &value, + RocksItr_t &itr, + bool searchMode, + bool forwardDirection, + ColumnFamiliesIndex colIndex) +{ + if (startKey.empty()) + { + log_info("empty retrieve key!"); + return -1; + } + + ReadOptions options = ReadOptions(); + + // this option is effective only when customer both define 'prefix_extractor' and the + // 'total_order_seek' is false + options.prefix_same_as_start = searchMode; + + // if open rocksdb with prefix bloom filter mode, related search operation will be work on + // prefix bloom filter, and it only guarantee the order for the partial keys those with the + // same prefix, the total key order will not been guaranteed when iteration the whole keys, + // if we want to scan the keys with total order, set 'total_order_seek' to true + if (!searchMode) + options.total_order_seek = true; + options.fill_cache = false; + + // start retrieving + itr = mRocksDb->NewIterator(options, mColumnHandles[colIndex]); + if (forwardDirection) + { + // point at the first key that equal or larger then the 'startKey', search forward + itr->Seek(startKey); + } + else + { + // point at the last key that equal or less than the 'startKey', search backward + itr->SeekForPrev(startKey); + } + + if (!itr->Valid()) + { + if (itr->status().ok()) + { + // reach to the end, no more key left + log_info("retrieing reach to the end!"); + return 1; + } + + // internal error + log_error("retrieving internal error, not found the given key, it maybe deleted\ + during we held the snapshot! key:%s, errMsg:%s", + startKey.c_str(), itr->status().getState()); + return -1; + } + + // bring back the key + startKey = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +// release the snapshot that the iterator held +int RocksDBConn::retrieve_end(RocksItr_t itr) +{ + if (itr) + delete itr; + // mIterator = NULL; + + return 0; +} + +// return value: +// -1 : error occur +// 0 : get value success and not reach to the end +// 1: reach to the end, no more key left +int RocksDBConn::prev_entry( + RocksItr_t itr, + std::string &key, + std::string &value) +{ + assert(itr); + + itr->Prev(); + if (!itr->Valid()) + { + if (itr->status().ok()) + { + // reach to the end, no more key left + log_info("retrieing reach to the end!"); + return 1; + } + + // internal error + log_error("retrieving internal error, not found the given key, it maybe deleted\ + during we held the snapshot! errMsg:%s", + itr->status().getState()); + return -1; + } + + key = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +// return value: +// -1 : error occur +// 0 : get value success and not reach to the end +// 1: reach to the end, no more key left +int RocksDBConn::next_entry( + RocksItr_t itr, + std::string &key, + std::string &value) +{ + assert(itr); + + itr->Next(); + if (!itr->Valid()) + { + if (itr->status().ok()) + { + // reach to the end, no more key left + log_info("retrieing reach to the end!"); + return 1; + } + + // internal error + log_error("retrieving internal error, not found the given key, it maybe deleted during we held the snapshot! errMsg:%s", itr->status().getState()); + return -1; + } + + key = itr->key().ToString(); + + /* + // check whether read to the end(prefix search finished) + size_t len = prefixKey.length(); + if ( len > key.length() || key.compare(0, len, prefixKey) != 0 ) + { + log_info("retrieve over!"); + return 1; + } + */ + value = itr->value().ToString(); + + return 0; +} + +// searching not in prefix mode +int RocksDBConn::search_lower_bound( + std::string &targetKey, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex) +{ + if (targetKey.empty()) + { + log_error("not permited to search empty key!"); + return -1; + } + + ReadOptions options = ReadOptions(); + options.fill_cache = false; + options.total_order_seek = true; + + // start retrieving + itr = mRocksDb->NewIterator(options, mColumnHandles[colIndex]); + { + // iterator point at the first element which >= 'targetKey' + itr->Seek(targetKey); + } + + if (!itr->Valid()) + { + if (itr->status().ok()) + { + // reach to the end, no more key left + log_info("retrieing reach to the end! key:%s", targetKey.c_str()); + return 1; + } + + // internal error + log_error("retrieving internal error, not found the given key, it maybe deleted during we held the snapshot! key:%s, errMsg:%s", targetKey.c_str(), itr->status().getState()); + return -1; + } + + targetKey = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +int RocksDBConn::search_upper_bound( + std::string &targetKey, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex) +{ + if (targetKey.empty()) + { + log_error("not permited to search empty key!"); + return -1; + } + + ReadOptions options = ReadOptions(); + options.fill_cache = false; + options.total_order_seek = true; + + // start retrieving + itr = mRocksDb->NewIterator(options, mColumnHandles[colIndex]); + { + // iterator point at the last element that <= 'targetKey' + itr->SeekForPrev(targetKey); + } + + if (!itr->Valid()) + { + if (itr->status().ok()) + { + // reach to the end, no more key left + log_info("retrieing reach to the end! key:%s", targetKey.c_str()); + return 1; + } + + // internal error + log_error("retrieving internal error, not found the given key, it maybe deleted during we held the snapshot! key:%s, errMsg:%s", targetKey.c_str(), itr->status().getState()); + return -1; + } + + targetKey = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +int RocksDBConn::get_first_entry( + std::string &key, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex) +{ + ReadOptions options = ReadOptions(); + options.fill_cache = false; + options.readahead_size = 1; + options.total_order_seek = true; + + // start retrieving + itr = mRocksDb->NewIterator(options, mColumnHandles[colIndex]); + { + // search the firsr key + itr->SeekToFirst(); + } + + if (!itr->Valid()) + { + // empty database + log_info("empty database! errMsg:%s", itr->status().getState()); + return 1; + } + + key = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +int RocksDBConn::get_last_entry( + std::string &key, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex) +{ + ReadOptions options = ReadOptions(); + options.fill_cache = false; + options.readahead_size = 1; // disable readahead + options.total_order_seek = true; + + // start retrieving + itr = mRocksDb->NewIterator(options, mColumnHandles[colIndex]); + { + // search the firsr key + itr->SeekToLast(); + } + + if (!itr->Valid()) + { + // empty database + log_info("empty database! errMsg:%s", itr->status().getState()); + return 1; + } + + key = itr->key().ToString(); + value = itr->value().ToString(); + + return 0; +} + +uint32_t RocksDBConn::escape_string(char To[], const char *From) +{ + + // not support now + return -1; +} + +RocksDBConn::~RocksDBConn() +{ + Close(); +} + +int RocksDBConn::key_exist( + const std::string &key, + ColumnFamiliesIndex colIndex) +{ + // bool hasValue = false; + + ReadOptions options = ReadOptions(); + options.fill_cache = false; + + std::string originValue; + // bool rslt = mRocksDb->KeyMayExist(options, mColumnHandles[colIndex], key, &originValue, &hasValue); + bool rslt = mRocksDb->KeyMayExist(options, mColumnHandles[colIndex], key, &originValue); + if (!rslt) + { + // key definitely does not exist in the database + return 0; + } + + // use default bloom filter to figure out the key exist or not but still have + // 1% false positive rate, double check + // if ( !hasValue ) return false; + int ret = get_entry(key, originValue, colIndex); + if (ret == -ERROR_KEY_NOT_FOUND) + return 0; + if (ret == 0) + { + // value has been exist in the db + log_error("value already exist in the db, ret:%d, key:%s, value:%s", ret, key.c_str(), originValue.c_str()); + return 1; + } + + log_error("find key failed, ret:%d, key:%s", ret, key.c_str()); + return ret; +} + +int RocksDBConn::create_rocks_dir(const std::string &stuffDir) +{ + int ret = access(stuffDir.c_str(), F_OK); + if (ret != 0) + { + int err = errno; + if (errno == ENOENT) + { + // create log dir + if (mkdir(stuffDir.c_str(), 0755) != 0) + { + log_error("create rocksdb dir failed! path:%s, errno:%d", stuffDir.c_str(), errno); + return -1; + } + } + else + { + log_error("access rocksdb stuff dir failed!, path:%s, errno:%d", stuffDir.c_str(), errno); + return -1; + } + } + + return 0; +} + +/* open statistic switch maybe lead 5% ~ 10% performance hit */ +void RocksDBConn::init_statistic_info(Options &option) +{ + option.statistics = rocksdb::CreateDBStatistics(); + + // time period to print statistic info into log file, set it to 30min + option.stats_dump_period_sec = (60 * 30); + + return; +} + +void RocksDBConn::init_prefix_extractor(ColumnFamilyOptions &option) +{ +#ifdef USE_PREFIX_EXTRATOR + { + if (!mPrefixExtractor) + mPrefixExtractor.reset(new AdaptivePrefixExtractor(mDtckey_type)); + } + option.prefix_extractor = mPrefixExtractor; +#endif + + // enable prefix bloom filter for memtable, bloom filter size is base on write_buffer_size + option.memtable_prefix_bloom_size_ratio = 0.25; + + // create bloom filter for entire key in memtable + option.memtable_whole_key_filtering = true; + + // disable the bloom filter in the last level + option.optimize_filters_for_hits = true; + + return; +} + +// storing data in block-based table with hard disk and in Plain table for pure-memory or ssd +void RocksDBConn::init_table_options(Options &option) +{ + BlockBasedTableOptions tableOpt; + + // false: hold all bloom filter in the memory + // true: evict corrupt bloom filter and index blocks from cache when memory exhausted + tableOpt.cache_index_and_filter_blocks = true; + tableOpt.pin_l0_filter_and_index_blocks_in_cache = true; + + // init the datablock index type, friendly for prefix search + // tableOpt.index_type = BlockBasedTableOptions::kHashSearch; + + // init datablock to hash index feature, even though use this feature is very hard(must make + // sure that the key's with different byte content not be equal, case sensitive), but it can + // backward compatible to the binary search + tableOpt.data_block_index_type = BlockBasedTableOptions::kDataBlockBinaryAndHash; + tableOpt.data_block_hash_table_util_ratio = 0.75; + + // enable block cache and use evict algorithm + tableOpt.no_block_cache = false; // disable block cache + // tableOpt.block_restart_interval = 4; + + tableOpt.read_amp_bytes_per_bit = 0; + tableOpt.block_size = 64 << 10; + + // also build bloom filter for the entire key + tableOpt.whole_key_filtering = true; + + // create bloom filter for SST file to reduce read amplification, and use 10 bits to store + // one key in bloom filter. + // true: block-based filter, create bloom filter for each seprated block in sst file + // false: full filter, only create one bloom filter for the whole sst file + // full filter will be 40% performance improve for consulting + tableOpt.filter_policy.reset(NewBloomFilterPolicy(10, false)); + + option.table_factory.reset(NewBlockBasedTableFactory(tableOpt)); + + return; +} + +// set option for controlling rocksdb writing and flushing +void RocksDBConn::init_flushing_options(Options &option) +{ + // single memtable size(the same as immutable memtable) + option.write_buffer_size = 64 << 20; + + // total number of memtable in memory, include memtable and immutable-memtable + option.max_write_buffer_number = 5; + + // minimum number of memtables to be merged before flushing them to storage, if assign a + // If multiple memtables are merged together, less data may be written to storage since two + // updates are merged to a single key. However, every Get() must traverse all immutable + // memtables linearly to check if the key is there. Setting this option too high may hurt + // read performance. + option.min_write_buffer_number_to_merge = 2; + + return; +} + +// set disk and memory in posix environment +void RocksDBConn::init_environment_options(Options &option) +{ + option.new_table_reader_for_compaction_inputs = true; +} + +void RocksDBConn::init_compaction_options(Options &colOptions) +{ + colOptions.num_levels = 7; + + // use finer control of compression style of each level to instead of options.compression + // configure compression type for each level + // It usually makes sense to avoid compressing levels 0 and 1 and to compress data only in + // higher levels. You can even set slower compression in highest level and faster compression + // in lower levels (by highest we mean Lmax). + std::vector levelCompressionStyles; + levelCompressionStyles.push_back(CompressionType::kNoCompression); // no compression in level 0 + levelCompressionStyles.push_back(CompressionType::kNoCompression); // level 1 also not compression + levelCompressionStyles.push_back(CompressionType::kSnappyCompression); // level 2 use quick snappy compress + levelCompressionStyles.push_back(CompressionType::kSnappyCompression); + levelCompressionStyles.push_back(CompressionType::kSnappyCompression); + levelCompressionStyles.push_back(CompressionType::kSnappyCompression); + levelCompressionStyles.push_back(CompressionType::kZlibCompression); + colOptions.compression_per_level = levelCompressionStyles; + + // Usually the bottommost level contains majority of the data, so users get an almost + // optimal space setting, without paying CPU for compressing all the data at any level. + // We recommend ZSTD. If it is not available, Zlib is the second choice. + //colOptions.bottommost_compression = CompressionType::kZSTD; + + // use level style compaction to reduce read amplification and the universal style compaction + // for reducing write amplification. + colOptions.compaction_style = rocksdb::CompactionStyle::kCompactionStyleLevel; + + // when number of files reside on level 0 bigger than it, trigger compaction + colOptions.level0_file_num_compaction_trigger = 2; + + colOptions.level0_slowdown_writes_trigger = 20; + colOptions.level0_stop_writes_trigger = 36; + colOptions.write_buffer_size = 64 << 20; + + // file size in level1 + colOptions.target_file_size_base = 32 << 20; + colOptions.max_bytes_for_level_base = 64 << 20; + + colOptions.max_compaction_bytes = 100 << 20; + colOptions.hard_pending_compaction_bytes_limit = 512ULL << 30; + + return; +} + +void RocksDBConn::init_column_family_options(ColumnFamilyOptions &colOptions) +{ + colOptions.max_write_buffer_number = 5; + + // number should follwed the 'max_write_buffer_number' + colOptions.min_write_buffer_number_to_merge = 1; + + // due to its single-threaded compaction + // init_compaction_options(colOptions); + + // use default bytewise comparator + // mCommKeyComparator = new CaseSensitiveFreeComparator(); + // colOptions.comparator = mCommKeyComparator; +} + +void RocksDBConn::init_db_options(Options &options) +{ + // set concurrently operate + options.IncreaseParallelism(5); + + mRocksEnv = Env::Default(); + // To increase the number of threads in each pool manually + // mRocksEnv->SetBackgroundThreads(1, Env::BOTTOM); + // mRocksEnv->SetBackgroundThreads(1, Env::LOW); + // mRocksEnv->SetBackgroundThreads(1, Env::HIGH); // flush thread is + // usually good enough to be set to 1 by rocksdb automatically, no need to add + options.env = mRocksEnv; + + // create the DB if it's not already present + options.create_if_missing = true; + + options.wal_dir = std::move(std::string(mRocksDbPath).append("/wallog")); + options.max_total_wal_size = 4 << 30; + + // init rocksdb statistic info for monitoring the performance bottleneck +#ifdef USE_STATISTIC_COUNTER + init_statistic_info(options); +#endif + + options.compaction_readahead_size = 2 << 20; // 2M + + // 1.in LSM architecture, there are two background process:flush and compaction, + // they can execute concurrently, the flush thread are in the HIGH priority thread + // pool and the compaction threads are in the LOW priority thread pool + + // max_background_compactions and max_background_flushes is no longer supported now, + // they automatically decides by 'max_background_jobs' + options.max_background_jobs = 8; + + options.max_file_opening_threads = 3; + options.max_manifest_file_size = 20 << 20; + + options.db_log_dir = std::move(std::string(mRocksDbPath).append("/log")); + options.max_log_file_size = 50 << 20; // 50M per log file size + + options.keep_log_file_num = 100; // keep 1GB log in disk + options.info_log_level = InfoLogLevel::ERROR_LEVEL; // set default log level to ERROR + + options.target_file_size_base = 32 << 20; + + options.max_successive_merges = 1000; + + return; +} + +int RocksDBConn::create_tracer() +{ +#ifndef USE_ROCKS_TRACER + return 0; +#endif + + std::string tracerFilePath = std::move(std::string(mRocksDbPath).append("/tracer")); + int ret = create_rocks_dir(tracerFilePath); + if (ret != 0) + return -1; + + EnvOptions envOptions; + NewFileTraceWriter(mRocksEnv, envOptions, tracerFilePath, &mTraceWriter); +} diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.h new file mode 100644 index 0000000..8f502eb --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_conn.h @@ -0,0 +1,237 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_conn.h + * + * Description: Operation for RocksDB and use Default column family + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ROCKS_DB_CONN_H__ +#define __ROCKS_DB_CONN_H__ + +#include +#include +#include +#include +#include + +#include "singleton.h" + +#include "rocksdb/db.h" +#include "rocksdb/slice.h" +#include "rocksdb/options.h" +#include "auto_roll_logger.h" +#include "protocol.h" +using namespace rocksdb; + +class CaseSensitiveFreeComparator; +class CommKeyComparator; + +class RocksDBConn +{ +public: + typedef Iterator* RocksItr_t; + + // errcode bring back to dtc for error handling, it must match the MYSQL errno + enum ErrCode + { + ERROR_INTERNAL, + ERROR_KEY_NOT_FOUND = 1032, + ERROR_DUPLICATE_KEY = 1062, + }; + + enum ColumnFamiliesIndex + { + COLUMN_REAL_DATA = 0, // save DTC user data, use Default column family + COLUMN_META_DATA = 1, // save metadata info, such as migration state + COLUMN_DEFAULT, // NOT use, but must have it + + COLUMN_FAMILIES_NUM, + }; + +private: + std::string mRocksDbPath; + Env* mRocksEnv; + DB *mRocksDb; + // Iterator* mIterator; + std::vector mColumnHandles; // element index match the enum ColumnFamiliesIndex + + std::shared_ptr mRocksdbLogger; + + int mDtckey_type; + int mFixedKeyLen; + + // CaseSensitiveFreeComparator* mCommKeyComparator; + std::shared_ptr mPrefixExtractor; + + std::unique_ptr mTraceWriter; + + bool mHasConnected; + char achErr[400]; + int dberr; + +private: + +public: + RocksDBConn(); + ~RocksDBConn(); + + static RocksDBConn* Instance() { return Singleton::Instance(); } + + void set_key_type(int type) { mDtckey_type = type; } + int Open(const std::string &dbPath); + int Close(); + + const char* get_err_msg(); + int get_errno(); + int get_raw_errno(); + + // get value of the specific key that must full mached + int get_entry( + const std::string &key, + std::string& value, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // if key exist, return 'duplicate key' error + int insert_entry( + const std::string &key, + const std::string &value, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // if key not exist,do nothing; otherwise, update its value + int update_entry( + const std::string &key, + const std::string &value, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // if key not exist, insert it; otherwise, update its value + int replace_entry( + const std::string &key, + const std::string &value, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // append a 'delete' tombstone to indicate the key has been deleted, the tombstone can + // not been compacted unless in the bottom level compaction + int delete_entry( + const std::string &key, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // delete the exist key directly instead of append a tombstone for decreasing the number + // of tombstone those may duplicated and long time existen in the storage, that may inefficient + // the read perfomance. before use this interface, wo must guarantted the following rules: + // 1.the removed key must be written for only once + int single_delete_entry( + const std::string &key, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + /* + // use rocksdb 'DeleteRange' API to delete a range of keys among [startKey, endKey) + int deleteRangeEntry( + const std::string &startKey, + const std;:string &endKey, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA);*/ + + // not support atomic update across column families + int batch_update( + const std::set& deletedKeys, + const std::map& newEntries, + bool syncMode = false, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // API only for migration, delete the key and bring back its value + int migrate_entry( + const std::string &key, + std::string &value, + ColumnFamiliesIndex colIndex, + bool syncMode = false); + + /* the following function set are using for range query, replicateStart + * and replicateEnd must appear together */ + // start an replicateion with the given key + int retrieve_start( + std::string& startKey, + std::string& value, + RocksItr_t &itr, + bool searchMode = false, // use prefix mode to search the key, if false, use total order seek + bool forwardDirection = true, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // release the iteration snapshot + int retrieve_end(RocksItr_t itr); + + int prev_entry( + RocksItr_t itr, + std::string& key, + std::string& value); + + // for retrieving the key in the snapshot windows that allocate by 'retrieve_start' + int next_entry( + RocksItr_t itr, + std::string& key, + std::string& value); + + // search the first key in the source that at or past target( >= ). + int search_lower_bound( + std::string& targetKey, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // search the last key in the source that at or before target( <= ). + int search_upper_bound( + std::string& targetKey, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // get the first key-value in current database + int get_first_entry( + std::string &key, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + // get the last key-value in current database + int get_last_entry( + std::string &key, + std::string &value, + RocksItr_t &itr, + ColumnFamiliesIndex colIndex = COLUMN_REAL_DATA); + + uint32_t escape_string(char To[], const char* From); + +private: + int key_exist( + const std::string& key, + ColumnFamiliesIndex colIndex); + + int create_rocks_dir(const std::string& stuffDir); + + void init_db_options(Options& option); + void init_statistic_info(Options& option); + void init_prefix_extractor(ColumnFamilyOptions& option); + void init_table_options(Options& option); + void init_column_family_options(ColumnFamilyOptions& colOptions); + void init_flushing_options(Options& option); + void init_compaction_options(Options& colOptions); + void init_environment_options(Options& option); + int create_tracer(); +}; + +#endif // __ROCKS_DB_CONN_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_context.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_context.h new file mode 100644 index 0000000..d1dae51 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_context.h @@ -0,0 +1,260 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_context.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ROCKSDB_DIRECT_CONTEXT_H__ +#define __ROCKSDB_DIRECT_CONTEXT_H__ + +#if 1 + +#include +#include +#include +#include +#include +#include + +static const uint16_t sgMagicNum = 12345; // global magic number + +// operator must be matched with DTC with the same order +enum class CondOpr : uint8_t +{ + eEQ, // == 0 + eNE, // != 1 + eLT, // < 2 + eLE, // <= 3 + eGT, // > 4 + eGE // >= 5 +}; + +bool operator==(const CondOpr lc, const CondOpr rc) +{ + return (int)lc == (int)rc; +} + +#pragma pack(push, 1) +struct QueryCond +{ + uint8_t sFieldIndex; + uint8_t sCondOpr; // CondOpr + // uint8_t sCondValueLen; + // char sCondValue[]; + std::string sCondValue; + +private: + int binary_size() + { + static int fixHeaderLen = sizeof(sFieldIndex) + sizeof(sCondOpr) + sizeof(int) /* value len */; + return fixHeaderLen + sCondValue.length(); + } + + void serialize_to(char *&data) + { + *(uint8_t *)data = sFieldIndex; + data += sizeof(uint8_t); + + *(uint8_t *)data = sCondOpr; + data += sizeof(uint8_t); + + *(int *)data = sCondValue.length(); + data += sizeof(int); + + memmove((void *)data, (void *)sCondValue.c_str(), sCondValue.length()); + data += sCondValue.length(); + } + + void serialize_from(const char *data, int &condLen) + { + const char *begPos = data; + + sFieldIndex = *(uint8_t *)data; + data += sizeof(uint8_t); + + sCondOpr = *(uint8_t *)data; + data += sizeof(uint8_t); + + int len = *(int *)data; + data += sizeof(int); + + sCondValue.assign(data, len); + condLen = data - begPos + len; + } + + friend class DirectRequestContext; +}; + +struct LimitCond +{ + int sLimitStart = -1; + int sLimitStep = -1; + + void reset() { sLimitStart = -1, sLimitStep = -1; } +}; + +struct DirectRequestContext +{ + uint16_t sMagicNum; + uint64_t sSequenceId; + // uint8_t sCondValueNum; + // char sContextValue[]; + std::vector sFieldConds; + std::vector> sOrderbyFields; + LimitCond sLimitCond; + + void reset() + { + sMagicNum = 0; + sSequenceId = 0; + sFieldConds.clear(); + sLimitCond.reset(); + } + + // binary format size for transporting in + int binary_size() + { + static int fixHeaderLen = sizeof(sMagicNum) + sizeof(sSequenceId) + sizeof(uint8_t) * 2; + + int len = fixHeaderLen; + for (size_t idx = 0; idx < sFieldConds.size(); idx++) + { + len += sFieldConds[idx].binary_size(); + } + + for (size_t idx = 0; idx < sOrderbyFields.size(); idx++) + { + len += (sizeof(int) + sizeof(bool)); + } + len += sizeof(LimitCond); + + return len; + } + + // before call this function, should call 'binary_size' to evaluate the size of the struct + void serialize_to(char *data, int len) + { + *(uint16_t *)data = sMagicNum; + data += sizeof(uint16_t); + + *(uint64_t *)data = sSequenceId; + data += sizeof(uint64_t); + + *(uint8_t *)data = sFieldConds.size(); + data += sizeof(uint8_t); + for (size_t idx = 0; idx < sFieldConds.size(); idx++) + { + sFieldConds[idx].serialize_to(data); + } + + *(uint8_t *)data = sOrderbyFields.size(); + data += sizeof(uint8_t); + for (size_t idx = 0; idx < sOrderbyFields.size(); idx++) + { + *(int *)data = sOrderbyFields[idx].first; + data += sizeof(int); + + *(bool *)data = sOrderbyFields[idx].second; + data += sizeof(bool); + } + + memmove((void *)data, (void *)&sLimitCond, sizeof(LimitCond)); + data += sizeof(LimitCond); + } + + void serialize_from(const char *data, int dataLen) + { + sMagicNum = *(uint16_t *)data; + data += sizeof(uint16_t); + dataLen -= sizeof(uint16_t); + + sSequenceId = *(uint64_t *)data; + data += sizeof(uint64_t); + dataLen -= sizeof(uint64_t); + + uint8_t condNum = *(uint8_t *)data; + data += sizeof(uint8_t); + dataLen -= sizeof(uint8_t); + + QueryCond cond; + int condLen = 0; + for (uint8_t idx = 0; idx < condNum; idx++) + { + cond.serialize_from(data, condLen); + data += condLen; + dataLen -= condLen; + + sFieldConds.push_back(cond); + } + + std::pair orPair; + uint8_t orderNum = *(uint8_t *)data; + data += sizeof(uint8_t); + dataLen -= sizeof(uint8_t); + for (uint8_t idx = 0; idx < orderNum; idx++) + { + orPair.first = *(int *)data; + data += sizeof(int); + dataLen -= sizeof(int); + + orPair.second = *(bool *)data; + data += sizeof(bool); + dataLen -= sizeof(bool); + + sOrderbyFields.push_back(orPair); + } + + memmove((void *)&sLimitCond, (void *)data, sizeof(LimitCond)); + dataLen -= sizeof(LimitCond); + + assert(dataLen == 0); + } +}; + +struct DirectResponseContext +{ + uint16_t sMagicNum; + uint64_t sSequenceId; + int16_t sRowNums; // number of matched rows or errno + std::deque sRowValues; + // char* sRowValues; + + void serialize_to(std::string &data) + { + static int headerLen = sizeof(uint16_t) + sizeof(uint64_t) + sizeof(int16_t); + + data.clear(); + data = std::move(std::string((char *)this, headerLen)); + + for (size_t idx = 0; idx < sRowNums; idx++) + { + data.append(sRowValues.front()); + sRowValues.pop_front(); + } + } + + void free() + { + sMagicNum = 0; + sSequenceId = 0; + sRowNums = -1; + sRowValues.clear(); + } +}; +#pragma pack(pop) + +#endif + +#endif // __ROCKSDB_DIRECT_CONTEXT_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.cc b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.cc new file mode 100644 index 0000000..796fb56 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.cc @@ -0,0 +1,173 @@ +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_listener.cc + * + * Description: Handle detector cluster vote request + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include "rocksdb_direct_listener.h" +#include "rocksdb_direct_worker.h" +#include "poll_thread.h" +#include "log.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +RocksdbDirectListener::RocksdbDirectListener( + const std::string &name, + HelperProcessBase *processor, + PollThread *poll) + : PollerObject(poll, 0), + mDomainSocketPath(name), + mRocksdbProcess(processor), + mPollerThread(poll) +{ + // init(); +} + +RocksdbDirectListener::~RocksdbDirectListener() +{ + unlink(mDomainSocketPath.c_str()); +} + +int RocksdbDirectListener::Bind() +{ + struct sockaddr_un localAddr; + if (mDomainSocketPath.length() >= (int)sizeof(localAddr.sun_path)) + { + log_error("unix socket path is too long! path:%s", mDomainSocketPath.c_str()); + return -1; + } + + memset((void *)&localAddr, 0, sizeof(localAddr)); + localAddr.sun_family = AF_UNIX; + strncpy(localAddr.sun_path, mDomainSocketPath.c_str(), mDomainSocketPath.length()); + + netfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (netfd <= 0) + { + log_error("create unix sockst failed."); + return -1; + } + + int optval = 1; + setsockopt(netfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + setsockopt(netfd, SOL_TCP, TCP_NODELAY, &optval, sizeof(optval)); + + optval = 60; + setsockopt(netfd, SOL_TCP, TCP_DEFER_ACCEPT, &optval, sizeof(optval)); + + // bind addr, remove the maybe existen unique addr first + unlink(mDomainSocketPath.c_str()); + int ret = bind(netfd, (sockaddr *)&localAddr, sizeof(localAddr)); + if (ret < 0) + { + log_error("bind addr failed!, addr:%s, errno:%d", mDomainSocketPath.c_str(), errno); + close(netfd); + return -1; + } + + // listen for the connection + ret = listen(netfd, 256); + if (ret < 0) + { + log_error("listen to the socket failed!, addr:%s, errno:%d", mDomainSocketPath.c_str(), errno); + close(netfd); + return -1; + } + + return 0; +} + +int RocksdbDirectListener::attach_thread() +{ + enable_input(); + int ret = PollerObject::attach_poller(); + if (ret < 0) + { + log_error("add rocksdb direct listener to poll failed."); + return -1; + } + + log_info("add rocksdb direct listener to poll successful, fd:%d", netfd); + + return 0; +} + +// handle client connection +void RocksdbDirectListener::input_notify() +{ + int newFd, ret; + struct sockaddr_un peer; + socklen_t peerSize = sizeof(peer); + + // extracts all the connected connections in the pending queue until return EAGAIN + while (true) + { + newFd = accept(netfd, (struct sockaddr *)&peer, &peerSize); + if (-1 == newFd) + { + if (errno == EINTR) + { + // system call "accept" was interrupted by signal before a valid connection + // arrived, go on accept + continue; + } + + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + // no remaining connection on the pending queue, break out + // log_notice("accept new client error: %m, %d", errno); + return; + } + + // accept error + log_error("accept new client failed, netfd:%d, errno:%d", netfd, errno); + return; + } + + log_error("accept new client to rocksdb direct process, newFd:%d", newFd); + + // add the handler vote event to another poll driver + RocksdbDirectWorker *worker = new RocksdbDirectWorker(mRocksdbProcess, mPollerThread, newFd); + if (!worker) + { + log_error("create rocsdb direct workder failed!"); + continue; + } + + worker->add_event_to_poll(); + } + + return; +} + +/* +void RocksdbDirectListener::init() +{ + std::stringstream bindAddr; + const std::pair& addr = DtcMonitorConfigMgr::getInstance()->getListenAddr(); + bindAddr << addr.first << ":" << addr.second << "/tcp"; + + SocketAddress mListenAddr; + mListenAddr.set_address(bindAddr.str().c_str(), (const char*)NULL); + + return; +}*/ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.h new file mode 100644 index 0000000..17184a3 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_listener.h @@ -0,0 +1,53 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_listener.h + * + * Description: accept connection from search engine + * + * Version: 1.0 + * Created: Aug 17, 2020 + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ROCKSDB_DIRECT_LISTENER_H__ +#define __ROCKSDB_DIRECT_LISTENER_H__ + +#include "common/net_addr.h" +#include "common/poller.h" + +#include + +class HelperProcessBase; +class PollThread; + +class RocksdbDirectListener : public PollerObject +{ +private: + std::string mDomainSocketPath; + HelperProcessBase *mRocksdbProcess; + PollThread *mPollerThread; + +public: + RocksdbDirectListener( + const std::string &name, + HelperProcessBase *processor, + PollThread *poll); + + virtual ~RocksdbDirectListener(); + + int Bind(); + int attach_thread(); + virtual void input_notify(void); + +private: + // void init(); +}; + +#endif // __ROCKSDB_DIRECT_LISTENER_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.cc b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.cc new file mode 100644 index 0000000..c4e43df --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.cc @@ -0,0 +1,110 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_process.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "rocksdb_direct_process.h" +#include "rocksdb_direct_listener.h" +#include "common/poll_thread.h" +#include "log.h" + +#include + +RocksdbDirectProcess::RocksdbDirectProcess( + const std::string &name, + HelperProcessBase *processor) + : mDomainSocketPath(name), + mRocksdbProcess(processor), + mRocksDirectPoll(new PollThread("RocksdbDirectAccessPoll")), + mListener(NULL) +{ +} + +int RocksdbDirectProcess::init() +{ + assert(mRocksDirectPoll); + + int ret = mRocksDirectPoll->initialize_thread(); + if (ret < 0) + { + log_error("initialize thread poll failed."); + return -1; + } + + // add listener to poll + ret = add_listener_to_poll(); + if (ret != 0) + return -1; + + // add worker to poll + // ret = addRocksdbWorkToPoll(); + // if ( ret != 0 ) return -1; + + return 0; +} + +int RocksdbDirectProcess::add_listener_to_poll() +{ + mListener = new RocksdbDirectListener(mDomainSocketPath, mRocksdbProcess, mRocksDirectPoll); + if (!mListener) + { + log_error("create listener instance failed"); + return -1; + } + + int ret = mListener->Bind(); + if (ret < 0) + { + log_error("bind address failed."); + return -1; + } + + ret = mListener->attach_thread(); + if (ret < 0) + { + log_error("add listener to poll failed."); + return -1; + } + + return 0; +} + +/* +int RocksdbDirectProcess::addRocksdbWorkToPoll() +{ + mDirectWorker = new RocksdbDirectWorker(mRocksDirectPoll); + if ( !mDirectWorker ) + { + log_error("create rocksdb direct worker failed."); + return -1; + } + + int ret = mDirectWorker->attach_thread(); + if ( ret < 0 ) + { + log_error("add rocksdb direct worker to poll failed."); + return -1; + } + + return true; +}*/ + +int RocksdbDirectProcess::run_process() +{ + mRocksDirectPoll->running_thread(); + log_error("start rocksdb direct process!"); + return 0; +} diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.h new file mode 100644 index 0000000..988d02d --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_process.h @@ -0,0 +1,53 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_process.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ROCKSDB_DIRECT_PROCESS_H__ +#define __ROCKSDB_DIRECT_PROCESS_H__ + +#include + +class HelperProcessBase; +class RocksdbDirectListener; +class PollThread; + +class RocksdbDirectProcess +{ +private: + std::string mDomainSocketPath; + HelperProcessBase *mRocksdbProcess; + PollThread *mRocksDirectPoll; + RocksdbDirectListener *mListener; + // RocksdbDirectWorker* mDirectWorker; + +public: + RocksdbDirectProcess( + const std::string &name, + HelperProcessBase *processor); + + RocksdbDirectProcess(const RocksdbDirectProcess &) = delete; + void operator=(const RocksdbDirectProcess &) = delete; + + int init(); + int run_process(); + +private: + int add_listener_to_poll(); + // int addRocksdbWorkToPoll(); +}; + +#endif // __ROCKSDB_DIRECT_PROCESS_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.cc b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.cc new file mode 100644 index 0000000..bca1fb0 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.cc @@ -0,0 +1,258 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_worker.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include "rocksdb_direct_worker.h" +#include "db_process_base.h" +#include "poll_thread.h" +#include "log.h" + +#include + +#define __STDC_FORMAT_MACROS +#include + +RocksdbDirectWorker::RocksdbDirectWorker( + HelperProcessBase* processor, + PollThread* poll, + int fd) +: +PollerObject(poll, fd), +mDBProcessRocks(processor) +{ +} + +RocksdbDirectWorker::~RocksdbDirectWorker() +{ +} + +int RocksdbDirectWorker::add_event_to_poll() +{ + enable_input(); + int ret = PollerObject::attach_poller(); + if (ret < 0) + { + log_error("add event to poll failed."); + return -1; + } + + log_info("add rocksdb worker to event poll successful, fd:%d", netfd); + + return 0; +} + +int RocksdbDirectWorker::remove_from_event_poll() +{ + PollerObject::detach_poller(); + + log_error("delete rocksdb direct worker from poll successful, fd:%d", netfd); + + delete this; + + return 0; +} + +// because not thinking about the split or delay of data package, so if +// recieving unexpected message, just close the socket. the client will +// reconnect during the next invoking +void RocksdbDirectWorker::input_notify() +{ + int ret = recieve_request(); + if ( ret != 0 ) + { + remove_from_event_poll(); + return; + } + + proc_direct_request(); + + send_response(); + + return; +} + +void RocksdbDirectWorker::proc_direct_request() +{ + mResponseContext.free(); + + mDBProcessRocks->process_direct_query(&mRequestContext, &mResponseContext); + return; +} + +int RocksdbDirectWorker::recieve_request() +{ + static const int maxContextLen = 16 << 10; // 16k + static char dataBuffer[maxContextLen]; + + mRequestContext.reset(); + + int dataLen; + int ret = recieve_message((char*)&dataLen, sizeof(dataLen)); + if ( ret != 0 ) return -1; + assert( dataLen <= maxContextLen ); + + ret = recieve_message(dataBuffer, dataLen); + if ( ret != 0 ) return -1; + + mRequestContext.serialize_from(dataBuffer, dataLen); + + // priority of query conditions + std::sort(mRequestContext.sFieldConds.begin(), mRequestContext.sFieldConds.end(), + [this](const QueryCond& cond1, const QueryCond& cond2) ->bool { + return cond1.sFieldIndex < cond2.sFieldIndex || + (cond1.sFieldIndex == cond2.sFieldIndex && + condtion_priority((CondOpr)cond1.sCondOpr, (CondOpr)cond2.sCondOpr)); }); + assert(mRequestContext.sFieldConds[0].sFieldIndex == 0 ); + + return 0; +} + +void RocksdbDirectWorker::send_response() +{ + mResponseContext.sMagicNum = mRequestContext.sMagicNum; + mResponseContext.sSequenceId = mRequestContext.sSequenceId; + + std::string rowValue; + mResponseContext.serialize_to(rowValue); + + int valueLen = rowValue.length(); + int ret = send_message((char*)&valueLen, sizeof(int)); + if ( ret < 0 ) + { + log_error("send response failed, close the connection. netfd:%d, sequenceId:%" PRIu64, netfd, mResponseContext.sSequenceId); + remove_from_event_poll(); + return; + } + + // send row value + ret = send_message(rowValue.c_str(), valueLen); + if ( ret < 0 ) + { + log_error("send response failed, close the connection. netfd:%d, sequenceId:%" PRIu64, netfd, mResponseContext.sSequenceId); + remove_from_event_poll(); + return; + } + + log_info("send response successful. netfd:%d, sequenceId:%" PRIu64, netfd, mResponseContext.sSequenceId); + + return; +} + +int RocksdbDirectWorker::recieve_message( + char* data, + int dataLen) +{ + int readNum = 0; + int nRead = 0; + int ret = 0; + do { + ret = read(netfd, data + nRead, dataLen - nRead); + if ( ret > 0 ) + { + nRead += ret; + } + else if ( ret == 0 ) + { + // close the connection + log_error("peer close the socket, fd:%d", netfd); + return -1; + } + else + { + if ( readNum < 1000 && (errno == EAGAIN || errno == EINTR) ) + { + readNum++; + continue; + } + else + { + // close the connection + log_error("read socket failed, fd:%d, errno:%d", netfd, errno); + return -1; + } + } + } while ( nRead < dataLen ); + + return 0; +} + +int RocksdbDirectWorker::send_message( + const char* data, + int dataLen) +{ + int sendNum = 0; + int nWrite = 0; + int ret = 0; + do { + ret = write(netfd, data + nWrite, dataLen - nWrite); + if ( ret > 0 ) + { + nWrite += ret; + } + else if ( ret < 0 ) + { + if ( sendNum < 1000 && (errno == EINTR || errno == EAGAIN) ) + { + sendNum++; + continue; + } + else + { + // connection has issue, need to close the socket + log_error("write socket failed, fd:%d, errno:%d", netfd, errno); + return -1; + } + } + else + { + if ( dataLen == 0 ) return 0; + + log_error("unexpected error!!!!!!!, fd:%d, errno:%d", netfd, errno); + return -1; + } + } while ( nWrite < dataLen ); + + return 0; +} + +// judge the query condition priority those with the same field index +// (==) > (>, >=) > (<, <=) > (!=) +bool RocksdbDirectWorker::condtion_priority( + const CondOpr lc, + const CondOpr rc) +{ + switch ( lc ) + { + case CondOpr::eEQ: + return true; + case CondOpr::eNE: + return false; + case CondOpr::eLT: + case CondOpr::eLE: + if ( rc == CondOpr::eNE ) return true; + return false; + case CondOpr::eGT: + case CondOpr::eGE: + if ( rc == CondOpr::eEQ ) return false; + return true; + default: + log_error("unkonwn condtion opr:%d", (int)lc); + } + + return false; +} diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.h new file mode 100644 index 0000000..c579828 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_direct_worker.h @@ -0,0 +1,66 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_direct_worker.h + * + * Description: access rocksdb directly + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: zhuyao, zhuyao28@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __ROCKSDB_DIRECT_WORKER_H__ +#define __ROCKSDB_DIRECT_WORKER_H__ + +#include "rocksdb_direct_context.h" +#include "common/poller.h" + +class HelperProcessBase; +class PollThread; + +class RocksdbDirectWorker : public PollerObject +{ +private: + HelperProcessBase *mDBProcessRocks; + DirectRequestContext mRequestContext; + DirectResponseContext mResponseContext; + +public: + RocksdbDirectWorker( + HelperProcessBase *processor, + PollThread *poll, + int fd); + + virtual ~RocksdbDirectWorker(); + + int add_event_to_poll(); + virtual void input_notify(void); + +private: + int recieve_request(); + void proc_direct_request(); + void send_response(); + int remove_from_event_poll(); + + int recieve_message( + char *data, + int dataLen); + + int send_message( + const char *data, + int dataLen); + + bool condtion_priority( + const CondOpr lc, + const CondOpr rc); +}; + +#endif // __ROCKSDB_DIRECT_WORKER_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_key_comparator.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_key_comparator.h new file mode 100644 index 0000000..5492f0f --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_key_comparator.h @@ -0,0 +1,193 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_key_comparator.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __ROCKSDB_KEY_COMPARATOR_H__ +#define __ROCKSDB_KEY_COMPARATOR_H__ + +#include "log.h" +#include "key_format.h" +#include "rocksdb_conn.h" + +#include "rocksdb/comparator.h" +using namespace rocksdb; + +// compare the string with length of n, not the end of '\0' +extern int my_strn_case_cmp(const char *s1, const char *s2, size_t n) +{ + for (size_t idx = 0; idx < n; idx++) + { + char c1 = ::toupper(*(s1 + idx)); + char c2 = ::toupper(*(s2 + idx)); + if (c1 == c2) + continue; + if (c1 < c2) + return -1; + return 1; + } + + return 0; +} + +class CommKeyComparator +{ +private: + bool mCaseSensitiveKey; // key need to be compared with case sensitive + bool mHasVirtualField; + +public: + void set_compare_flag(bool keySensitive, bool hasVirtualField) + { + mCaseSensitiveKey = keySensitive; + mHasVirtualField = hasVirtualField; + } + + bool operator()(const std::string &lk, const std::string &rk) const + { + int ret = internal_compare(lk, rk); + return ret < 0; + } + + int internal_compare(const std::string &lk, const std::string &rk) const + { + int r; + size_t lkLen = lk.length(); + size_t rkLen = rk.length(); + if (mHasVirtualField) + { + std::string lkNextStr = lk.substr(lkLen - sizeof(uint64_t)); + std::string rkNextStr = rk.substr(rkLen - sizeof(uint64_t)); + uint64_t lkNextLen = 0, rkNextLen = 0; + key_format::DecodeBytes(lkNextStr, lkNextLen); + key_format::DecodeBytes(rkNextStr, rkNextLen); +#if 0 +std::vector fieldTypes; +fieldTypes.push_back(4); +fieldTypes.push_back(2); +fieldTypes.push_back(1); +std::vector keys1; +if (lkNextLen > 0) key_format::Decode(lk, fieldTypes, keys1); +std::vector keys2; +if (rkNextLen > 0) key_format::Decode(rk, fieldTypes, keys2); +#endif + size_t lkPrevLen = lkLen - lkNextLen - sizeof(uint64_t); + size_t rkPrevLen = rkLen - rkNextLen - sizeof(uint64_t); + size_t minLen = lkPrevLen < rkPrevLen ? lkPrevLen : rkPrevLen; + assert(minLen > 0); + if (mCaseSensitiveKey) + r = lk.compare(0, minLen, rk); + else + r = my_strn_case_cmp(lk.data(), rk.data(), minLen); + + if (r == 0) + { + if (lkPrevLen < rkPrevLen) + r = -1; + else if (lkPrevLen > rkPrevLen) + r = 1; + } + + if (r == 0) + { + assert(lkPrevLen == rkPrevLen); + + // need to compare the case sensitive free zone + minLen = lkNextLen < rkNextLen ? lkNextLen : rkNextLen; + if (!mCaseSensitiveKey) + r = memcmp(lk.data() + lkPrevLen, rk.data() + rkPrevLen, minLen); + else + r = my_strn_case_cmp(lk.data() + lkPrevLen, rk.data() + rkPrevLen, minLen); + } + } + else + { + size_t minLen = lkLen < rkLen ? lkLen : rkLen; + assert(minLen > 0); + if (mCaseSensitiveKey) + r = lk.compare(0, minLen, rk); + else + r = my_strn_case_cmp(lk.data(), rk.data(), minLen); + } + + if (r == 0) + { + if (lkLen < rkLen) + r = -1; + else if (lkLen > rkLen) + r = 1; + } + + return r; + } + + // check whether the rocksdb key, atten that, it's internal rocksdb key, not the 'key' + // in DTC user space + bool case_sensitive_rockskey() + { + return mCaseSensitiveKey == true && mHasVirtualField == false; + } +}; + +// whether ignoring the case of the characters when compare the rocksdb entire key base +// on the user definition key +extern CommKeyComparator gInternalComparator; +class CaseSensitiveFreeComparator : public rocksdb::Comparator +{ +public: + CaseSensitiveFreeComparator() {} + + const char *Name() const { return "CaseSensitiveFreeComparator"; } + + int Compare(const rocksdb::Slice &lhs, const rocksdb::Slice &rhs) const + { + assert(lhs.data_ != nullptr && rhs.data_ != nullptr); + + return gInternalComparator.internal_compare(lhs.ToString(), rhs.ToString()); + } + + void FindShortestSeparator(std::string *start, const Slice &limit) const override + { + // not implement now + return; + } + + void FindShortSuccessor(std::string *key) const override + { + // not implement now + return; + } + + // user define interface to using hash index feature + virtual bool CanKeysWithDifferentByteContentsBeEqual() const + { + // 1.rocksdb will use hashtable to index the binary seach key module that reside in the + // tail of the datablock, it use case sensitive hash function to create hash code, so + // if user want to use this feature, it must ensure that keys should be case sensitive + // 2.data block hash index only support point lookup, Range lookup request will fall back + // to BinarySeek + if (gInternalComparator.case_sensitive_rockskey()) + { + log_info("use hash index feature!!!!!!!!!"); + return false; + } + + log_info("use binary search index feature!!!!!!!!!"); + return true; + } +}; + +#endif // __ROCKSDB_KEY_COMPARATOR_H__ diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.cc b/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.cc new file mode 100644 index 0000000..448bd57 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.cc @@ -0,0 +1,53 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_orderby_unit.cc + * + * Description: +// + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#include "rocksdb_orderby_unit.h" + +RocksdbOrderByUnit::RocksdbOrderByUnit( + DTCTableDefinition *tabdef, + int rowSize, + const std::vector &fieldMap, + std::vector> &orderByFields) + : mMaxRowSize(rowSize), + mComparator(OrderByComparator(tabdef, fieldMap, orderByFields)), + mRawRows(mComparator) +{ +} + +void RocksdbOrderByUnit::add_row(const OrderByUnitElement &ele) +{ + mRawRows.push(ele); + + if (mRawRows.size() > mMaxRowSize) + mRawRows.pop(); + + return; +} + +int RocksdbOrderByUnit::get_row(OrderByUnitElement &ele) +{ + if (mRawRows.empty()) + return 0; + + ele = mRawRows.top(); + mRawRows.pop(); + + return 1; +} diff --git a/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.h b/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.h new file mode 100644 index 0000000..fee47e3 --- /dev/null +++ b/src/search_local/index_storage/rocksdb_helper/rocksdb_orderby_unit.h @@ -0,0 +1,162 @@ + +/* + * ===================================================================================== + * + * Filename: rocksdb_orderby_unit.h + * + * Description: order by using Heap sort to get the Top N elements + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +#ifndef __ROCKSDB_ORDER_BY_UNIT_H_ +#define __ROCKSDB_ORDER_BY_UNIT_H_ + +#include "table_def.h" +#include + +#include +#include +#include +#include + +#include +#include +#include + +struct OrderByUnitElement +{ + std::vector mRocksKeys; + std::vector mRocksValues; + + const std::string &getValue(int fid) const + { + if (fid < mRocksKeys.size()) + return mRocksKeys[fid]; + return mRocksValues[fid - mRocksKeys.size()]; + } +}; + +class RocksdbOrderByUnit +{ +private: + class OrderByComparator + { + private: + DTCTableDefinition *mTableDef; + std::vector mRockFieldIndexes; + std::vector> mOrderbyFields; + + public: + OrderByComparator( + DTCTableDefinition *tab, + const std::vector &fieldMap, + std::vector> &orderByFields) + : mTableDef(tab), + mRockFieldIndexes(fieldMap), + mOrderbyFields(orderByFields) + { + } + + bool operator()(const OrderByUnitElement &lhs, const OrderByUnitElement &rhs) const + { + for (size_t idx = 0; idx < mOrderbyFields.size(); idx++) + { + int fieldType = mTableDef->field_type(mOrderbyFields[idx].first); + int rocksFid = mRockFieldIndexes[mOrderbyFields[idx].first]; + const std::string &ls = lhs.getValue(rocksFid); + const std::string &rs = rhs.getValue(rocksFid); + switch (fieldType) + { + case DField::Signed: + { + int64_t li = strtoll(ls.c_str(), NULL, 10); + int64_t ri = strtoll(rs.c_str(), NULL, 10); + if (li == ri) + break; + return mOrderbyFields[idx].second /*asc*/ ? li < ri : li > ri; + } + + case DField::Unsigned: + { + uint64_t li = strtoull(ls.c_str(), NULL, 10); + uint64_t ri = strtoull(rs.c_str(), NULL, 10); + if (li == ri) + break; + return mOrderbyFields[idx].second /*asc*/ ? li < ri : li > ri; + } + + case DField::Float: + { + float li = strtod(ls.c_str(), NULL); + float ri = strtod(rs.c_str(), NULL); + if (li == ri) + break; + return mOrderbyFields[idx].second /*asc*/ ? li < ri : li > ri; + } + + case DField::String: + { + // case ignore + int llen = ls.length(); + int rlen = rs.length(); + int ret = strncasecmp(ls.c_str(), rs.c_str(), llen < rlen ? llen : rlen); + if (0 == ret && llen == rlen) + break; + return mOrderbyFields[idx].second /*asc*/ ? (ret < 0 || (ret == 0 && llen < rlen)) : (ret > 0 || (ret == 0 && llen > rlen)); + } + case DField::Binary: + { + int llen = ls.length(); + int rlen = rs.length(); + int ret = strncmp(ls.c_str(), rs.c_str(), llen < rlen ? llen : rlen); + if (0 == ret && llen == rlen) + break; + return mOrderbyFields[idx].second /*asc*/ ? (ret < 0 || (ret == 0 && llen < rlen)) : (ret > 0 || (ret == 0 && llen > rlen)); + } + + default: + log_error("unrecognized field type", mOrderbyFields[idx].first, fieldType); + return -1; + } + } + }; + }; + +private: + int mMaxRowSize; + OrderByComparator mComparator; + std::priority_queue, OrderByComparator> mRawRows; + +public: + RocksdbOrderByUnit( + DTCTableDefinition *tabdef, + int rowSize, + const std::vector &fieldMap, + std::vector> &orderByFields); + + void add_row(const OrderByUnitElement &ele); + int get_row(OrderByUnitElement &ele); + + /* + * order in heap is opposite with the 'order by' semantic, wo need to reverse it + * ig. + * order by 'field' asc + * element in heap will be: 7 5 3 1 -1, and the semantic of order by is: -1 1 3 5 7 + * */ + void reverse_rows() + { + // use deque to store rows can avoid this issuse, no need to implement + return; + } +}; + +#endif // __ROCKSDB_ORDER_BY_UNIT_H_ diff --git a/src/search_local/index_storage/stat/Makefile b/src/search_local/index_storage/stat/Makefile new file mode 100644 index 0000000..1d1ee7f --- /dev/null +++ b/src/search_local/index_storage/stat/Makefile @@ -0,0 +1,27 @@ +include ../Make.conf + +#################compile ############ +target = libstat.a stattool + +CFLAGS+=-I../common -I../../../3rdlib/attr_api +ifneq ($(findstring x86_64,$(PLATFORM)),) +BITS=64 +else +BITS=32 +endif + +filelist := stat_manager stat_dtc stat_thread stat_dtc_def stat_client stat_attn formater stat_manager_container_thread stat_alarm_reporter stat_tool +libstat_objs := $(filelist:%=%.o) +libstat_objs: CFLAGS += -pthread + + +stattool_src := stat_tool formater stat_manager stat_attn stat_client stat_dtc_def stat_alarm_reporter +stattool_objs := $(stattool_src:%=%.o) ../../../3rdlib/attr_api/libattr_api_$(BITS).a +stattool_libs := -L../common -lcommon -ldl -lpthread + +##################install ############# +target_install = stattool +install_dir = ../../bin + +include ../Make.rules + diff --git a/src/search_local/index_storage/stat/formater.cc b/src/search_local/index_storage/stat/formater.cc new file mode 100644 index 0000000..36ea433 --- /dev/null +++ b/src/search_local/index_storage/stat/formater.cc @@ -0,0 +1,130 @@ +/* + * ===================================================================================== + * + * Filename: formater.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "formater.h" + +TableFormater::TableFormater() +{ +} + +TableFormater::~TableFormater() +{ +} + +void TableFormater::new_row() +{ + table.resize(table.size() + 1); +} + +void TableFormater::clear_row() +{ + table.resize(table.size() - 1); +} + +void TableFormater::Cell(const char *format, ...) +{ + cell_t cell; + if (format && format[0]) + { + va_list ap; + cell.off = buf.size(); + va_start(ap, format); + cell.len = buf.vbprintf(format, ap); + va_end(ap); + } + + unsigned int c = table.back().size(); + if (c >= width.size()) + width.push_back(cell.len); + if (width[c] < cell.len) + width[c] = cell.len; + + table.back().push_back(cell); +} +void TableFormater::CellV(const char *format, ...) +{ + cell_t cell; + if (format && format[0]) + { + va_list ap; + cell.off = buf.size(); + va_start(ap, format); + cell.len = buf.vbprintf(format, ap); + va_end(ap); + } + + unsigned int c = table.back().size(); + if (c >= width.size()) + width.push_back(0); + + table.back().push_back(cell); +} + +void TableFormater::Dump(FILE *fp, int fmt) +{ + unsigned i, j; + char del = '\t'; + + switch (fmt) + { + case FORMAT_ALIGNED: + for (i = 0; i < table.size(); i++) + { + row_t &r = table[i]; + for (j = 0; j < r.size(); j++) + { + if (width[j] == 0) + continue; + cell_t &c = r[j]; + + if (j != 0) + fputc(' ', fp); + if (c.len < width[j]) + fprintf(fp, "%*s", width[j] - c.len, ""); + fprintf(fp, "%.*s", (int)(c.len), buf.c_str() + c.off); + } + fputc('\n', fp); + } + break; + + case FORMAT_TABBED: + del = '\t'; + goto out; + case FORMAT_COMMA: + del = ','; + goto out; + out: + for (i = 0; i < table.size(); i++) + { + row_t &r = table[i]; + for (j = 0; j < r.size(); j++) + { + cell_t &c = r[j]; + if (j != 0) + fputc(del, fp); + fprintf(fp, "%.*s", (int)(c.len), buf.c_str() + c.off); + } + for (; j < width.size(); j++) + if (j != 0) + fputc(del, fp); + fputc('\n', fp); + } + break; + } +} diff --git a/src/search_local/index_storage/stat/formater.h b/src/search_local/index_storage/stat/formater.h new file mode 100644 index 0000000..2817432 --- /dev/null +++ b/src/search_local/index_storage/stat/formater.h @@ -0,0 +1,53 @@ +/* + * ===================================================================================== + * + * Filename: formater.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "buffer.h" + +class TableFormater +{ +public: + enum + { + FORMAT_ALIGNED, + FORMAT_TABBED, + FORMAT_COMMA, + }; + TableFormater(); + ~TableFormater(); + void new_row(void); + void clear_row(void); + void Cell(const char *fmt, ...) __attribute__((__format__(printf, 2, 3))); + void CellV(const char *fmt, ...) __attribute__((__format__(printf, 2, 3))); + void Dump(FILE *, int fmt = FORMAT_ALIGNED); + +private: + struct cell_t + { + unsigned int off; + unsigned int len; + cell_t() : off(0), len(0) {} + ~cell_t() {} + }; + typedef std::vector row_t; + + class buffer buf; + std::vector table; + std::vector width; +}; diff --git a/src/search_local/index_storage/stat/stat_alarm_reporter.cc b/src/search_local/index_storage/stat/stat_alarm_reporter.cc new file mode 100644 index 0000000..a4bbd24 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_alarm_reporter.cc @@ -0,0 +1,299 @@ +/* + * ===================================================================================== + * + * Filename: stat_alarm_reporter.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include "stat_alarm_reporter.h" +#include +#include +#include +#include +#include "stat_info.h" +#include +#include +#include "dtcutils.h" +#include "log.h" +#include +void DtcStatAlarmReporter::set_time_out(int iTimeOut) +{ + m_PostTimeOut = iTimeOut; +} +/*当前运行路径的第四个位置是accesskey，access的前八个字节代表moduleid*/ +uint64_t DtcStatAlarmReporter::parse_module_id(const std::string &strCurWorkPath) +{ + + log_debug("CurWorkingPath is %s", strCurWorkPath.c_str()); + std::vector pathVec; + dtc::utils::split_str(strCurWorkPath, '/', pathVec); + if (pathVec.size() < 5) + { + log_error("error working path : %s", strCurWorkPath.c_str()); + return 0; + } + std::string strAccessKey = pathVec[4]; + std::string strModuleId = strAccessKey.substr(0, 8); + return atoi(strModuleId.c_str()); +} +bool DtcStatAlarmReporter::set_stat_client(StatClient *stc) +{ + + if (NULL == stc) + { + return false; + } + this->m_stc = stc; + return true; +} + +void DtcStatAlarmReporter::init_module_id() +{ + char buf[1024]; + getcwd(buf, sizeof(buf)); + m_ddwModuleId = parse_module_id(std::string(buf)); +} +/*优先尝试从/usr/local/dtc/ip读取本机的ip，如果该文件没有ip，则在使用GetLocalIp函数获取本机Ip*/ +void DtcStatAlarmReporter::init_local_id() +{ + std::string strIpFilePath("/usr/local/dtc/ip"); + + std::ifstream file; + file.open(strIpFilePath.c_str(), std::ios::in); + if (file.fail()) + { + log_error("open file %s fail", strIpFilePath.c_str()); + m_DtcIp = dtc::utils::get_local_ip(); + return; + } + std::string strLine; + while (std::getline(file, strLine)) + { + if (!strLine.empty()) + { + m_DtcIp = strLine; + log_debug("dtc ip is %s", m_DtcIp.c_str()); + return; + } + } + + m_DtcIp = dtc::utils::get_local_ip(); +} +bool DtcStatAlarmReporter::parse_alarm_cfg(uint64_t ddwStatId, const std::string &strCfgItem, AlarmCfg &cfg) +{ + + cfg.ddwStatItemId = ddwStatId; + std::vector itemVec; + dtc::utils::split_str(strCfgItem, ';', itemVec); + if (itemVec.size() < 3) + { + log_error("warning: bad cfg item : %s", strCfgItem.c_str()); + return false; + } + cfg.ddwThresholdValue = atol(itemVec[0].c_str()); + if ("cur" == itemVec[1]) + { + cfg.cat = SC_CUR; + } + else if ("10s" == itemVec[1]) + { + cfg.cat = SCC_10S; + } + else if ("10m" == itemVec[1]) + { + cfg.cat = SCC_10M; + } + else if ("all" == itemVec[1]) + { + cfg.cat = SCC_ALL; + } + else + { + log_error("warning: bad cat cfg : %s", itemVec[1].c_str()); + return false; + } + + cfg.strAlarmContent = itemVec[2]; + log_debug("strAlarmContent is %s", cfg.strAlarmContent.c_str()); + + if (NULL == m_stc) + { + log_error("warning: m_stc is null:"); + return false; + } + StatClient &stc = *m_stc; + StatClient::iterator si = stc[(unsigned int)ddwStatId]; + if (si == NULL) + { + log_error("warning: si is null"); + return false; + } + if (si->is_counter() || si->is_value()) + { + cfg.info = si; + } + else + { + log_error("warning: cfg[%lu] type is not value or counter", ddwStatId); + return false; + } + return true; +} +/*************************************************************** +看配置文件是否被修改了，如果被修改需要重新加载配置 +如果stat文件失败也认为需要重新加载配置 +***************************************************************/ +bool DtcStatAlarmReporter::is_alarm_cfg_file_modify(const std::string &strAlarmConfFilePath) +{ + struct stat st; + + if (stat(strAlarmConfFilePath.c_str(), &st) != 0) + { + log_error("warning: stat cfg file faile"); + return true; + } + + if (st.st_mtime == m_ModifyTime) + { + return false; + } + /*配置文件被修改了，更新文件修改时间，同时打error日志（此类操作较少）记录该文件修改事件*/ + log_error("alarm cfg file has been modified ,the last modify time is %lu, this modify time is %lu", m_ModifyTime, st.st_mtime); + m_ModifyTime = st.st_mtime; + return true; +} +/*************************************************************** +告警的配置项分为如下三种： +1、 上报的url(结束处不加?号) +url=http://192.168.214.62/api/dtc_sms_alarm_sender.php +2、短信通知的手机号码,中间以英文分号分开(结束处加分号) +cellphone=1283930303;1020123123;1212312312; +3、配置的告警项(结束处不加分号) +statItemId=thresholdValue;cat;alarmContent +其中cat分为10s 、cur、10m、all，如下以inc0的cpu占用率为例, +20000=8000;10s;inc0 thread cpu overload(80%) +这个配置项的含义就是接入线程cpu使用率的统计值（从10s文件中取值）大 +小超过了80%，发短信通知cpu超载 +所有配置的设置都采用小写英文 +***************************************************************/ +bool DtcStatAlarmReporter::init_alarm_cfg(const std::string &strAlarmConfFilePath, bool IsDtcServer) +{ + /*如果配置文件没有被修改过，就不再加载配置文件*/ + if (!is_alarm_cfg_file_modify(strAlarmConfFilePath)) + { + return true; + } + std::ifstream file; + file.open(strAlarmConfFilePath.c_str(), std::ios::in); + if (file.fail()) + { + return false; + } + + m_AlarmCfg.clear(); + std::string strLine; + while (std::getline(file, strLine)) + { + log_debug("Cfg Line is %s", strLine.c_str()); + std::vector cfgVec; + dtc::utils::split_str(strLine, '=', cfgVec); + if (cfgVec.size() < 2) + { + + continue; + } + + std::string strCfgKey = cfgVec[0]; + if ("url" == strCfgKey) + { + m_strReportURL = cfgVec[1]; + } + else if ("cellphone" == strCfgKey) + { + m_CellPhoneList = cfgVec[1]; + } + else + { + + if (IsDtcServer) + { + continue; + } + uint64_t ddwStatId = atoi(strCfgKey.c_str()); + if (ddwStatId > 0) + { + + std::string strCfgItem = cfgVec[1]; + ALARM_CONF cfg; + if (parse_alarm_cfg(ddwStatId, strCfgItem, cfg)) + { + + log_debug("push back to alarmcfg, statid %lu ", cfg.ddwStatItemId); + m_AlarmCfg.push_back(cfg); + } + } + else + { + log_error("warning: bad cfg key :%s", strCfgKey.c_str()); + } + } + } + file.close(); + return true; +} +void DtcStatAlarmReporter::do_singe_stat_item_alm_report(const ALARM_CONF &almCfg) +{ + if (NULL == m_stc) + { + + log_error("m_stc is null"); + return; + } + + uint64_t ddwStatValue = m_stc->read_counter_value(almCfg.info, almCfg.cat); + log_debug("RealStatValue is %lu, ThreSholdValue is %lu", ddwStatValue, almCfg.ddwThresholdValue); + + if (ddwStatValue < almCfg.ddwThresholdValue) + { + log_debug("RealStatValue le than ThreSholdValue "); + return; + } + log_debug("post alarm ,realstatvalue %lu", ddwStatValue); + post_alarm(almCfg.strAlarmContent); +} +void DtcStatAlarmReporter::post_alarm(const std::string &strAlarmContent) +{ + return; +} +void DtcStatAlarmReporter::report_alarm(const std::string &strAlarmContent) +{ + post_alarm(strAlarmContent); +} +void DtcStatAlarmReporter::report_alarm() +{ + static int chkpt = 0; + int curcptpoint = m_stc->check_point(); + if (curcptpoint == chkpt) + { + log_debug("curcptpoint[%d] is equal chkptr [%d]", curcptpoint, chkpt); + return; + } + chkpt = curcptpoint; + + log_debug("m_AlarmCfg size is %ld ", m_AlarmCfg.size()); + for (AlarmCfgInfo::const_iterator iter = m_AlarmCfg.begin(); iter != m_AlarmCfg.end(); iter++) + { + + do_singe_stat_item_alm_report(*iter); + } +} diff --git a/src/search_local/index_storage/stat/stat_alarm_reporter.h b/src/search_local/index_storage/stat/stat_alarm_reporter.h new file mode 100644 index 0000000..424597a --- /dev/null +++ b/src/search_local/index_storage/stat/stat_alarm_reporter.h @@ -0,0 +1,81 @@ +/* + * ===================================================================================== + * + * Filename: stat_alarm_reporter.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef DTC_STAT_ALARM_REPORTER_H +#define DTC_STAT_ALARM_REPORTER_H +#include +#include +#include "stat_client.h" +#include "singleton.h" +typedef struct AlarmCfg +{ + uint64_t ddwStatItemId; /*统计项对应的统计项id*/ + uint64_t ddwThresholdValue; /*统计项对应的告警阈值*/ + unsigned char cat; /*统计项告警时依据的数据，分别为cur，10s、10m、all，默认为10s*/ + StatClient::iterator info; /*告警项在统计文件中的位置*/ + std::string strAlarmContent; /*告警内容字符串*/ +} ALARM_CONF; +typedef std::vector AlarmCfgInfo; +typedef long File_Last_Modify_Time_T; + +class DtcStatAlarmReporter +{ +public: + DtcStatAlarmReporter() + { + m_ModifyTime = 0; + m_PostTimeOut = 1; + init_module_id(); + init_local_id(); + } + ~DtcStatAlarmReporter() + { + } + /*统计进程上报告警接口，通过配置文件的阈值和告警内容上报*/ + void report_alarm(); + /*其他进程上报告警的接口，直接上报告警内容*/ + void report_alarm(const std::string &strAlarmContent); + bool set_stat_client(StatClient *stc); + void set_time_out(int iTimeOut); + /**************************************************************************************** + 初始化配置的时候需要考虑两种场景： + 1、dtcserver，只需要url和cellphonenum，对于dtcserver来说是直接触发告警的 + 2、stattool，即需要url和cellphonenum，又需要各个统计项的阈值 告警内容等 + *****************************************************************************************/ + bool init_alarm_cfg(const std::string &strAlarmConfFilePath, bool IsDtcServer = false); + +private: + void post_alarm(const std::string &strAlarmContent); + void init_module_id(); + void init_local_id(); + bool parse_alarm_cfg(uint64_t ddwStatId, const std::string &strCfgItem, AlarmCfg &cfg); + uint64_t parse_module_id(const std::string &strCurWorkPath); + void do_singe_stat_item_alm_report(const ALARM_CONF &almCfg); + bool is_alarm_cfg_file_modify(const std::string &strAlarmConfFilePath); + +private: + std::string m_strReportURL; + AlarmCfgInfo m_AlarmCfg; + uint64_t m_ddwModuleId; /*本业务对应的ModuleId*/ + StatClient *m_stc; + std::string m_CellPhoneList; + std::string m_DtcIp; + int m_PostTimeOut; + File_Last_Modify_Time_T m_ModifyTime; +}; +#define ALARM_REPORTER Singleton::Instance() +#endif \ No newline at end of file diff --git a/src/search_local/index_storage/stat/stat_attn.cc b/src/search_local/index_storage/stat/stat_attn.cc new file mode 100644 index 0000000..53362f7 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_attn.cc @@ -0,0 +1,247 @@ +/* + * ===================================================================================== + * + * Filename: stat_attn.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../attr_api/Attr_API.h" +#include "stat_attn.h" +#include "stat_client.h" +#include "stat_alarm_reporter.h" +#ifndef LONG_MAX +#define LONG_MAX (long)2147483647 +#endif + +struct StatAttn +{ + int attnId; + StatClient::iterator info; + unsigned char hide; + unsigned char add; + unsigned char subid; + unsigned char cat; +}; + +static std::vector attnInfo; + +int ReloadConfigFile(StatClient &stc, const char *filename) +{ + /* timestamp checking */ + { + static time_t mtime = 0; + struct stat st; + + if (stat(filename, &st) != 0) + st.st_mtime = 0; + if (st.st_mtime == mtime) + return 0; + mtime = st.st_mtime; + } + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) + return -1; + + // hide all info + for (unsigned int i = 0; i < attnInfo.size(); i++) + { + if (attnInfo[i].hide) + attnInfo[i].hide = 2; + else + attnInfo[i].hide = 1; + } + + char buf[128]; + while (fgets(buf, sizeof(buf), fp) != NULL) + { + + char *p = buf; + long id = strtol(p, &p, 10); + if (id == LONG_MAX) + continue; + + while (isblank(*p)) + p++; + + int add = 0; + if (p[0] == '+') + { + add = 1; + p++; + if (*p == '=') + p++; + } + else if (p[0] == '=') + { + p++; + } + else if (p[0] == ':' && p[1] == '=') + { + p += 2; + } + + long sid = strtol(p, &p, 10); + if (sid == LONG_MAX) + continue; + + StatClient::iterator si = stc[(unsigned int)sid]; + if (si == NULL) + continue; + + int sub = 0; + if (*p == '.') + { + long lsub = strtol(p + 1, &p, 10); + if (lsub == LONG_MAX) + continue; + if (lsub >= 0x10000000) + continue; + sub = lsub; + } + if (sub < 0) + continue; + if (!si->is_sample() && sub > 0) + continue; + if (si->is_sample() && (unsigned)sub >= si->count() + 2) + continue; + + while (isblank(*p)) + p++; + int cat = SCC_10S; + if (!strncmp(p, "[cur]", 5)) + cat = SC_CUR; + if (!strncmp(p, "[10s]", 5)) + cat = SCC_10S; + if (!strncmp(p, "[10m]", 5)) + cat = SCC_10M; + if (!strncmp(p, "[all]", 5)) + cat = SCC_ALL; + + unsigned int i; + for (i = 0; i < attnInfo.size(); i++) + { + if (attnInfo[i].attnId == id) + break; + } + if (i == attnInfo.size()) + { + attnInfo.resize(i + 1); + attnInfo[i].hide = 2; + } + + if (attnInfo[i].hide == 2) + printf("NEW: %d %c %d.%d %d\n", (int)id, add ? '+' : '=', (int)sid, (int)sub, cat); + attnInfo[i].attnId = id; + attnInfo[i].info = si; + attnInfo[i].hide = 0; + attnInfo[i].subid = sub; + attnInfo[i].cat = cat; + attnInfo[i].add = add; + } + fclose(fp); + for (unsigned int i = 0; i < attnInfo.size(); i++) + { + if (attnInfo[i].hide == 1) + printf("DEL: %d %c %d.%d %d\n", + (int)attnInfo[i].attnId, + attnInfo[i].add ? '+' : '=', + (int)attnInfo[i].info->id(), + (int)attnInfo[i].subid, + attnInfo[i].cat); + } + return 0; +} + +int ReportData(StatClient &stc) +{ + static int chkpt = 0; + + int c = stc.check_point(); + if (c == chkpt) + return 0; + chkpt = c; + + for (unsigned int i = 0; i < attnInfo.size(); i++) + { + StatAttn &a = attnInfo[i]; + if (a.hide) + continue; + int v; + if (a.info->is_sample()) + { + if (a.subid == 0) + v = stc.read_sample_average(a.info, a.cat); + else + v = stc.read_sample_counter(a.info, a.cat, a.subid - 1); + } + else + { + v = stc.read_counter_value(a.info, a.cat); + } + if (a.add) + Attr_API(a.attnId, v); + else + Attr_API_Set(a.attnId, v); + } + return 0; +} + +void LockReporter(StatClient &stc) +{ + for (int i = 0; i < 60; i++) + { + if (stc.Lock("attn") == 0) + return; + if (i == 0) + fprintf(stderr, "Reporter locked, trying 1 minute.\n"); + sleep(1); + } + fprintf(stderr, "Can't lock reporter. Exiting.\n"); + exit(-1); +} + +static void term(int signo) +{ + exit(0); +} + +int run_reporter(StatClient &stc, const char *filename) +{ + int ppid = getppid(); + signal(SIGTERM, term); + signal(SIGHUP, SIG_IGN); + signal(SIGINT, SIG_IGN); + + LockReporter(stc); + for (; getppid() == ppid; sleep(10)) + { + + //ReloadConfigFile(stc, filename); + //ReportData(stc); + ALARM_REPORTER->init_alarm_cfg(std::string(filename)); + ALARM_REPORTER->report_alarm(); + } + return 0; +} diff --git a/src/search_local/index_storage/stat/stat_attn.h b/src/search_local/index_storage/stat/stat_attn.h new file mode 100644 index 0000000..14c96b5 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_attn.h @@ -0,0 +1,24 @@ +/* + * ===================================================================================== + * + * Filename: stat_attn.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __STAT_ATTN_H +#define __STAT_ATTN_H + +class StatClient; +int run_reporter(StatClient &, const char *); + +#endif diff --git a/src/search_local/index_storage/stat/stat_client.cc b/src/search_local/index_storage/stat/stat_client.cc new file mode 100644 index 0000000..5a8895a --- /dev/null +++ b/src/search_local/index_storage/stat/stat_client.cc @@ -0,0 +1,83 @@ +/* + * ===================================================================================== + * + * Filename: stat_client.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include "stat_client.h" + +StatClient::StatClient(void) +{ + lastSN = 0; +} + +StatClient::~StatClient(void) +{ +} + +int64_t StatClient::read_counter_value(iterator i, unsigned int cat) +{ + if (cat > nfmap()) + return 0; + int64_t *ptr = (int64_t *)(fmap[cat] + i->offset()); + if (cat == SC_10M || cat == SCC_10M) + return ptr[0] >> 10; + return ptr[0]; +} + +int64_t StatClient::read_sample_average(iterator i, unsigned int cat) +{ + if (cat > nfmap() || i->is_sample() == 0) + return 0; + + int64_t *ptr = (int64_t *)(fmap[cat] + i->offset()); + + return ptr[1] ? ptr[0] / ptr[1] : 0; +} + +int64_t StatClient::read_sample_counter(iterator i, unsigned int cat, unsigned int n) +{ + if (cat > nfmap() || i->is_sample() == 0 || n > i->count()) + return 0; + int64_t *ptr = (int64_t *)(fmap[cat] + i->offset()); + if (cat == SC_10M || cat == SCC_10M) + return ptr[n + 1] >> 10; + return ptr[n + 1]; +} + +int StatClient::init_stat_info(const char *name, const char *fn) +{ + int ret = StatManager::init_stat_info(name, fn, 1); + if (ret < 0) + return ret; + + mprotect(fmap[SC_CUR], header->datasize, PROT_READ); + mprotect(fmap[SC_10S], header->datasize, PROT_READ); + mprotect(fmap[SC_10M], header->datasize, PROT_READ); + mprotect(fmap[SC_ALL], header->datasize, PROT_READ); + fmap[SCC_10S] = (char *)mmap(0, header->datasize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fmap[SCC_10M] = (char *)mmap(0, header->datasize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fmap[SCC_ALL] = (char *)mmap(0, header->datasize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + return ret; +} + +int StatClient::check_point(void) +{ + memcpy(fmap[SCC_10S], fmap[SC_10S], header->datasize); + memcpy(fmap[SCC_10M], fmap[SC_10M], header->datasize); + memcpy(fmap[SCC_ALL], fmap[SC_ALL], header->datasize); + return 0; +} diff --git a/src/search_local/index_storage/stat/stat_client.h b/src/search_local/index_storage/stat/stat_client.h new file mode 100644 index 0000000..6b6c01d --- /dev/null +++ b/src/search_local/index_storage/stat/stat_client.h @@ -0,0 +1,46 @@ +/* + * ===================================================================================== + * + * Filename: stat_client.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __STAT_MGR_H +#define __STAT_MGR_H + +#include "stat_manager.h" + +class StatClient : public StatManager +{ +public: + int init_stat_info(const char *name, const char *fn); + + typedef StatInfo *iterator; + int check_point(void); + inline iterator begin(void) { return info; } + inline iterator end(void) { return info + numinfo; } + + int64_t read_counter_value(iterator, unsigned int cat); + int64_t read_sample_counter(iterator, unsigned int cat, unsigned int idx = 0); + int64_t read_sample_average(iterator, unsigned int cat); + iterator operator[](unsigned int id) { return idmap[id]; } + +public: // client/tools access + StatClient(); + ~StatClient(); + +private: + int lastSN; +}; + +#endif diff --git a/src/search_local/index_storage/stat/stat_dtc.cc b/src/search_local/index_storage/stat/stat_dtc.cc new file mode 100644 index 0000000..7e84d7b --- /dev/null +++ b/src/search_local/index_storage/stat/stat_dtc.cc @@ -0,0 +1,57 @@ +/* + * ===================================================================================== + * + * Filename: stat_dtc.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include "stat_dtc.h" +#include "version.h" +#include "log.h" + +StatThread statmgr; + +int InitStat(void) +{ + int ret; + ret = statmgr.init_stat_info("dtcd", STATIDX); + // -1, recreate, -2, failed + if (ret == -1) + { + unlink(STATIDX); + char buf[64]; + ret = statmgr.CreateStatIndex("dtcd", STATIDX, StatDefinition, buf, sizeof(buf)); + if (ret != 0) + { + log_crit("CreateStatIndex failed: %s", statmgr.error_message()); + exit(ret); + } + ret = statmgr.init_stat_info("dtcd", STATIDX); + } + if (ret == 0) + { + int v1, v2, v3; + sscanf(DTC_VERSION, "%d.%d.%d", &v1, &v2, &v3); + statmgr.get_item(S_VERSION) = v1 * 10000 + v2 * 100 + v3; + statmgr.get_item(C_TIME) = compile_time; + _init_log_stat_(); + } + else + { + log_crit("init_stat_info failed: %s", statmgr.error_message()); + exit(ret); + } + return ret; +} diff --git a/src/search_local/index_storage/stat/stat_dtc.h b/src/search_local/index_storage/stat/stat_dtc.h new file mode 100644 index 0000000..4fac2fb --- /dev/null +++ b/src/search_local/index_storage/stat/stat_dtc.h @@ -0,0 +1,29 @@ +/* + * ===================================================================================== + * + * Filename: stat_dtc.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _STAT_DTC_H__ +#define _STAT_DTC_H__ + +#include "stat_thread.h" +#include "stat_dtc_def.h" + +#define STATIDX "../stat/dtc.stat.idx" + +extern StatThread statmgr; +extern int InitStat(void); + +#endif diff --git a/src/search_local/index_storage/stat/stat_dtc_def.cc b/src/search_local/index_storage/stat/stat_dtc_def.cc new file mode 100644 index 0000000..dc46fc3 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_dtc_def.cc @@ -0,0 +1,196 @@ +/* + * ===================================================================================== + * + * Filename: stat_dtc_def.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "stat_info.h" +#include "stat_dtc_def.h" + +DTCStatDefinition StatDefinition[] = + { + // 相同的统计项目 + {S_VERSION, "server version", SA_CONST, SU_VERSION}, + {C_TIME, "compile time", SA_CONST, SU_DATETIME}, + + {LOG_COUNT_0, "logcount - emerg(0)", SA_COUNT, SU_INT}, + {LOG_COUNT_1, "logcount - alert(1)", SA_COUNT, SU_INT}, + {LOG_COUNT_2, "logcount - crit(2)", SA_COUNT, SU_INT}, + {LOG_COUNT_3, "logcount - error(3)", SA_COUNT, SU_INT}, + {LOG_COUNT_4, "logcount - warning(4)", SA_COUNT, SU_INT}, + {LOG_COUNT_5, "logcount - notice(5)", SA_COUNT, SU_INT}, + {LOG_COUNT_6, "logcount - info(6)", SA_COUNT, SU_INT}, + {LOG_COUNT_7, "logcount - debug(7)", SA_COUNT, SU_INT}, + + {REQ_USEC_ALL, "request usec - ALL", SA_SAMPLE, SU_USEC, 0, 0, {100, 200, 300, 400, 500, 600, 800, 1200, 2000, 10000, 20000, 100000, 200000, 1000000, 2000000, 10000000}}, + {REQ_USEC_GET, "request usec - select", SA_SAMPLE, SU_USEC}, + {REQ_USEC_INS, "request usec - insert", SA_SAMPLE, SU_USEC}, + {REQ_USEC_UPD, "request usec - update", SA_SAMPLE, SU_USEC}, + {REQ_USEC_DEL, "request usec - delete", SA_SAMPLE, SU_USEC}, + {REQ_USEC_FLUSH, "request usec - flush", SA_SAMPLE, SU_USEC}, + {REQ_USEC_HIT, "request usec - hit", SA_SAMPLE, SU_USEC}, + {REQ_USEC_REPLACE, "request usec - replace", SA_SAMPLE, SU_USEC}, + + {ACCEPT_COUNT, "network - accept reqs", SA_COUNT, SU_INT}, + {CONN_COUNT, "network - cur connect", SA_VALUE, SU_INT}, + + {CUR_QUEUE_COUNT, "queue - cur count", SA_VALUE, SU_INT}, + {MAX_QUEUE_COUNT, "queue - max count", SA_VALUE, SU_INT}, + {QUEUE_EFF, "queue - cur efficiency", SA_EXPR, SU_PERCENT_2, 1, 1, { + EXPR_IDV(CUR_QUEUE_COUNT, 0, 10000), + EXPR_IDV(MAX_QUEUE_COUNT, 0, 1), + }}, + {AGENT_ACCEPT_COUNT, "network - agent accept reqs", SA_COUNT, SU_INT}, + {AGENT_CONN_COUNT, "network - agent cur connect", SA_VALUE, SU_INT}, + + {SERVER_READONLY, "server - readonly", SA_CONST, SU_BOOL}, + {SERVER_OPENNING_FD, "server - openning fd", SA_CONST, SU_INT}, + {SUPER_GROUP_ENABLE, "server - super_group enable", SA_CONST, SU_BOOL}, + + /************************** dtc ***********************/ + {DTC_CACHE_SIZE, "cache - mem size", SA_CONST, SU_INT}, + {DTC_CACHE_KEY, "cache - shm key", SA_CONST, SU_INT}, + {DTC_CACHE_VERSION, "cache - version", SA_CONST, SU_VERSION}, + {DTC_UPDATE_MODE, "cache - sync mode", SA_CONST, SU_BOOL}, + {DTC_EMPTY_FILTER, "cache - empty filter", SA_CONST, SU_BOOL}, + + {DTC_USED_NGS, "cache - total NGs", SA_VALUE, SU_INT}, + {DTC_USED_NODES, "cache - total nodes", SA_VALUE, SU_INT}, + {DTC_DIRTY_NODES, "cache - dirty nodes", SA_VALUE, SU_INT}, + {DTC_USED_ROWS, "cache - total rows", SA_VALUE, SU_INT}, + {DTC_DIRTY_ROWS, "cache - dirty rows", SA_VALUE, SU_INT}, + {DTC_BUCKET_TOTAL, "cache - total bucket", SA_CONST, SU_INT}, + {DTC_FREE_BUCKET, "cache - free bucket", SA_VALUE, SU_INT}, + {DTC_DIRTY_AGE, "cache - dirty age", SA_VALUE, SU_TIME}, + {DTC_DIRTY_ELDEST, "cache - dirty eldest", SA_CONST, SU_DATETIME}, // no really const + + {DTC_CHUNK_TOTAL, "cache - chunk total", SA_VALUE, SU_INT}, + {DTC_DATA_SIZE, "cache - data bytes", SA_VALUE, SU_INT}, + {DTC_DATA_EFF, "cache - data efficiency", SA_EXPR, SU_PERCENT_2, 1, 1, {EXPR_IDV(DTC_DATA_SIZE, 0, 10000), EXPR_IDV(DTC_CACHE_SIZE, 0, 1)}}, + {DTC_DROP_COUNT, "cache - drop nodes", SA_COUNT, SU_INT}, + {DTC_DROP_ROWS, "cache - drop rows", SA_COUNT, SU_INT}, + {DTC_FLUSH_COUNT, "cache - flush nodes", SA_COUNT, SU_INT}, + {DTC_FLUSH_ROWS, "cache - flush rows", SA_COUNT, SU_INT}, + {DTC_GET_COUNT, "cache - select reqs", SA_COUNT, SU_INT}, + {DTC_GET_HITS, "cache - select hits", SA_COUNT, SU_INT}, + {DTC_INSERT_COUNT, "cache - insert reqs", SA_COUNT, SU_INT}, + {DTC_INSERT_HITS, "cache - insert hits", SA_COUNT, SU_INT}, + {DTC_UPDATE_COUNT, "cache - update reqs", SA_COUNT, SU_INT}, + {DTC_UPDATE_HITS, "cache - update hits", SA_COUNT, SU_INT}, + {DTC_DELETE_COUNT, "cache - delete reqs", SA_COUNT, SU_INT}, + {DTC_DELETE_HITS, "cache - delete hits", SA_COUNT, SU_INT}, + {DTC_PURGE_COUNT, "cache - purge reqs", SA_COUNT, SU_INT}, + + {DTC_HIT_RATIO, "cache - select hit ratio", SA_EXPR, SU_PERCENT_2, 1, 1, { + EXPR_IDV(DTC_GET_HITS, 0, 10000), + EXPR_IDV(DTC_GET_COUNT, 0, 1), + }}, + {DTC_ASYNC_RATIO, "cache - async hit ratio", SA_EXPR, SU_PERCENT_2, 5, 3, { + EXPR_IDV(DTC_INSERT_HITS, 0, 10000), + EXPR_IDV(DTC_UPDATE_HITS, 0, 10000), + EXPR_IDV(DTC_DELETE_HITS, 0, 10000), + EXPR_IDV(DTC_FLUSH_ROWS, 0, -10000), + EXPR_IDV(DTC_DROP_ROWS, 0, -10000), + EXPR_IDV(DTC_INSERT_COUNT, 0, 1), + EXPR_IDV(DTC_UPDATE_COUNT, 0, 1), + EXPR_IDV(DTC_DELETE_COUNT, 0, 1), + }}, + + {DTC_SQL_USEC_ALL, "helper usec - ALL", SA_SAMPLE, SU_USEC, 0, 0, {100, 200, 300, 400, 500, 600, 800, 1200, 2000, 10000, 20000, 100000, 200000, 1000000, 2000000, 10000000}}, + {DTC_SQL_USEC_GET, "helper usec - select", SA_SAMPLE, SU_USEC}, + {DTC_SQL_USEC_INS, "helper usec - insert", SA_SAMPLE, SU_USEC}, + {DTC_SQL_USEC_UPD, "helper usec - update", SA_SAMPLE, SU_USEC}, + {DTC_SQL_USEC_DEL, "helper usec - delete", SA_SAMPLE, SU_USEC}, + {DTC_SQL_USEC_FLUSH, "helper usec - flush", SA_SAMPLE, SU_USEC}, + + {DTC_FORWARD_USEC_ALL, "forward usec - ALL", SA_SAMPLE, SU_USEC, 0, 0, {100, 200, 300, 400, 500, 600, 800, 1200, 2000, 10000, 20000, 100000, 200000, 1000000, 2000000, 10000000}}, + {DTC_FORWARD_USEC_GET, "forward usec - select", SA_SAMPLE, SU_USEC}, + {DTC_FORWARD_USEC_INS, "forward usec - insert", SA_SAMPLE, SU_USEC}, + {DTC_FORWARD_USEC_UPD, "forward usec - update", SA_SAMPLE, SU_USEC}, + {DTC_FORWARD_USEC_DEL, "forward usec - delete", SA_SAMPLE, SU_USEC}, + {DTC_FORWARD_USEC_FLUSH, "forward usec - flush", SA_SAMPLE, SU_USEC}, + + {DTC_EMPTY_NODES, "cache - empty nodes", SA_VALUE, SU_INT}, + {DTC_MEMORY_TOP, "cache - memory topline", SA_VALUE, SU_INT}, + {DTC_MAX_FLUSH_REQ, "cache - max flush request", SA_VALUE, SU_INT}, + {DTC_CURR_FLUSH_REQ, "cache - curr flush request", SA_VALUE, SU_INT}, + {DTC_OLDEST_DIRTY_TIME, "cache - oldest dirty node time", SA_VALUE, SU_INT}, + + /***********************purge alerting************************/ + {LAST_PURGE_NODE_MOD_TIME, "cache - last purge data modify", SA_CONST, SU_DATETIME}, + {DATA_EXIST_TIME, "cache - data exist time", SA_CONST, SU_INT}, + {DATA_SIZE_AVG_RECENT, "cache - average alloc(recently)", SA_VALUE, SU_INT}, + {DTC_ASYNC_FLUSH_COUNT, "cache - flush db", SA_COUNT, SU_INT}, + + /***************** key expire time **************************/ + {DTC_KEY_EXPIRE_USER_COUNT, "cache - user key expire count", SA_COUNT, SU_INT}, + {DTC_KEY_EXPIRE_DTC_COUNT, "cache - dtc key expire count", SA_COUNT, SU_INT}, + + /************************** bitmapsvr ***********************/ + {BTM_INDEX_1, "Mem - index(1)", SA_COUNT, SU_INT}, + {BTM_INDEX_2, "Mem - index(2)", SA_COUNT, SU_INT}, + {BTM_INDEX_3, "Mem - index(3)", SA_COUNT, SU_INT}, + {BTM_DATA, "Mem - data", SA_COUNT, SU_INT}, + {BTM_DATA_DELETE, "Mem - delete", SA_COUNT, SU_INT}, + + {HBP_LRU_SCAN_TM, "hbp - lru scan time(msec)", SA_SAMPLE, SU_MSEC}, + {HBP_LRU_TOTAL_BITS, "hbp - lru total bits", SA_VALUE, SU_INT}, + {HBP_LRU_TOTAL_1_BITS, "hbp - lru total 1 bits", SA_VALUE, SU_INT}, + {HBP_LRU_SET_COUNT, "hbp - lru set op count", SA_COUNT, SU_INT}, + {HBP_LRU_SET_HIT_COUNT, "hbp - lru set hit count", SA_COUNT, SU_INT}, + {HBP_LRU_CLR_COUNT, "hbp - lru clr op count", SA_COUNT, SU_INT}, + + // {HBP_INC_SYNC_STEP, "hbp - inc-sync step", SA_SAMPLE, SU_INT, 0, 0, + // { 1, 2, 5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000} }, + + {BLACKLIST_CURRENT_SLOT, "blacklist - current count", SA_VALUE, SU_INT}, + {BLACKLIST_SIZE, "blacklist - size distribution", SA_SAMPLE, SU_INT, 0, 0, {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144}}, + + /* try_purge_size 每次purge的节点个数 */ + {TRY_PURGE_COUNT, "try purge - count distribution", SA_SAMPLE, SU_INT, 0, 0, {10, 20, 40, 80, 120, 200, 400, 800, 1000, 2000, 4000, 8000}}, + {TRY_PURGE_NODES, "try purge - auto purged nodes", SA_COUNT, SU_INT, 0, 0}, + {PLUGIN_REQ_USEC_ALL, "request sb usec - ALL", SA_SAMPLE, SU_USEC, 0, 0, {100, 200, 300, 400, 500, 600, 800, 1200, 2000, 10000, 20000, 100000, 200000, 1000000, 2000000, 10000000}}, + + {INC_THREAD_CPU_STAT_0, "incomming thread 0 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_1, "incomming thread 1 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_2, "incomming thread 2 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_3, "incomming thread 3 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_4, "incomming thread 4 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_5, "incomming thread 5 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_6, "incomming thread 6 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_7, "incomming thread 7 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_8, "incomming thread 8 cpu", SA_VALUE, SU_PERCENT_2}, + {INC_THREAD_CPU_STAT_9, "incomming thread 9 cpu", SA_VALUE, SU_PERCENT_2}, + {CACHE_THREAD_CPU_STAT, "cache thread cpu", SA_VALUE, SU_PERCENT_2}, + {DATA_SOURCE_CPU_STAT, "data source thread cpu", SA_VALUE, SU_PERCENT_2}, + {WORKER_THREAD_CPU_STAT, "worker thread cpu", SA_VALUE, SU_PERCENT_2}, + {DTC_FRONT_BARRIER_COUNT, "front barrier number", SA_VALUE, SU_INT}, + {DTC_FRONT_BARRIER_MAX_TASK, "front barrier max task number", SA_VALUE, SU_INT}, + {DTC_BACK_BARRIER_COUNT, "end barrier number", SA_VALUE, SU_INT}, + {DTC_BACK_BARRIER_MAX_TASK, "end barrier max task number", SA_VALUE, SU_INT}, + {DATA_SIZE_HISTORY_STAT, "history data - size distribution", SA_SAMPLE, SU_INT, 0, 0, {64, 128, 256, 512, 1024, 2048, 4096}}, + {ROW_SIZE_HISTORY_STAT, "history row - size distribution", SA_SAMPLE, SU_INT, 0, 0, {1, 2, 3, 4, 8, 16, 32}}, + {DATA_SURVIVAL_HOUR_STAT, "data survival time(By hour)", SA_SAMPLE, SU_INT, 0, 0, {1, 2, 4, 8, 16, 24, 36, 48, 72, 96, 192, 360, 720, 1440}}, + {PURGE_CREATE_UPDATE_STAT, "purge size for create and update distribution", SA_SAMPLE, SU_INT, 0, 0, {100, 200, 400, 800, 1200, 1600, 2000, 2500}}, + {HELPER_READ_GROUR_CUR_QUEUE_MAX_SIZE, "read group queue max count", SA_COUNT, SU_INT}, + {HELPER_WRITE_GROUR_CUR_QUEUE_MAX_SIZE, "write group queue max count", SA_COUNT, SU_INT}, + {HELPER_COMMIT_GROUR_CUR_QUEUE_MAX_SIZE, "commit grour queue max count", SA_COUNT, SU_INT}, + {HELPER_SLAVE_READ_GROUR_CUR_QUEUE_MAX_SIZE, "slave read group queue max count", SA_COUNT, SU_INT}, + {INCOMING_EXPIRE_REQ, "incoming -expire req(send rsp)", SA_COUNT, SU_INT}, + {CACHE_EXPIRE_REQ, "cache -expire req ", SA_COUNT, SU_INT}, + {DATA_SOURCE_EXPIRE_REQ, "data source -expire req ", SA_COUNT, SU_INT}, + {TASK_CLIENT_TIMEOUT, "task client timeout", SA_VALUE, SU_INT}, + {0}}; diff --git a/src/search_local/index_storage/stat/stat_dtc_def.h b/src/search_local/index_storage/stat/stat_dtc_def.h new file mode 100644 index 0000000..ccfca0b --- /dev/null +++ b/src/search_local/index_storage/stat/stat_dtc_def.h @@ -0,0 +1,191 @@ +/* + * ===================================================================================== + * + * Filename: stat_dtc_def.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __STAT_DTC_DEFINITION__ +#define __STAT_DTC_DEFINITION__ + +#include "stat_info.h" + +enum +{ + //相同的统计项目 + S_VERSION = 5, + C_TIME, + + LOG_COUNT_0 = 10, + LOG_COUNT_1, + LOG_COUNT_2, + LOG_COUNT_3, + LOG_COUNT_4, + LOG_COUNT_5, + LOG_COUNT_6, + LOG_COUNT_7, + + REQ_USEC_ALL = 20, + REQ_USEC_GET, + REQ_USEC_INS, + REQ_USEC_UPD, + REQ_USEC_DEL, + REQ_USEC_FLUSH, + REQ_USEC_HIT, + REQ_USEC_REPLACE, + + ACCEPT_COUNT = 30, // accept连接次数 + CONN_COUNT, // 当前连接数 + + CUR_QUEUE_COUNT, + MAX_QUEUE_COUNT, + QUEUE_EFF, + + AGENT_ACCEPT_COUNT, + AGENT_CONN_COUNT, + + SERVER_READONLY = 40, //server是否为只读状态 + SERVER_OPENNING_FD, + SUPER_GROUP_ENABLE, + + //DTC + DTC_CACHE_SIZE = 1000, + DTC_CACHE_KEY, + DTC_CACHE_VERSION, + DTC_UPDATE_MODE, + DTC_EMPTY_FILTER, + + DTC_USED_NGS, + DTC_USED_NODES, + DTC_DIRTY_NODES, + DTC_USED_ROWS, + DTC_DIRTY_ROWS, + DTC_BUCKET_TOTAL, + DTC_FREE_BUCKET, + DTC_DIRTY_AGE, + DTC_DIRTY_ELDEST, + + DTC_CHUNK_TOTAL, + DTC_DATA_SIZE, + DTC_DATA_EFF, + + DTC_DROP_COUNT, + DTC_DROP_ROWS, + DTC_FLUSH_COUNT, + DTC_FLUSH_ROWS, + DTC_GET_COUNT, + DTC_GET_HITS, + DTC_INSERT_COUNT, + DTC_INSERT_HITS, + DTC_UPDATE_COUNT, + DTC_UPDATE_HITS, + DTC_DELETE_COUNT, + DTC_DELETE_HITS, + DTC_PURGE_COUNT, + DTC_HIT_RATIO, + DTC_ASYNC_RATIO, + + DTC_SQL_USEC_ALL, + DTC_SQL_USEC_GET, + DTC_SQL_USEC_INS, + DTC_SQL_USEC_UPD, + DTC_SQL_USEC_DEL, + DTC_SQL_USEC_FLUSH, + + DTC_FORWARD_USEC_ALL, + DTC_FORWARD_USEC_GET, + DTC_FORWARD_USEC_INS, + DTC_FORWARD_USEC_UPD, + DTC_FORWARD_USEC_DEL, + DTC_FORWARD_USEC_FLUSH, + + DTC_FRONT_BARRIER_COUNT, + DTC_FRONT_BARRIER_MAX_TASK, + DTC_BACK_BARRIER_COUNT, + DTC_BACK_BARRIER_MAX_TASK, + + DTC_EMPTY_NODES, // empty node count + DTC_MEMORY_TOP, + DTC_MAX_FLUSH_REQ, + DTC_CURR_FLUSH_REQ, + DTC_OLDEST_DIRTY_TIME, + LAST_PURGE_NODE_MOD_TIME, + DATA_EXIST_TIME, + DATA_SIZE_AVG_RECENT, + DTC_ASYNC_FLUSH_COUNT, + + DTC_KEY_EXPIRE_USER_COUNT, + DTC_KEY_EXPIRE_DTC_COUNT, + + BTM_INDEX_1 = 2000, + BTM_INDEX_2, + BTM_INDEX_3, + BTM_DATA, + BTM_DATA_DELETE, + + /* statisitc item for hotbackup */ + HBP_LRU_SCAN_TM = 3000, + HBP_LRU_TOTAL_BITS, + HBP_LRU_TOTAL_1_BITS, + HBP_LRU_SET_COUNT, + HBP_LRU_SET_HIT_COUNT, + HBP_LRU_CLR_COUNT, + HBP_INC_SYNC_STEP, + + /* statistic item for blacklist */ + BLACKLIST_CURRENT_SLOT = 3010, + BLACKLIST_SIZE, + TRY_PURGE_COUNT, /* try_purge_size 每次purge的节点个数 */ + TRY_PURGE_NODES, + + PLUGIN_REQ_USEC_ALL = 10000, + + /*thread cpu statistic*/ + INC_THREAD_CPU_STAT_0 = 20000, + INC_THREAD_CPU_STAT_1, + INC_THREAD_CPU_STAT_2, + INC_THREAD_CPU_STAT_3, + INC_THREAD_CPU_STAT_4, + INC_THREAD_CPU_STAT_5, + INC_THREAD_CPU_STAT_6, + INC_THREAD_CPU_STAT_7, + INC_THREAD_CPU_STAT_8, + INC_THREAD_CPU_STAT_9, + + CACHE_THREAD_CPU_STAT = 20100, + DATA_SOURCE_CPU_STAT = 20200, + + DATA_SIZE_HISTORY_STAT = 20300, + ROW_SIZE_HISTORY_STAT = 20301, + DATA_SURVIVAL_HOUR_STAT = 20302, + PURGE_CREATE_UPDATE_STAT = 20303, + /*新增Helper的统计项, 从20400开始编号*/ + HELPER_READ_GROUR_CUR_QUEUE_MAX_SIZE = 20400, + HELPER_WRITE_GROUR_CUR_QUEUE_MAX_SIZE = 20401, + HELPER_COMMIT_GROUR_CUR_QUEUE_MAX_SIZE = 20402, + HELPER_SLAVE_READ_GROUR_CUR_QUEUE_MAX_SIZE = 20403, + + /*single thread*/ + WORKER_THREAD_CPU_STAT = 20500, + + /*expire time*/ + INCOMING_EXPIRE_REQ = 30000, + CACHE_EXPIRE_REQ = 30001, + DATA_SOURCE_EXPIRE_REQ = 30002, + + /* task expire time set by client */ + TASK_CLIENT_TIMEOUT = 30100, +}; + +extern DTCStatDefinition StatDefinition[]; +#endif diff --git a/src/search_local/index_storage/stat/stat_info.h b/src/search_local/index_storage/stat/stat_info.h new file mode 100644 index 0000000..b72ed57 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_info.h @@ -0,0 +1,136 @@ +/* + * ===================================================================================== + * + * Filename: stat_info.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_STAT_INFO_INFO_H_ +#define __H_STAT_INFO_INFO_H_ + +#include +#include + +enum +{ + SA_VALUE = 0, + SA_COUNT, + SA_SAMPLE, + SA_CONST, + SA_EXPR, +}; + +enum +{ + SC_CUR = 0, + SC_10S, + SC_10M, + SC_ALL, + SCC_10S, + SCC_10M, + SCC_ALL, +}; + +enum +{ + SU_HIDE = 0, + SU_INT, + SU_INT_1, + SU_INT_2, + SU_INT_3, + SU_INT_4, + SU_INT_5, + SU_INT_6, + SU_DATETIME, + SU_VERSION, + SU_DATE, + SU_TIME, + SU_MSEC, + SU_USEC, + SU_PERCENT, + SU_PERCENT_1, + SU_PERCENT_2, + SU_PERCENT_3, + SU_BOOL, +}; + +#define STAT_CREATE_TIME 1 +#define STAT_STARTUP_TIME 2 +#define STAT_CHECKPOINT_TIME 3 + +#define MIN_STAT_ID 10 +#define MAX_STAT_ID 100000000 + +#define EXPR_NUM(x) (((int64_t)(x)) << 32) +#define EXPR_ID_(x, y) ((x)*20 + (y)) +#define EXPR_IDV(x, y, z) (EXPR_ID_(x, y) + EXPR_NUM(z)) +#define EXPR_ID2(x, y, x1, y1) (EXPR_ID_(x, y) + 0x80000000 + EXPR_NUM(EXPR_ID_(x1, y1))) + +struct DTCStatInfo +{ + unsigned int id; // stat id + unsigned int off; // offset from data file + unsigned char type; // stat attr + unsigned char unit; // item count + unsigned char cnt; // item count + unsigned char cnt1; // item count + unsigned int resv; // item count + char name[32]; + int64_t vptr[0]; + + int is_sample(void) const { return type == SA_SAMPLE; } + int is_counter(void) const { return type == SA_COUNT; } + int is_value(void) const { return type == SA_VALUE; } + int is_const(void) const { return type == SA_CONST; } + int is_expr(void) const { return type == SA_EXPR; } + int data_size(void) const + { + return is_sample() ? sizeof(int64_t) * (16 + 2) : sizeof(int64_t); + } + DTCStatInfo *next(void) const + { + return (DTCStatInfo *)((char *)this + + offsetof(DTCStatInfo, vptr) + + (is_sample() ? 16 * sizeof(int64_t) : 0) + + (is_expr() ? (cnt + cnt1) * sizeof(int64_t) : 0)); + } +}; + +struct DTCStatHeader +{ + unsigned int signature; + unsigned char version; + unsigned char zero[3]; + unsigned int numinfo; + unsigned int indexsize; + unsigned int datasize; + unsigned int createtime; + unsigned int reserved[2]; + char name[64]; + + DTCStatInfo *first(void) const { return (DTCStatInfo *)(this + 1); } + DTCStatInfo *last(void) const { return (DTCStatInfo *)((char *)this + indexsize); } +}; + +struct DTCStatDefinition +{ + unsigned int id; + const char *name; + unsigned char type; + unsigned char unit; + unsigned char cnt; + unsigned char cnt1; + int64_t arg[16]; +}; + +#endif diff --git a/src/search_local/index_storage/stat/stat_manager.cc b/src/search_local/index_storage/stat/stat_manager.cc new file mode 100644 index 0000000..6387a32 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_manager.cc @@ -0,0 +1,883 @@ +/* + * ===================================================================================== + * + * Filename: stat_manager.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "system_lock.h" +#include "stat_manager.h" + +#if HAS_ATOMIC8 +int64_t StatItem::dummy; +#else +uint32_t StatItemU32::dummy; +int32_t StatItemS32::dummy; +StatItemObject StatItem::dummy; +#endif +int64_t StatSampleObject::dummyCnt[2]; +const DTCStatInfo StatSampleObject::dummyInfo = {0, 0, SA_SAMPLE, 0}; +StatSampleObject StatSample::dummy; + +int64_t StatSampleObject::count(unsigned int n) +{ + P a(this); + if (n > info->cnt) + return 0; + return cnt[n]; +} +int64_t StatSampleObject::sum(void) +{ + P a(this); + return cnt[0]; +} +int64_t StatSampleObject::average(int64_t o) +{ + P a(this); + return cnt[1] ? cnt[0] / cnt[1] : o; +} +void StatSampleObject::push(int64_t v) +{ + P a(this); + cnt[0] += v; + cnt[1]++; + for (unsigned int n = 0; n < info->cnt; n++) + if (v >= info->vptr[n]) + cnt[2 + n]++; +} +void StatSampleObject::output(int64_t *v) +{ + P a(this); + memcpy(v, cnt, (2 + 16) * sizeof(int64_t)); + memset(cnt, 0, (2 + 16) * sizeof(int64_t)); +} + +StatManager::StatManager() +{ + header = NULL; + indexsize = 0; + memset(fmap, 0, sizeof(fmap)); + info = NULL; + numinfo = 0; + lockdev = 0; + lockino = 0; + lockfd = -1; +} + +StatManager::~StatManager() +{ + if (header) + { + for (unsigned int i = 0; i < nfmap(); i++) + if (fmap[i]) + munmap(fmap[i], header->datasize); + munmap(header, indexsize); + } + if (info) + delete[] info; + if (lockfd >= 0) + close(lockfd); +} + +int StatManager::init_stat_info(const char *name, const char *indexfile, int isc) +{ + P __a(this); + + int fd; + + fd = open(indexfile, O_RDWR); + if (fd < 0) + { + snprintf(szErrMsg, sizeof(szErrMsg), "cannot open index file"); + return -1; + } + + struct stat st; + if (fstat(fd, &st) != 0) + { + snprintf(szErrMsg, sizeof(szErrMsg), "fstat() failed on index file"); + close(fd); + return -2; + } + lockdev = st.st_dev; + lockino = st.st_ino; + + if (isc == 0) + { + if (Lock("serv") < 0) + { + snprintf(szErrMsg, sizeof(szErrMsg), "stat data locked by other process"); + close(fd); + return -2; + } + } + + indexsize = lseek(fd, 0L, SEEK_END); + if (indexsize < sizeof(DTCStatHeader) + sizeof(DTCStatInfo)) + { + // file too short + close(fd); + snprintf(szErrMsg, sizeof(szErrMsg), "index file too short"); + return -1; + } + + header = (DTCStatHeader *)mmap(NULL, indexsize, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + + if (header == MAP_FAILED) + { + header = NULL; + snprintf(szErrMsg, sizeof(szErrMsg), "mmap index file failed"); + return -2; + } + + if (header->signature != *(unsigned int *)"sTaT") + { + snprintf(szErrMsg, sizeof(szErrMsg), "bad index file signature"); + return -1; + } + if (header->version != 1) + { + snprintf(szErrMsg, sizeof(szErrMsg), "bad index file version"); + return -1; + } + if (indexsize < header->indexsize) + { + // file too short + snprintf(szErrMsg, sizeof(szErrMsg), "index file too short"); + return -1; + } + if (header->indexsize > (4 << 20)) + { // data too large + snprintf(szErrMsg, sizeof(szErrMsg), "index size too large"); + return -1; + } + if (header->datasize > (1 << 20)) + { // data too large + snprintf(szErrMsg, sizeof(szErrMsg), "data size too large"); + return -1; + } + if (strncmp(name, header->name, sizeof(header->name)) != 0) + { + // name mismatch + snprintf(szErrMsg, sizeof(szErrMsg), "stat name mismatch"); + return -1; + } + + if (header->numinfo == 0) + { + snprintf(szErrMsg, sizeof(szErrMsg), "No Stat ID defined"); + return -1; + } + + numinfo = header->numinfo; + info = new StatInfo[numinfo]; + DTCStatInfo *si = header->first(); + for (unsigned int i = 0; i < numinfo; i++, si = si->next()) + { + if (si->next() > header->last()) + { + snprintf(szErrMsg, sizeof(szErrMsg), "index info exceed EOF"); + return -1; + } + + if (si->off + si->data_size() > header->datasize) + { + snprintf(szErrMsg, sizeof(szErrMsg), "data offset exceed EOF"); + return -1; + } + + // first 16 bytes reserved by header + if (si->off < 16) + { + snprintf(szErrMsg, sizeof(szErrMsg), "data offset < 16"); + return -1; + } + + if (si->cnt + si->cnt1 > 16) + { + snprintf(szErrMsg, sizeof(szErrMsg), "too many base value"); + return -1; + } + + info[i].owner = this; + info[i].si = si; + + idmap[si->id] = &info[i]; + } + + mprotect(header, indexsize, PROT_READ); + + char buf[strlen(indexfile) + 10]; + strncpy(buf, indexfile, strlen(indexfile) + 10); + char *p = strrchr(buf, '.'); + if (p == NULL) + p = buf + strlen(buf); + + fmap[SC_CUR] = (char *)mapfile(NULL, header->datasize); + strncpy(p, ".10s", 5); + fmap[SC_10S] = (char *)mapfile(buf, header->datasize); + strncpy(p, ".10m", 5); + fmap[SC_10M] = (char *)mapfile(buf, header->datasize); + strncpy(p, ".all", 5); + fmap[SC_ALL] = (char *)mapfile(buf, header->datasize); + + at_cur(2 * 8) = header->createtime; + if (isc == 0) + { + int n; + do + { + n = 0; + for (unsigned int i = 0; i < numinfo; i++) + { + if (!info[i].is_expr()) + continue; + if (info[i].expr != NULL) + continue; + unsigned int cnt = info[i].si->cnt + info[i].si->cnt1; + if (cnt == 0) + continue; + info[i].expr = init_expr(cnt, info[i].si->vptr); + if (info[i].expr) + { + expr.push_back(&info[i]); + n++; + } + } + } while (n > 0); + at_cur(3 * 8) = time(NULL); // startup time + atomic_t *a0 = (atomic_t *)&at_cur(0); + int v = atomic_add_return(1, a0); + atomic_set(a0 + 1, v); + } + return 0; +} + +static const DTCStatDefinition SysStatDefinition[] = + { + {STAT_CREATE_TIME, "statinfo create time", SA_CONST, SU_DATETIME}, + {STAT_STARTUP_TIME, "startup time", SA_CONST, SU_DATETIME}, + {STAT_CHECKPOINT_TIME, "checkpoint time", SA_CONST, SU_DATETIME}, +}; +#define NSYSID (sizeof(SysStatDefinition) / sizeof(DTCStatDefinition)) + +int StatManager::CreateStatIndex( + const char *name, + const char *indexfile, + const DTCStatDefinition *statdef, + char *szErrMsg, int iErrLen) +{ + if (access(indexfile, F_OK) == 0) + { + snprintf(szErrMsg, iErrLen, "index file already exists"); + return -1; + } + + int numinfo = NSYSID; + int indexsize = sizeof(DTCStatHeader) + sizeof(DTCStatInfo) * NSYSID; + int datasize = 16 + sizeof(int64_t) * NSYSID; + + unsigned int i; + for (i = 0; statdef[i].id; i++) + { + if (statdef[i].cnt + statdef[i].cnt1 > 16) + { + snprintf(szErrMsg, iErrLen, "too many argument counts"); + return -1; + } + numinfo++; + indexsize += sizeof(DTCStatInfo); + switch (statdef[i].type) + { + case SA_SAMPLE: + indexsize += 16 * sizeof(int64_t); + datasize += 18 * sizeof(int64_t); + break; + case SA_EXPR: + indexsize += (statdef[i].cnt + statdef[i].cnt1) * sizeof(int64_t); + default: + datasize += sizeof(int64_t); + } + } + + if (indexsize > (4 << 20)) + { + snprintf(szErrMsg, iErrLen, "index file size too large"); + return -1; + } + if (datasize > (1 << 20)) + { + snprintf(szErrMsg, iErrLen, "data file size too large"); + return -1; + } + + DTCStatHeader *header = (DTCStatHeader *)mapfile(indexfile, indexsize); + if (header == NULL) + { + snprintf(szErrMsg, iErrLen, "map stat file error"); + return -1; + } + + header->signature = *(int *)"sTaT"; + header->version = 1; + header->numinfo = numinfo; + header->indexsize = indexsize; + header->datasize = datasize; + strncpy(header->name, name, sizeof(header->name)); + + DTCStatInfo *si = header->first(); + unsigned int off = 16; + + for (i = 0; i < NSYSID; i++, si = si->next()) + { + si->id = SysStatDefinition[i].id; + si->type = SysStatDefinition[i].type; + si->unit = SysStatDefinition[i].unit; + si->off = off; + si->cnt = 0; + si->cnt1 = 0; + strncpy(si->name, SysStatDefinition[i].name, sizeof(si->name)); + off += si->data_size(); + } + for (i = 0; statdef[i].id; i++, si = si->next()) + { + si->id = statdef[i].id; + si->type = statdef[i].type; + si->unit = statdef[i].unit; + si->off = off; + si->cnt = 0; + si->cnt1 = 0; + strncpy(si->name, statdef[i].name, sizeof(si->name)); + off += si->data_size(); + + if (si->is_sample()) + { + unsigned char &j = si->cnt; + for (j = 0; j < 16 && statdef[i].arg[j]; j++) + si->vptr[j] = statdef[i].arg[j]; + } + else if (si->is_expr()) + { + si->cnt = statdef[i].cnt; + si->cnt1 = statdef[i].cnt1; + for (int j = 0; j < si->cnt + si->cnt1; j++) + si->vptr[j] = statdef[i].arg[j]; + } + } + header->createtime = time(NULL); // create time + munmap(header, indexsize); + char buf[strlen(indexfile) + 10]; + strncpy(buf, indexfile, strlen(indexfile) + 10); + char *p = strrchr(buf, '.'); + if (p == NULL) + p = buf + strlen(buf); + + strncpy(p, ".dat", 5); + unlink(buf); + strncpy(p, ".10s", 5); + unlink(buf); + strncpy(p, ".10m", 5); + unlink(buf); + strncpy(p, ".all", 5); + unlink(buf); + return 0; +} + +void *StatManager::mapfile(const char *fn, int size) +{ + int fd = open(fn, O_RDWR | O_CREAT, 0666); + void *map = NULL; + if (fd >= 0) + { + if (size > 0) + ftruncate(fd, size); + else + size = lseek(fd, 0L, SEEK_END); + + if (size > 0) + map = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); + } + else if (size > 0) + { + map = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + } + if (map == MAP_FAILED) + { + map = NULL; + } + return map; +} + +#if HAS_ATOMIC8 +StatItem StatManager::get_item(unsigned int id) +{ + //P __a(this); + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample()) + { + StatItem v; + return v; + } + return (StatItem)&at_cur(i->offset()); +} + +StatItem StatManager::get10s_item(unsigned int id) +{ + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample()) + { + StatItem v; + return v; + } + return (StatItem)&at_10s(i->offset()); +} +#else + +StatItemS32 StatManager::get_item_s32(unsigned int id) +{ + P __a(this); + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample() || !i->istype(1)) + { + StatItemS32 v; + return v; + } + i->ltype = 1; + return (StatItemS32)(int32_t *)&at_cur(i->offset()); +} + +StatItemU32 StatManager::get_item_u32(unsigned int id) +{ + P __a(this); + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample() || i->istype(2)) + { + StatItemU32 v; + return v; + } + i->ltype = 2; + return (StatItemU32)(uint32_t *)&at_cur(i->offset()); +} + +StatItem StatManager::get_item(unsigned int id) +{ + P __a(this); + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample() || !i->istype(3)) + { + StatItem v; + return v; + } + i->ltype = 3; + + if (i->vobj == NULL) + i->vobj = new StatItemObject(&at_cur(i->offset())); + return StatItem(i->vobj); +} + +StatItem StatManager::get10s_item(unsigned int id) +{ + P __a(this); + + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample() || !i->istype(3)) + { + StatItem v; + return v; + } + i->ltype = 3; + + if (i->vobj == NULL) + i->vobj = new StatItemObject(&at_10s(i->offset())); + return StatItem(i->vobj); +} + +#endif +int64_t StatManager::get10_s_item_value(unsigned int id) +{ + P __a(this); + int64_t ddwValue = 0; + StatInfo *i = idmap[id]; + if (i == NULL || i->is_sample()) + { + return ddwValue; + } + ddwValue = at_10s(i->offset()); + return ddwValue; +} + +StatSample StatManager::get_sample(unsigned int id) +{ + P __a(this); + StatInfo *i = idmap[id]; + if (i == NULL || !i->is_sample()) + { + StatSample v; + return v; + } + + if (i->sobj == NULL) + i->sobj = new StatSampleObject(i->si, &at_cur(i->offset())); + + StatSample v(i->sobj); + return v; +} + +int StatManager::set_count_base(unsigned int id, const int64_t *v, int c) +{ + if (c < 0) + c = 0; + else if (c > 16) + c = 16; + + P __a(this); + StatInfo *i = idmap[id]; + if (i == NULL || !i->is_sample()) + return -1; + + mprotect(header, indexsize, PROT_READ | PROT_WRITE); + if (c > 0) + memcpy(i->si->vptr, v, sizeof(int64_t) * c); + i->si->cnt = c; + mprotect(header, indexsize, PROT_READ); + return c; +} + +int StatManager::get_count_base(unsigned int id, int64_t *v) +{ + P __a(this); + StatInfo *i = idmap[id]; + if (i == NULL || !i->is_sample()) + return -1; + + if (i->si->cnt > 0) + memcpy(v, i->si->vptr, sizeof(int64_t) * i->si->cnt); + + return i->si->cnt; +} + +int64_t call_imm(StatManager::StatExpr *info, const char *map) +{ + return info->val; +} + +int64_t call_id(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0); +} + +int64_t call_negative(StatManager::StatExpr *info, const char *map) +{ + return -*(const int64_t *)(map + info->off0); +} + +int64_t call_shift_1(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) << 1; +} + +int64_t call_shift_2(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) << 2; +} + +int64_t call_shift_3(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) << 3; +} + +int64_t call_shift_4(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) << 4; +} + +int64_t call_id_multi(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) * info->val; +} + +int64_t call_id_multi2(StatManager::StatExpr *info, const char *map) +{ + return *(const int64_t *)(map + info->off0) * *(const int64_t *)(map + info->off1); +} + +StatManager::StatExpr *StatManager::init_expr(unsigned int cnt, int64_t *arg) +{ + StatInfo *info; + StatExpr *expr = new StatExpr[cnt]; + int val, id; + int sub; + for (unsigned int i = 0; i < cnt; i++) + { + val = arg[i] >> 32; + id = arg[i] & 0xFFFFFFFF; + if (id == 0) + { + expr[i].val = val; + expr[i].off0 = 0; + expr[i].off1 = 0; + + expr[i].call = &call_imm; + } + else if ((id & 0x80000000) == 0) + { + expr[i].val = val; + + sub = id % 20; + id /= 20; + info = idmap[id]; + if (info == NULL || (sub > 0 && !info->is_sample())) + { + goto bad; + } + // not yet initialized + if (info->is_expr() && info->expr == NULL) + goto bad; + expr[i].off0 = info->offset() + sub * sizeof(int64_t); + expr[i].off1 = 0; + + switch (val) + { + case -1: + expr[i].call = &call_negative; + break; + case 0: + expr[i].call = &call_imm; + break; + case 1: + expr[i].call = &call_id; + break; + case 2: + expr[i].call = &call_shift_1; + break; + case 4: + expr[i].call = &call_shift_2; + break; + case 8: + expr[i].call = &call_shift_3; + break; + case 16: + expr[i].call = &call_shift_4; + break; + default: + expr[i].call = &call_id_multi; + break; + } + } + else + { + expr[i].val = 0; + + id &= 0x7FFFFFFF; + sub = id % 20; + id /= 20; + info = idmap[id]; + if (info == NULL || (sub > 0 && !info->is_sample())) + { + goto bad; + } + // not yet initialized + if (info->is_expr() && info->expr == NULL) + goto bad; + expr[i].off0 = info->offset() + sub * sizeof(int64_t); + + id = val & 0x7FFFFFFF; + sub = id % 20; + id /= 20; + info = idmap[id]; + if (info == NULL || (sub > 0 && !info->is_sample())) + { + goto bad; + } + // not yet initialized + if (info->is_expr() && info->expr == NULL) + goto bad; + expr[i].off1 = info->offset() + sub * sizeof(int64_t); + + expr[i].call = &call_id_multi2; + } + } + return expr; +bad: + delete[] expr; + return NULL; +} + +int64_t StatManager::calc_expr(const char *map, unsigned int cnt, StatExpr *expr) +{ + int64_t v = 0; + + for (unsigned int i = 0; i < cnt; i++) + v += expr[i].call(&expr[i], map); + return v; +} + +static inline void trend(int64_t &m, int64_t s) { m = (m * 63 + (s << 10)) >> 6; } +static inline void ltrend(int64_t &m, int64_t s) { m = (m * 255 + s) >> 8; } + +void StatManager::run_job_once(void) +{ + atomic_t *a0 = (atomic_t *)&at_cur(0); + int a0v = atomic_add_return(1, a0); + at_cur(4 * 8) = time(NULL); // checkpoint time + + for (unsigned i = 0; i < numinfo; i++) + { + unsigned off = info[i].offset(); + switch (info[i].type()) + { + case SA_SAMPLE: + if (info[i].sobj) + { + info[i].sobj->output(&at_10s(off)); + } + else + { + memcpy(&at_10s(off), &at_cur(off), 18 * sizeof(int64_t)); + memset(&at_cur(off), 0, 18 * sizeof(int64_t)); + } + + for (unsigned n = 0; n < 18; n++) + { + // count all + at_all(off, n) += at_10s(off, n); + // 10m + trend(at_10m(off, n), at_10s(off, n)); + } + break; + case SA_COUNT: +#if !HAS_ATOMIC8 + if (info[i].ltype == 1) + { + StatItemS32 vi((int32_t *)&at_cur(off)); + at_10s(off) = vi.clear(); + } + else +#else + { + StatItem vi(&at_cur(off)); + at_10s(off) = vi.clear(); + } +#endif + // count all + at_all(off) += at_10s(off); + + // 10m + trend(at_10m(off), at_10s(off)); + break; + case SA_VALUE: +#if !HAS_ATOMIC8 + if (info[i].ltype == 1) + { + StatItemS32 vi((int32_t *)&at_cur(off)); + at_10s(off) = vi.get(); + } + else +#else + { + StatItem vi(&at_cur(off)); + at_10s(off) = vi.get(); + } +#endif + // 10m + trend(at_10m(off), at_10s(off)); + ltrend(at_all(off), at_10m(off)); + break; + case SA_CONST: + at_10s(off) = at_cur(off); + at_10m(off) = at_cur(off); + at_all(off) = at_cur(off); + break; + } + } + + // calculate expression + for (unsigned i = 0; i < expr.size(); i++) + { + StatInfo &e = *(expr[i]); + unsigned off = e.offset(); + int64_t sum, div; + + if (e.si->cnt1 == 0) + { + at_10s(off) = calc_expr(fmap[SC_10S], e.si->cnt, e.expr); + at_all(off) = calc_expr(fmap[SC_ALL], e.si->cnt, e.expr); + } + else + { + div = calc_expr(fmap[SC_10S], e.si->cnt1, e.expr + e.si->cnt); + if (div != 0) + { + sum = calc_expr(fmap[SC_10S], e.si->cnt, e.expr); + at_10s(off) = sum / div; + } + div = calc_expr(fmap[SC_ALL], e.si->cnt1, e.expr + e.si->cnt); + if (div != 0) + { + sum = calc_expr(fmap[SC_ALL], e.si->cnt, e.expr); + at_all(off) = sum / div; + } + } + trend(at_10m(off), at_10s(off)); + } + atomic_set(a0 + 1, a0v); +} + +int StatManager::Lock(const char *type) +{ + if (lockfd >= 0) + return 0; + + lockfd = unix_socket_lock("tlock-stat-%s-%llu-%llu", + type, (long long)lockdev, (long long)lockino); + return lockfd >= 0 ? 0 : -1; +} + +void StatManager::clear(void) +{ + for (unsigned i = 0; i < numinfo; i++) + { + unsigned off = info[i].offset(); + switch (info[i].type()) + { + case SA_SAMPLE: + break; + case SA_COUNT: + at_cur(off) = 0; + at_10s(off) = 0; + at_10m(off) = 0; + // at_all(off) =0; + break; + case SA_VALUE: + break; + case SA_CONST: + break; + } + } +} diff --git a/src/search_local/index_storage/stat/stat_manager.h b/src/search_local/index_storage/stat/stat_manager.h new file mode 100644 index 0000000..c1cc656 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_manager.h @@ -0,0 +1,419 @@ +/* + * ===================================================================================== + * + * Filename: stat_manager.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef _STAT_MANAGER_H_ +#define _STAT_MANAGER_H_ + +#include +#include +#include +#include +#include +#include + +#include "stat_info.h" +#include "atomic.h" + +struct StatLock +{ +public: + class P + { + private: + StatLock *ptr; + + public: + P(StatLock *p) + { + ptr = p; + pthread_mutex_lock(&ptr->lock); + } + ~P() { pthread_mutex_unlock(&ptr->lock); } + }; + +private: + friend class P; + pthread_mutex_t lock; + +public: + ~StatLock() {} + StatLock() { pthread_mutex_init(&lock, 0); } +}; + +#if HAS_ATOMIC8 +struct StatItem +{ +private: + typedef int64_t V; + static int64_t dummy; + atomic8_t *ptr; + +public: + ~StatItem(void) {} + StatItem(void) { ptr = (atomic8_t *)&dummy; } + StatItem(volatile V *p) : ptr((atomic8_t *)p) {} + StatItem(const StatItem &f) : ptr(f.ptr) {} + + inline V get(void) const { return atomic8_read(ptr); } + inline V set(V v) + { + atomic8_set(ptr, v); + return v; + } + inline V add(V v) { return atomic8_add_return(v, ptr); } + inline V sub(V v) { return atomic8_sub_return(v, ptr); } + inline V clear(void) { return atomic8_clear(ptr); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; +typedef StatItem StatItemU32; +typedef StatItem StatItemS32; + +#else +struct StatItemU32 +{ +private: + typedef uint32_t V; + static uint32_t dummy; + atomic_t *ptr; + +public: + ~StatItemU32(void) {} + StatItemU32(void) { ptr = (atomic_t *)&dummy; } + StatItemU32(volatile V *p) : ptr((atomic_t *)p) {} + StatItemU32(const StatItemU32 &f) : ptr(f.ptr) {} + + inline V get(void) const { return atomic_read(ptr); } + inline V set(V v) + { + atomic_set(ptr, v); + return v; + } + inline V add(V v) { return atomic_add_return(v, ptr); } + inline V sub(V v) { return atomic_sub_return(v, ptr); } + inline V clear(void) { return atomic_clear(ptr); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; +struct StatItemS32 +{ +private: + typedef int32_t V; + static int32_t dummy; + atomic_t *ptr; + +public: + ~StatItemS32(void) {} + StatItemS32(void) { ptr = (atomic_t *)&dummy; } + StatItemS32(volatile V *p) : ptr((atomic_t *)p) {} + StatItemS32(const StatItemS32 &f) : ptr(f.ptr) {} + + inline V get(void) const { return atomic_read(ptr); } + inline V set(V v) + { + atomic_set(ptr, v); + return v; + } + inline V add(V v) { return atomic_add_return(v, ptr); } + inline V sub(V v) { return atomic_sub_return(v, ptr); } + inline V clear(void) { return atomic_clear(ptr); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) const { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; + +struct StatItemObject : private StatLock +{ +private: + typedef int64_t V; + volatile V *ptr; + + StatItemObject(const StatItemObject &); + +public: + ~StatItemObject(void) {} + StatItemObject(void) {} + StatItemObject(volatile V *p) : ptr(p) {} + + inline V get(void) + { + P a(this); + return *ptr; + } + inline V set(V v) + { + P a(this); + *ptr = v; + return *ptr; + } + inline V add(V v) + { + P a(this); + *ptr += v; + return *ptr; + } + inline V sub(V v) + { + P a(this); + *ptr -= v; + return *ptr; + } + inline V clear(void) + { + P a(this); + V v = *ptr; + *ptr = 0; + return v; + } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; + +struct StatItem +{ +private: + typedef int64_t V; + StatItemObject *ptr; + static StatItemObject dummy; + +public: + ~StatItem(void) {} + StatItem(void) { ptr = &dummy; } + StatItem(StatItemObject *p) : ptr(p) {} + StatItem(const StatItem &f) : ptr(f.ptr) {} + + inline V get(void) { return ptr->get(); } + inline V set(V v) { return ptr->set(v); } + inline V add(V v) { return ptr->add(v); } + inline V sub(V v) { return ptr->sub(v); } + inline V clear(void) { return ptr->clear(); } + inline V inc(void) { return add(1); } + inline V dec(void) { return sub(1); } + inline operator V(void) { return get(); } + inline V operator=(V v) { return set(v); } + inline V operator+=(V v) { return add(v); } + inline V operator-=(V v) { return sub(v); } + inline V operator++(void) { return inc(); } + inline V operator--(void) { return dec(); } + inline V operator++(int) { return inc() - 1; } + inline V operator--(int) { return dec() + 1; } +}; +#endif + +struct StatSampleObject : private StatLock +{ +private: + const DTCStatInfo *info; + int64_t *cnt; + StatSampleObject(const StatSampleObject &); + static int64_t dummyCnt[2]; + static const DTCStatInfo dummyInfo; + +public: + ~StatSampleObject(void) {} + StatSampleObject(void) : info(&dummyInfo), cnt(dummyCnt) {} + StatSampleObject(const DTCStatInfo *i, int64_t *c) : info(i), cnt(c) {} + + int64_t count(unsigned int n = 0); + int64_t sum(void); + int64_t average(int64_t o); + void push(int64_t v); + void output(int64_t *v); +}; + +struct StatSample +{ +private: + StatSampleObject *ptr; + static StatSampleObject dummy; + +public: + ~StatSample(void) {} + StatSample(void) { ptr = &dummy; } + StatSample(StatSampleObject *p) : ptr(p) {} + StatSample(const StatSample &f) : ptr(f.ptr) {} + + int64_t count(unsigned int n = 0) { return ptr->count(n); } + int64_t sum(void) { return ptr->sum(); } + int64_t average(int64_t o) { return ptr->average(o); } + void push(int64_t v) { return ptr->push(v); } + void operator<<(int64_t v) { return ptr->push(v); } +}; + +class StatManager : protected StatLock +{ +public: // types + struct StatExpr + { + unsigned int off0; + unsigned int off1; + int val; + int64_t (*call)(StatExpr *info, const char *map); + }; + +protected: // types + struct StatInfo + { + private: + friend class StatManager; + +#if !HAS_ATOMIC8 + StatItemObject *vobj; + int ltype; +#endif + StatManager *owner; + DTCStatInfo *si; + StatSampleObject *sobj; + StatExpr *expr; + + StatInfo() : +#if !HAS_ATOMIC8 + vobj(0), ltype(0), +#endif + owner(0), si(0), sobj(0), expr(0) + { + } + ~StatInfo() + { +#if !HAS_ATOMIC8 + if (vobj) + delete vobj; +#endif + if (sobj) + delete sobj; + if (expr) + delete[] expr; + } + +#if !HAS_ATOMIC8 + int istype(int t) + { + return ltype == 0 || ltype == t; + } +#endif + public: + const DTCStatInfo &info(void) { return *si; } + inline unsigned id(void) const { return si->id; } + inline unsigned int type(void) const { return si->type; } + inline unsigned int unit(void) const { return si->unit; } + inline int is_sample(void) const { return si->is_sample(); } + inline int is_counter(void) const { return si->is_counter(); } + inline int is_value(void) const { return si->is_value(); } + inline int is_const(void) const { return si->is_const(); } + inline int is_expr(void) const { return si->is_expr(); } + inline unsigned offset(void) const { return si->off; } + inline unsigned count(void) const { return si->cnt; } + inline const char *name(void) const { return si->name; } + }; + + static void *mapfile(const char *fn, int size); + +protected: // members + int lockfd; + dev_t lockdev; + ino_t lockino; + + char szErrMsg[128]; + DTCStatHeader *header; + unsigned int indexsize; + char *fmap[7]; + const unsigned int nfmap(void) const { return sizeof(fmap) / sizeof(*fmap); } + int64_t &at(unsigned int cat, unsigned off, unsigned int n = 0) { return ((int64_t *)(fmap[cat] + off))[n]; } + int64_t &at_cur(unsigned off, unsigned int n = 0) { return at(SC_CUR, off, n); } + int64_t &at_10s(unsigned off, unsigned int n = 0) { return at(SC_10S, off, n); } + int64_t &at_10m(unsigned off, unsigned int n = 0) { return at(SC_10M, off, n); } + int64_t &at_all(unsigned off, unsigned int n = 0) { return at(SC_ALL, off, n); } + int64_t &atc_10s(unsigned off, unsigned int n = 0) { return at(SCC_10S, off, n); } + int64_t &atc_10m(unsigned off, unsigned int n = 0) { return at(SCC_10M, off, n); } + int64_t &atc_all(unsigned off, unsigned int n = 0) { return at(SCC_ALL, off, n); } + + std::map idmap; + std::vector expr; + StatInfo *info; + unsigned int numinfo; + +private: + StatManager(const StatManager &); + +public: // public method + StatManager(void); + ~StatManager(void); + + const char *error_message(void) const { return szErrMsg; } + int init_stat_info(const char *, const char *, int isc = 0); + static int CreateStatIndex(const char *, const char *, const DTCStatDefinition *, char *, int); + + StatItem get_item(unsigned int id); + StatItem get10s_item(unsigned int id); + int64_t get10_s_item_value(unsigned int id); /*get 10s static value , add by tomchen*/ +#if HAS_ATOMIC8 + inline StatItemU32 get_item_u32(unsigned int id) + { + return get_item(id); + } + inline StatItemS32 get_item_s32(unsigned int id) { return get_item(id); } +#else + StatItemU32 get_item_u32(unsigned int id); + StatItemS32 get_item_s32(unsigned int id); +#endif + StatSample get_sample(unsigned int id); + + int set_count_base(unsigned int id, const int64_t *v, int c); + int get_count_base(unsigned int id, int64_t *v); + + int Lock(const char *type); + int Unlock(void); + void run_job_once(void); + void clear(void); + +protected: + StatExpr *init_expr(unsigned int cnt, int64_t *arg); + int64_t calc_expr(const char *, unsigned int, StatExpr *); +}; + +#endif diff --git a/src/search_local/index_storage/stat/stat_manager_container_thread.cc b/src/search_local/index_storage/stat/stat_manager_container_thread.cc new file mode 100644 index 0000000..0b0cfb0 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_manager_container_thread.cc @@ -0,0 +1,179 @@ +/* + * ===================================================================================== + * + * Filename: StatManagerContainerThread.cc + * + * Description: + * + * Version: 1.0 + * Created: 27/05/2014 08:50:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "stat_manager_container_thread.h" + +StatManagerContainerThread *StatManagerContainerThread::getInstance() +{ + static StatManagerContainerThread msStatThread; + + return &msStatThread; +} + +StatManagerContainerThread::StatManagerContainerThread() +{ + pthread_mutex_init(&wakeLock, NULL); +} + +StatManagerContainerThread::~StatManagerContainerThread() +{ + //log_debug("the ~StatManagerContainerThread been called when killall!"); + std::map::iterator iter = m_statManagers.begin(); + std::map::const_iterator endIter = m_statManagers.end(); + + for (; iter != endIter; ++iter) + { + iter->second->clear(); + } +} + +void *StatManagerContainerThread::__threadentry(void *p) +{ + StatManagerContainerThread *my = (StatManagerContainerThread *)p; + my->thread_loop(); + return NULL; +} + +int StatManagerContainerThread::start_background_thread(void) +{ + P __a(this); + + if (pthread_mutex_trylock(&wakeLock) == 0) + { + int ret = pthread_create(&threadid, 0, __threadentry, (void *)this); + if (ret != 0) + { + errno = ret; + return -1; + } + } + return 0; +} + +int StatManagerContainerThread::stop_background_thread(void) +{ + P __a(this); + + if (pthread_mutex_trylock(&wakeLock) == 0) + { + pthread_mutex_unlock(&wakeLock); + return 0; + } + pthread_mutex_unlock(&wakeLock); + int ret = pthread_join(threadid, 0); + if (ret == 0) + log_info("Thread[stat] stopped."); + else + log_notice("cannot stop thread[stat]"); + return ret; +} + +static void BlockAllSignals(void) +{ + sigset_t sset; + sigfillset(&sset); + sigdelset(&sset, SIGSEGV); + sigdelset(&sset, SIGBUS); + sigdelset(&sset, SIGABRT); + sigdelset(&sset, SIGILL); + sigdelset(&sset, SIGCHLD); + sigdelset(&sset, SIGFPE); + pthread_sigmask(SIG_BLOCK, &sset, &sset); +} + +void StatManagerContainerThread::thread_loop(void) +{ + _set_log_thread_name_("stat"); + log_info("thread stat[%d] started.", _gettid_()); + + BlockAllSignals(); + + time_t next = 0; + struct timeval tv; + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + struct timespec ts; + ts.tv_sec = next; + ts.tv_nsec = 0; + + while (pthread_mutex_timedlock(&wakeLock, &ts) != 0) + { + gettimeofday(&tv, NULL); + if (tv.tv_sec >= next) + { + mLock.lock(); + std::map::iterator iter = m_statManagers.begin(); + std::map::const_iterator endIter = m_statManagers.end(); + + for (; iter != endIter; ++iter) + { + iter->second->run_job_once(); + } + mLock.unlock(); + + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + } + ts.tv_sec = next; + ts.tv_nsec = 0; + } + pthread_mutex_unlock(&wakeLock); +} + +int StatManagerContainerThread::add_stat_manager(uint32_t moudleId, StatManager *stat) +{ + ScopedLock autoLock(mLock); + if (m_statManagers.find(moudleId) != m_statManagers.end()) + return -1; + + m_statManagers[moudleId] = stat; + return 0; +} + +int StatManagerContainerThread::delete_stat_manager(uint32_t moudleId) +{ + ScopedLock autoLock(mLock); + std::map::iterator i = m_statManagers.find(moudleId); + if (i != m_statManagers.end()) + { + delete i->second; + m_statManagers.erase(i); + + return 0; + } + return -1; +} + +StatItemU32 StatManagerContainerThread::operator()(int moduleId, int cId) +{ + StatItemU32 dummy; + std::map::iterator i = m_statManagers.find(moduleId); + if (i != m_statManagers.end()) + { + return i->second->get_item_u32(cId); + } + return dummy; +} diff --git a/src/search_local/index_storage/stat/stat_manager_container_thread.h b/src/search_local/index_storage/stat/stat_manager_container_thread.h new file mode 100644 index 0000000..d1af8c0 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_manager_container_thread.h @@ -0,0 +1,50 @@ +/* + * ===================================================================================== + * + * Filename: StatManagerContainerThread.h + * + * Description: + * + * Version: 1.0 + * Created: 27/05/2014 08:50:19 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming (prudence), linjinming@jd.com + * Company: JD, China + * + * ===================================================================================== + */ +#ifndef __H_STAT_MANAGER_CONTAINER_THREAD___ +#define __H_STAT_MANAGER_CONTAINER_THREAD___ + +#include "stat_manager.h" +#include "lock.h" + +class StatManagerContainerThread : protected StatLock +{ +public: // public method + StatManagerContainerThread(void); + ~StatManagerContainerThread(void); + static StatManagerContainerThread *getInstance(); + +public: // background access + int start_background_thread(void); + int stop_background_thread(void); + int add_stat_manager(uint32_t moudleId, StatManager *stat); + int delete_stat_manager(uint32_t moudleId); + StatItemU32 operator()(int moduleId, int cId); + + //int init_stat_info(const char *name, const char *fn) { return StatManager::init_stat_info(name, fn, 0); } +private: + pthread_t threadid; + ; + pthread_mutex_t wakeLock; + ; + static void *__threadentry(void *); + void thread_loop(void); + std::map m_statManagers; + Mutex mLock; +}; + +#endif diff --git a/src/search_local/index_storage/stat/stat_thread.cc b/src/search_local/index_storage/stat/stat_thread.cc new file mode 100644 index 0000000..da34099 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_thread.cc @@ -0,0 +1,121 @@ +/* + * ===================================================================================== + * + * Filename: stat_thread.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stat_thread.h" + +StatThread::StatThread() +{ + pthread_mutex_init(&wakeLock, NULL); +} + +StatThread::~StatThread() +{ +} + +void *StatThread::__threadentry(void *p) +{ + StatThread *my = (StatThread *)p; + my->thread_loop(); + return NULL; +} + +int StatThread::start_background_thread(void) +{ + P __a(this); + + if (pthread_mutex_trylock(&wakeLock) == 0) + { + int ret = pthread_create(&threadid, 0, __threadentry, (void *)this); + if (ret != 0) + { + errno = ret; + return -1; + } + } + return 0; +} + +int StatThread::stop_background_thread(void) +{ + P __a(this); + + if (pthread_mutex_trylock(&wakeLock) == 0) + { + pthread_mutex_unlock(&wakeLock); + return 0; + } + pthread_mutex_unlock(&wakeLock); + int ret = pthread_join(threadid, 0); + if (ret == 0) + log_info("Thread[stat] stopped."); + else + log_notice("cannot stop thread[stat]"); + return ret; +} + +static void BlockAllSignals(void) +{ + sigset_t sset; + sigfillset(&sset); + sigdelset(&sset, SIGSEGV); + sigdelset(&sset, SIGBUS); + sigdelset(&sset, SIGABRT); + sigdelset(&sset, SIGILL); + sigdelset(&sset, SIGCHLD); + sigdelset(&sset, SIGFPE); + pthread_sigmask(SIG_BLOCK, &sset, &sset); +} + +void StatThread::thread_loop(void) +{ + _set_log_thread_name_("stat"); + log_info("thread stat[%d] started.", _gettid_()); + + BlockAllSignals(); + + time_t next = 0; + struct timeval tv; + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + struct timespec ts; + ts.tv_sec = next; + ts.tv_nsec = 0; + + while (pthread_mutex_timedlock(&wakeLock, &ts) != 0) + { + gettimeofday(&tv, NULL); + if (tv.tv_sec >= next) + { + run_job_once(); + gettimeofday(&tv, NULL); + next = (tv.tv_sec / 10) * 10 + 10; + } + ts.tv_sec = next; + ts.tv_nsec = 0; + } + pthread_mutex_unlock(&wakeLock); +} diff --git a/src/search_local/index_storage/stat/stat_thread.h b/src/search_local/index_storage/stat/stat_thread.h new file mode 100644 index 0000000..48f22f9 --- /dev/null +++ b/src/search_local/index_storage/stat/stat_thread.h @@ -0,0 +1,43 @@ +/* + * ===================================================================================== + * + * Filename: stat_thread.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_STAT_THREAD___ +#define __H_STAT_THREAD___ + +#include "stat_manager.h" + +class StatThread : public StatManager +{ +public: // public method + StatThread(void); + ~StatThread(void); + +public: // background access + int start_background_thread(void); + int stop_background_thread(void); + int init_stat_info(const char *name, const char *fn) { return StatManager::init_stat_info(name, fn, 0); } + +private: + pthread_t threadid; + ; + pthread_mutex_t wakeLock; + ; + static void *__threadentry(void *); + void thread_loop(void); +}; + +#endif diff --git a/src/search_local/index_storage/stat/stat_tool.cc b/src/search_local/index_storage/stat/stat_tool.cc new file mode 100644 index 0000000..7daed0a --- /dev/null +++ b/src/search_local/index_storage/stat/stat_tool.cc @@ -0,0 +1,660 @@ +/* + * ===================================================================================== + * + * Filename: stat_tool.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: Norton, yangshuang68@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +#include "stat_client.h" +#include "stat_dtc.h" +#include "stat_attn.h" +#include "formater.h" +#include "stat_alarm_reporter.h" +#include +#include + +const char progname[] = "stattool"; +const char usage_argv[] = "dump Id or reporter"; +char cacheFile[256] = CACHE_CONF_NAME; +char tableFile[256] = TABLE_CONF_NAME; +StatClient stc; +std::vector idset; +TableFormater out; +int outfmt = TableFormater::FORMAT_ALIGNED; +unsigned char outnoname; +unsigned char rawdata; +unsigned char alldata; +unsigned char nobase; + +#if __WORDSIZE >= 64 +#define F64 "%ld" +#else +#define F64 "%lld" +#endif + +static inline void cell_id(unsigned int id) +{ + out.Cell("%u", id); +} + +static inline void cell_sample_id(unsigned int id, unsigned int n) +{ + if (outnoname) + out.Cell("%u.%u", id, n); + else + out.Cell("%u", id); +} + +static inline void cell_name(const char *name) +{ + if (outnoname) + return; + if (outfmt == TableFormater::FORMAT_ALIGNED) + out.Cell("%s:", name); + else + out.Cell("%s", name); +} + +static inline void cell_dummy(void) +{ + if (outfmt == TableFormater::FORMAT_ALIGNED) + out.Cell("-"); + else +#if GCC_MAJOR < 3 + out.Cell(" "); +#else + out.Cell(NULL); +#endif +} + +static inline void cell_base(int64_t v) +{ + if (outnoname) + return; + if (outfmt == TableFormater::FORMAT_ALIGNED) + out.Cell("count[>=" F64 "]:", v); + else + out.Cell("count[>=" F64 "]", v); +} + +static inline void cell_nbase(int v) +{ + if (outfmt == TableFormater::FORMAT_ALIGNED) + out.Cell("[%d]", v); + else + out.Cell("%d", v); +} + +static inline void cell_int(int64_t val) +{ + out.Cell(F64, val); +} + +static inline void cell_fixed(int64_t val, int n, int div) +{ + const char *sign = ""; + if (val < 0) + { + val = -val; + sign = "-"; + } + + out.Cell("%s" F64 ".%0*d", sign, val / div, n, (int)(val % div)); +} + +static inline void cell_percent(int64_t val) +{ + out.Cell(F64 "%%", val); +} + +static inline void cell_percent_fixed(int64_t val, int n, int div) +{ + const char *sign = ""; + if (val < 0) + { + val = -val; + sign = "-"; + } + + out.Cell("%s" F64 ".%0*d%%", sign, val / div, n, (int)(val % div)); +} + +static inline void cell_hms(int64_t val) +{ + const char *sign = ""; + if (val < 0) + { + val = -val; + sign = "-"; + } + + if (val < 60) + out.Cell("%s%d", sign, (int)val); + else if (val < 60 * 60) + out.Cell("%s%d:%02d", sign, (int)(val / 60), (int)(val % 60)); + else + out.Cell("%s" F64 ":%02d:%02d", sign, val / 3600, (int)((val / 60) % 60), (int)(val % 60)); +} + +static inline void cell_hmsmsec(int64_t val) +{ + const char *sign = ""; + if (val < 0) + { + val = -val; + sign = "-"; + } + if (val < 60 * 1000) + out.Cell("%s%d.%03d", sign, (int)(val / 1000), (int)(val % 1000)); + else if (val < 60 * 60 * 1000) + out.Cell("%s%d:%02d.%03d", sign, (int)(val / 60000), (int)((val / 1000) % 60), (int)(val % 1000)); + else + out.Cell("%s" F64 ":%02d:%02d.%03d", sign, + val / 3600000, (int)((val / 60000) % 60), (int)((val / 1000) % 60), (int)(val % 1000)); +} + +static inline void cell_hmsusec(int64_t val) +{ + const char *sign = ""; + if (val < 0) + { + val = -val; + sign = "-"; + } + out.Cell("%s" F64 ".%06d", sign, val / 1000000, (int)(val % 1000000)); +} + +static inline void cell_datetime(int64_t v) +{ + if (v == 0) + { + out.CellV("-"); + return; + } + time_t t = v; + struct tm tm; + localtime_r(&t, &tm); + out.CellV("%d-%d-%d %d:%02d:%02d", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec); +} + +static inline void cell_date(int64_t v) +{ + if (v == 0) + { + out.CellV("-"); + return; + } + time_t t = v; + struct tm tm; + localtime_r(&t, &tm); + out.CellV("%d-%d-%d", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday); +} + +static inline void cell_version(int64_t v) +{ + out.CellV("%d.%d.%d", (int)v / 10000, (int)v / 100 % 100, (int)v % 100); +} + +static inline void cell_bool(int64_t v) +{ + if (0 == v) + { + out.CellV("NO"); + } + else + { + out.CellV("YES"); + } +} + +inline void row_init(void) +{ + out.new_row(); +} + +inline void row_clear(void) +{ + out.clear_row(); +} + +inline void dump_table(void) +{ + out.Dump(stdout, outfmt); +} + +void cell_value(int unit, int64_t val) +{ + if (rawdata) + unit = SU_INT; + switch (unit) + { + default: + case SU_HIDE: + case SU_INT: + cell_int(val); + break; + case SU_INT_1: + cell_fixed(val, 1, 10); + break; + case SU_INT_2: + cell_fixed(val, 2, 100); + break; + case SU_INT_3: + cell_fixed(val, 3, 1000); + break; + case SU_INT_4: + cell_fixed(val, 4, 10000); + break; + case SU_INT_5: + cell_fixed(val, 5, 100000); + break; + case SU_INT_6: + cell_fixed(val, 6, 1000000); + break; + case SU_MSEC: + cell_hmsmsec(val); + break; + case SU_USEC: + cell_hmsusec(val); + break; + case SU_TIME: + cell_hms(val); + break; + case SU_DATE: + cell_date(val); + break; + case SU_DATETIME: + cell_datetime(val); + break; + case SU_VERSION: + cell_version(val); + break; + case SU_BOOL: + cell_bool(val); + break; + case SU_PERCENT: + cell_percent(val); + break; + case SU_PERCENT_1: + cell_percent_fixed(val, 1, 10); + break; + case SU_PERCENT_2: + cell_percent_fixed(val, 2, 100); + break; + case SU_PERCENT_3: + cell_percent_fixed(val, 3, 1000); + break; + } +} + +void dump_data(void) +{ + int64_t sc[16]; + int sn; + + unsigned int i; + StatClient::iterator s; + + stc.check_point(); + + for (i = 0; i < idset.size(); i++) + { + s = idset[i]; + + if (alldata == 0 && s->unit() == SU_HIDE) + continue; + + row_init(); + switch (s->type()) + { + case SA_SAMPLE: + cell_id(s->id()); + cell_name(s->name()); + //cell_value(s->unit(), stc.read_sample_average(s, SC_CUR)); + cell_value(s->unit(), stc.read_sample_average(s, SCC_10S)); + cell_value(s->unit(), stc.read_sample_average(s, SCC_10M)); + cell_value(s->unit(), stc.read_sample_average(s, SCC_ALL)); + + if (nobase < 2) + { + row_init(); + cell_sample_id(s->id(), 1); + cell_name("count[all]"); + //cell_value(SU_INT, stc.read_sample_counter(s, SC_CUR)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_10S)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_10M)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_ALL)); + } + if (nobase == 0) + { + sn = stc.get_count_base(s->id(), sc); + for (int n = 1; n <= sn; n++) + { + row_init(); + cell_sample_id(s->id(), n + 1); + cell_base(sc[n - 1]); + //cell_value(SU_INT, stc.read_sample_counter(s, SC_CUR, n)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_10S, n)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_10M, n)); + cell_value(SU_INT, stc.read_sample_counter(s, SCC_ALL, n)); + } + } + + break; + + case SA_COUNT: + cell_id(s->id()); + cell_name(s->name()); + //cell_value(s->unit(), stc.read_counter_value(s, SC_CUR)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10S)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10M)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_ALL)); + break; + case SA_VALUE: + cell_id(s->id()); + cell_name(s->name()); + //cell_value(s->unit(), stc.read_counter_value(s, SC_CUR)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10S)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10M)); + cell_dummy(); + break; + + case SA_CONST: + cell_id(s->id()); + cell_name(s->name()); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10S)); + switch (s->unit()) + { + case SU_DATETIME: + case SU_DATE: + case SU_VERSION: + case SU_BOOL: + break; + default: + //cell_dummy(); + cell_dummy(); + cell_dummy(); + break; + } + break; + + case SA_EXPR: + cell_id(s->id()); + cell_name(s->name()); + cell_dummy(); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10S)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_10M)); + cell_value(s->unit(), stc.read_counter_value(s, SCC_ALL)); + break; + + default: + row_clear(); + } + } + dump_table(); +} + +void dump_base(void) +{ + int64_t sc[16]; + int sn; + + unsigned int i; + StatClient::iterator s; + + stc.check_point(); + + for (i = 0; i < idset.size(); i++) + { + s = idset[i]; + row_init(); + sn = stc.get_count_base(s->id(), sc); + cell_id(s->id()); + cell_name(s->name()); + cell_nbase(sn); + for (int n = 0; n < sn; n++) + cell_value(s->unit(), sc[n]); + } + dump_table(); +} + +void create_files(void) +{ + char buf[256]; + if (statmgr.CreateStatIndex("dtcd", STATIDX, StatDefinition, buf, sizeof(buf)) < 0) + { + fprintf(stderr, "Fail to create stat index file: %s\n", buf); + exit(-3); + } + fprintf(stderr, "stat index created: %s\n", STATIDX); +} + +void init(void) +{ + int ret; + + ret = stc.init_stat_info("dtcd", STATIDX); + if (ret < 0) + { + fprintf(stderr, "Cannot Initialize StatInfo: %s\n", stc.error_message()); + exit(-1); + } +} + +void parse_stat_id(int argc, char **argv) +{ + StatClient::iterator n; + + if (argc == 0) + { + for (n = stc.begin(); n != stc.end(); n++) + idset.push_back(n); + return; + } + + for (; argc > 0; argc--, argv++) + { + int s, e; + switch (sscanf(argv[0], "%d-%d", &s, &e)) + { + case 2: + for (n = stc.begin(); n != stc.end(); n++) + { + if ((int)n->id() >= s && (int)n->id() <= e) + idset.push_back(n); + } + break; + case 1: + if ((n = stc[s]) != NULL) + { + idset.push_back(n); + break; + } + // fall through + default: + fprintf(stderr, "Invalid stat id [%s]\n", argv[0]); + exit(-4); + } + } +} + +void parse_sample_id(int argc, char **argv) +{ + StatClient::iterator n; + + if (argc == 0) + { + for (n = stc.begin(); n != stc.end(); n++) + if (n->is_sample()) + idset.push_back(n); + return; + } + + for (; argc > 0; argc--, argv++) + { + int s, e; + switch (sscanf(argv[0], "%d-%d", &s, &e)) + { + case 2: + for (n = stc.begin(); n != stc.end(); n++) + { + if ((int)n->id() >= s && (int)n->id() <= e && n->is_sample()) + idset.push_back(n); + } + break; + case 1: + if ((n = stc[s]) != NULL && n->is_sample()) + { + idset.push_back(n); + break; + } + // fall through + default: + fprintf(stderr, "Invalid stat sample id [%s]\n", argv[0]); + exit(-4); + } + } +} + +void alter_base(int argc, char **argv) +{ + StatClient::iterator n = NULL; + int64_t sc[16]; + + if (argc == 0 || (n = stc[atoi(argv[0])]) == NULL) + { + fprintf(stderr, "A stat sample id required\n"); + exit(-5); + } + argv++, argc--; + if (argc > 16) + { + fprintf(stderr, "number of count base must <= 16\n"); + exit(-5); + } + for (int i = 0; i < argc; i++) + sc[i] = strtoll(argv[i], 0, 0); + int ret = stc.set_count_base(n->id(), sc, argc); + if (ret < 0) + { + fprintf(stderr, "setbase failed for id: %d\n", n->id()); + exit(-5); + } + idset.push_back(n); + dump_base(); +} + +void usage(void) +{ + fprintf(stderr, + "Usage: stattool [-nct] cmd [args...]\n" + "options list:\n" + " -a output hidden id too\n" + " -r output unformatted data\n" + " -n Don't output stat name\n" + " -t use tab seperated format\n" + " -c use [,] seperated format\n" + "command list:\n" + " create\n" + " dump [id|id-id]...\n" + " getbase [id|id-id]...\n" + " setbase id v1 v2...\n"); + exit(-2); +} + +int main(int argc, char **argv) +{ + argv++, --argc; + + while (argc > 0 && argv[0][0] == '-') + { + const char *p = argv[0]; + char c; + while ((c = *++p)) + { + switch (c) + { + case 'a': + alldata = 1; + break; + case 'b': + nobase++; + break; + case 'r': + rawdata = 1; + break; + case 't': + outfmt = TableFormater::FORMAT_TABBED; + break; + case 'c': + outfmt = TableFormater::FORMAT_COMMA; + break; + case 'n': + outnoname = 1; + break; + case 'h': + case '?': + default: + fprintf(stderr, "Unknown options [%c]\n", c); + usage(); + } + } + argv++, --argc; + } + + if (argc <= 0) + usage(); + else if (!strcasecmp(argv[0], "help")) + usage(); + else if (!strcasecmp(argv[0], "create")) + { + create_files(); + } + else if (!strcasecmp(argv[0], "dump")) + { + init(); + parse_stat_id(--argc, ++argv); + dump_data(); + } + else if (!strcasecmp(argv[0], "getbase")) + { + init(); + parse_sample_id(--argc, ++argv); + dump_base(); + } + else if (!strcasecmp(argv[0], "setbase")) + { + init(); + alter_base(--argc, ++argv); + } + else if (!strcasecmp(argv[0], "reporter")) + { + if (dtc_daemon_init(argc, argv) < 0) + return -1; + + init(); + ALARM_REPORTER->set_stat_client(&stc); + ALARM_REPORTER->set_time_out(5); + run_reporter(stc, argv[1] ?: ALARM_CONF_FILE); + } + else + usage(); + return 0; +} diff --git a/src/search_local/index_storage/watchdog/Makefile b/src/search_local/index_storage/watchdog/Makefile new file mode 100644 index 0000000..8cb0fb1 --- /dev/null +++ b/src/search_local/index_storage/watchdog/Makefile @@ -0,0 +1,13 @@ +include ../Make.conf + +target = libwatchdog.a + +CFLAGS+=-I../common -I../stat +VPATH := ../stat +filelist := watchdog watchdog_daemon watchdog_fork watchdog_helper watchdog_listener watchdog_main watchdog_run watchdog_stattool watchdog_unit watchdog_logger fault gdb gdb_srv stat_alarm_reporter +libwatchdog_objs := $(filelist:%=%.o) + +install_dir = ../../bin + +include ../Make.rules + diff --git a/src/search_local/index_storage/watchdog/fault.cc b/src/search_local/index_storage/watchdog/fault.cc new file mode 100644 index 0000000..e1f4816 --- /dev/null +++ b/src/search_local/index_storage/watchdog/fault.cc @@ -0,0 +1,107 @@ +/* + * ===================================================================================== + * + * Filename: fault.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include +#include + +#include "compiler.h" +#include "fault.h" + +__HIDDEN +int FaultHandler::DogPid = 0; + +__HIDDEN +FaultHandler FaultHandler::Instance; + +extern "C" + __attribute__((__weak__)) void + crash_hook(int signo); + +extern "C" +{ + __EXPORT volatile int crash_continue; +}; + +static int __crash_hook = 0; + +__HIDDEN +void FaultHandler::Handler(int signo, siginfo_t *dummy, void *dummy2) +{ + signal(signo, SIG_DFL); + if (DogPid > 1) + { + sigval tid; + tid.sival_int = syscall(__NR_gettid); + sigqueue(DogPid, SIGWINCH, tid); + for (int i = 0; i < 50 && !crash_continue; i++) + usleep(100 * 1000); + if ((__crash_hook != 0) && (&crash_hook != 0)) + crash_hook(signo); + } +} + +__HIDDEN +FaultHandler::FaultHandler(void) +{ + Initialize(1); +} + +__HIDDEN +void FaultHandler::Initialize(int protect) +{ + char *p = getenv(ENV_FAULT_LOGGER_PID); + if (p && (DogPid = atoi(p)) > 1) + { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = FaultHandler::Handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + sigaction(SIGSEGV, &sa, NULL); + sigaction(SIGBUS, &sa, NULL); + sigaction(SIGILL, &sa, NULL); + sigaction(SIGABRT, &sa, NULL); + sigaction(SIGFPE, &sa, NULL); + if (protect) + __crash_hook = 1; + } +} + +#if __pic__ || __PIC__ +extern "C" const char __invoke_dynamic_linker__[] + __attribute__((section(".interp"))) + __HIDDEN = +#if __x86_64__ + "/lib64/ld-linux-x86-64.so.2" +#else + "/lib/ld-linux.so.2" +#endif + ; + +extern "C" __HIDDEN int _so_start(void) +{ +#define BANNER "DTC FaultLogger v" DTC_VERSION_DETAIL "\n" \ + " - USED BY DTCD INTERNAL ONLY!!!\n" + int unused; + unused = write(1, BANNER, sizeof(BANNER) - 1); + _exit(0); +} +#endif diff --git a/src/search_local/index_storage/watchdog/fault.h b/src/search_local/index_storage/watchdog/fault.h new file mode 100644 index 0000000..386ac15 --- /dev/null +++ b/src/search_local/index_storage/watchdog/fault.h @@ -0,0 +1,31 @@ +/* + * ===================================================================================== + * + * Filename: fault.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#define ENV_FAULT_LOGGER_PID "__FAULT_LOGGER_PID" +class FaultHandler +{ +private: + static void Handler(int, siginfo_t *, void *); + static FaultHandler Instance; + static int DogPid; + FaultHandler(void); + +public: + static void Initialize(int); +}; diff --git a/src/search_local/index_storage/watchdog/gdb.cc b/src/search_local/index_storage/watchdog/gdb.cc new file mode 100644 index 0000000..e2ad59e --- /dev/null +++ b/src/search_local/index_storage/watchdog/gdb.cc @@ -0,0 +1,218 @@ +/* + * ===================================================================================== + * + * Filename: gdb.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "gdb.h" +#include "log.h" + +#define printf(fmt, args...) log_bare(2, fmt, ##args) + +struct frame_t +{ + int index; + int ftype; // 0 normal, 1 *this, 2 signal +}; + +static void wait_prompt(FILE *outfp) +{ + char buf[1024]; + while (fgets(buf, sizeof(buf) - 1, outfp)) + { + if (!strncmp(buf, "GDB>", 4)) + break; + } +} + +static void dump_result(FILE *outfp) +{ + char buf[1024]; + while (fgets(buf, sizeof(buf) - 1, outfp)) + { + if (!strncmp(buf, "GDB>", 4)) + break; + printf("%s", buf); + } +} + +static void parse_frame(FILE *outfp, std::vector &frame) +{ + char buf[1024]; + int first = 1; + while (fgets(buf, sizeof(buf) - 1, outfp)) + { + if (!strncmp(buf, "GDB>", 4)) + break; + printf("%s", buf); + if (buf[0] == '#') + { + struct frame_t f; + char *p = strstr(buf, "this"); + f.index = atoi(buf + 1); + f.ftype = p && (!isalnum(p[-1] && !isalnum(p[4]))); + if (first && strstr(buf, "signal handler called") != NULL) + { + first = 0; + frame.clear(); + f.ftype = 2; + } + frame.push_back(f); + } + } +} + +static void dump_info(int pid) +{ + char fn[64]; + char buf[1024]; + int rv; + + printf("Process/Thread %d Crashed\n", pid); + snprintf(fn, sizeof(fn), "/proc/%d/exe", pid); + rv = readlink(fn, buf, sizeof(buf) - 1); + if (rv > 0) + { + buf[rv] = 0; + printf("Executable: %s\n", buf); + } + + snprintf(fn, sizeof(fn), "/proc/%d/cwd", pid); + rv = readlink(fn, buf, sizeof(buf) - 1); + if (rv > 0) + { + buf[rv] = 0; + printf("Working Directory: %s\n", buf); + } + + snprintf(fn, sizeof(fn), "/proc/%d/cmdline", pid); + FILE *fp = fopen(fn, "r"); + if (fp) + { + if (fgets(buf, sizeof(buf), fp)) + printf("Command Line: %s\n", buf); + fclose(fp); + } + + snprintf(fn, sizeof(fn), "/proc/%d/maps", pid); + fp = fopen(fn, "r"); + if (fp) + { + printf("Dump memory maps:\n"); + while (fgets(buf, sizeof(buf), fp)) + { + printf(" %s", buf); + } + fclose(fp); + } +} + +void gdb_dump(int pid) +{ + std::vector frame; + int pcmd[2]; + int pout[2]; + int ret; + + ret = pipe(pcmd); + ret = pipe(pout); + + if (fork() == 0) + { + dup2(pcmd[0], 0); + dup2(pout[1], 1); + dup2(pout[1], 2); + close(pcmd[0]); + close(pcmd[1]); + close(pout[0]); + close(pout[1]); + execlp("gdb", "gdb", NULL); + exit(1); + } + + close(pcmd[0]); + close(pout[1]); + + //always succ because the fd is valid + FILE *cmdfp = fdopen(pcmd[1], "w"); + FILE *outfp = fdopen(pout[0], "r"); + + setbuf(cmdfp, NULL); + setbuf(outfp, NULL); + + fprintf(cmdfp, "set prompt GDB>\\n\n\n"); + fflush(cmdfp); + wait_prompt(outfp); + fprintf(cmdfp, "attach %d\n", pid); + dump_info(pid); + dump_result(outfp); + printf("(gdb) backtrace\n"); + fprintf(cmdfp, "backtrace\n"); + parse_frame(outfp, frame); + +#define DUMP(fmt, args...) \ + do \ + { \ + printf("(gdb) " fmt "\n", ##args); \ + fprintf(cmdfp, fmt "\n", ##args); \ + dump_result(outfp); \ + } while (0) + + for (unsigned int i = 0; i < frame.size(); i++) + { + DUMP("frame %d", frame[i].index); + if (frame[i].ftype <= 1) + DUMP("info locals"); + if (frame[i].ftype == 1) + DUMP("print *this"); + if (frame[i].ftype == 2) + DUMP("info registers"); + } + + fprintf(cmdfp, "set variable crash_continue = 1\n"); + dump_result(outfp); + fprintf(cmdfp, "quit\n"); + fclose(cmdfp); + fclose(outfp); +} + +void gdb_attach(int pid, const char *fn) +{ + int ret; + if (fn == NULL) + fn = getenv("DISPLAY"); + char buf[256]; + if (fn == NULL || !fn[0] || !strcmp(fn, "screen")) + { + snprintf(buf, sizeof(buf), "screen -X -S gdb screen -t gdb.%d gdb /proc/%d/exe %d", pid, pid, pid); + ret = system(buf); + if (ret != 0) + { + snprintf(buf, sizeof(buf), "screen -S gdb -t gdb.%d -d -m gdb /proc/%d/exe %d", pid, pid, pid); + ret = system(buf); + } + } + else + { + snprintf(buf, sizeof(buf), "xterm -T gdb.%d -e gdb /proc/%d/exe %d &", pid, pid, pid); + ret = system(buf); + } + dump_info(pid); +} diff --git a/src/search_local/index_storage/watchdog/gdb.h b/src/search_local/index_storage/watchdog/gdb.h new file mode 100644 index 0000000..6e5163d --- /dev/null +++ b/src/search_local/index_storage/watchdog/gdb.h @@ -0,0 +1,22 @@ +/* + * ===================================================================================== + * + * Filename: gdb.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +void gdb_dump(int); +void gdb_attach(int, const char *); +void gdb_server(int, const char *); diff --git a/src/search_local/index_storage/watchdog/gdb_srv.cc b/src/search_local/index_storage/watchdog/gdb_srv.cc new file mode 100644 index 0000000..eedded9 --- /dev/null +++ b/src/search_local/index_storage/watchdog/gdb_srv.cc @@ -0,0 +1,76 @@ +/* + * ===================================================================================== + * + * Filename: gdb_srv.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include + +#include "gdb.h" + +static volatile int stop; +static void sigusr1(int signo) {} +static void sigstop(int signo) { stop = 1; } + +void gdb_server(int debug, const char *display) +{ + signal(SIGINT, sigstop); + signal(SIGTERM, sigstop); + signal(SIGQUIT, sigstop); + signal(SIGHUP, sigstop); + signal(SIGPIPE, SIG_IGN); + signal(SIGCHLD, SIG_DFL); + + signal(SIGWINCH, sigusr1); + signal(SIGPIPE, SIG_IGN); + signal(SIGCHLD, SIG_IGN); + + sigset_t sset; + + sigemptyset(&sset); + sigaddset(&sset, SIGWINCH); + sigprocmask(SIG_BLOCK, &sset, NULL); + + const struct timespec timeout = {5, 0}; + siginfo_t info; + while (!stop) + { + if (getppid() == 1) + break; + int signo = sigtimedwait(&sset, &info, &timeout); + if (signo == -1 && errno == -EAGAIN) + continue; + if (signo <= 0) + { + usleep(100 * 1000); + continue; + } + + if (info.si_code != SI_QUEUE) + continue; + + int tid = info.si_int; + if (debug == 0) + { + gdb_dump(tid); + } + else + { + gdb_attach(tid, display); + } + } +} diff --git a/src/search_local/index_storage/watchdog/watchdog.cc b/src/search_local/index_storage/watchdog/watchdog.cc new file mode 100644 index 0000000..becf450 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog.cc @@ -0,0 +1,134 @@ +/* + * ===================================================================================== + * + * Filename: watchdog.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include + +#include "watchdog.h" +#include "daemon.h" +#include "log.h" + +WatchDogPipe::WatchDogPipe(void) +{ + int fd[2]; + int unused; + + unused = pipe(fd); + netfd = fd[0]; + peerfd = fd[1]; + fcntl(netfd, F_SETFL, O_NONBLOCK); + fcntl(peerfd, F_SETFL, O_NONBLOCK); + fcntl(netfd, F_SETFD, FD_CLOEXEC); + fcntl(peerfd, F_SETFD, FD_CLOEXEC); + enable_input(); +} + +WatchDogPipe::~WatchDogPipe(void) +{ + if (peerfd >= 0) + close(peerfd); +} + +void WatchDogPipe::input_notify(void) +{ + char buf[100]; + while (read(netfd, buf, sizeof(buf)) == sizeof(buf)) + ; +} + +void WatchDogPipe::Wake(void) +{ + char c = 0; + c = write(peerfd, &c, 1); +} + +static WatchDogPipe *notifier; +static void sighdlr(int signo) { notifier->Wake(); } + +WatchDog::WatchDog(void) +{ + //立马注册进程退出处理函数，解决启动时创建太多进程导致部分进程退出没有收到信号linjinming 2014-06-14 + notifier = new WatchDogPipe; + signal(SIGCHLD, sighdlr); +} + +WatchDog::~WatchDog(void) +{ +} + +void WatchDog::run_loop(void) +{ + struct pollfd pfd[2]; + notifier->init_poll_fd(&pfd[0]); + + if (listener) + { + listener->init_poll_fd(&pfd[1]); + } + else + { + pfd[1].fd = -1; + pfd[1].events = 0; + pfd[1].revents = 0; + } + + while (!stop) + { + int timeout = expire_micro_seconds(3600 * 1000, 1 /*nonzero*/); + int interrupted = poll(pfd, 2, timeout); + update_now_time(timeout, interrupted); + if (stop) + break; + + if (pfd[0].revents & POLLIN) + notifier->input_notify(); + if (pfd[1].revents & POLLIN) + listener->input_notify(); + + CheckWatchDog(); + check_expired(); + check_ready(); + } + + log_debug("prepare stopping"); + KillAll(); + CheckWatchDog(); + + time_t stoptimer = time(NULL) + 5; + int stopretry = 0; + while (stopretry < 6 && ProcessCount() > 0) + { + time_t now = time(NULL); + if (stoptimer <= now) + { + stopretry++; + stoptimer = now + 5; + log_debug("notify all children again"); + KillAll(); + } + poll(pfd, 1, 1000); + if (pfd[0].revents & POLLIN) + notifier->input_notify(); + CheckWatchDog(); + } + + delete notifier; + ForceKillAll(); + log_info("all children stopped, watchdog ended"); + exit(0); +} diff --git a/src/search_local/index_storage/watchdog/watchdog.h b/src/search_local/index_storage/watchdog/watchdog.h new file mode 100644 index 0000000..a2ee287 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog.h @@ -0,0 +1,52 @@ +/* + * ===================================================================================== + * + * Filename: watchdog.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG__H__ +#define __H_WATCHDOG__H__ + +#include "watchdog_unit.h" +#include "poller.h" +#include "timer_list.h" + +class WatchDogPipe : public PollerObject +{ +private: + int peerfd; + +public: + WatchDogPipe(void); + virtual ~WatchDogPipe(void); + void Wake(void); + virtual void input_notify(void); +}; + +class WatchDog : public WatchDogUnit, + public TimerUnit +{ +private: + PollerObject *listener; + +public: + WatchDog(void); + virtual ~WatchDog(void); + + void set_listener(PollerObject *l) { listener = l; } + void run_loop(void); +}; + +extern int start_watch_dog(int (*entry)(void *), void *); +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_daemon.cc b/src/search_local/index_storage/watchdog/watchdog_daemon.cc new file mode 100644 index 0000000..69d4ee7 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_daemon.cc @@ -0,0 +1,91 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_daemon.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "watchdog_daemon.h" +#include "log.h" + +WatchDogDaemon::WatchDogDaemon(WatchDog *o, int sec) + : WatchDogObject(o) +{ + if (o) + timerList = o->get_timer_list(sec); +} + +WatchDogDaemon::~WatchDogDaemon(void) +{ +} + +int WatchDogDaemon::Fork(void) +{ + // an error detection pipe + int err, fd[2]; + int unused; + + unused = pipe(fd); + + // fork child process + pid = fork(); + if (pid == -1) + return pid; + + if (pid == 0) + { + // close pipe if exec succ + close(fd[0]); + fcntl(fd[1], F_SETFD, FD_CLOEXEC); + Exec(); + err = errno; + log_error("%s: exec(): %m", name); + unused = write(fd[1], &err, sizeof(err)); + exit(-1); + } + + close(fd[1]); + + if (read(fd[0], &err, sizeof(err)) == sizeof(err)) + { + errno = err; + return -1; + } + close(fd[0]); + + attach_watch_dog(); + return pid; +} + +void WatchDogDaemon::timer_notify(void) +{ + if (Fork() < 0) + if (timerList) + attach_timer(timerList); +} + +void WatchDogDaemon::killed_notify(int sig, int cd) +{ + if (timerList) + attach_timer(timerList); + report_kill_alarm(sig, cd); +} + +void WatchDogDaemon::exited_notify(int retval) +{ + if (timerList) + attach_timer(timerList); +} diff --git a/src/search_local/index_storage/watchdog/watchdog_daemon.h b/src/search_local/index_storage/watchdog/watchdog_daemon.h new file mode 100644 index 0000000..a3b023b --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_daemon.h @@ -0,0 +1,42 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_daemon.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_DAEMON_H__ +#define __H_WATCHDOG_DAEMON_H__ + +#include "watchdog.h" + +class WatchDogDaemon : public WatchDogObject, + private TimerObject +{ +private: + TimerList *timerList; + +private: + virtual void killed_notify(int, int); + virtual void exited_notify(int); + virtual void timer_notify(void); + +public: + WatchDogDaemon(WatchDog *o, int sec); + ~WatchDogDaemon(void); + + virtual int Fork(void); + virtual void Exec(void) = 0; +}; + +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_fork.cc b/src/search_local/index_storage/watchdog/watchdog_fork.cc new file mode 100644 index 0000000..8dbf4aa --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_fork.cc @@ -0,0 +1,77 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_fork.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include +#include + +#ifndef CLONE_PARENT +#define CLONE_PARENT 0x00008000 +#endif + +#include "watchdog_listener.h" +#include "log.h" + +struct ForkInfo +{ + char name[16]; + int fd; + int (*entry)(void *); + void *args; +}; + +static int CloneEntry(void *arg) +{ + struct ForkInfo *info = (ForkInfo *)arg; + + send(info->fd, info->name, sizeof(info->name), 0); + //close(info->fd); + exit(info->entry(info->args)); +} + +int watch_dog_fork(const char *name, int (*entry)(void *), void *args) +{ + char stack[4096]; // 4K stack is enough for CloneEntry + struct ForkInfo info; + char *env = getenv(ENV_WATCHDOG_SOCKET_FD); + + info.fd = env == NULL ? -1 : atoi(env); + log_debug("watchdog fork, fd: %d, name: %s, args: %p", info.fd, name, args); + if (info.fd <= 0) + { + int pid = fork(); + if (pid == 0) + { + exit(entry(args)); + } + return pid; + } + + strncpy(info.name, name, sizeof(info.name)); + info.entry = entry; + info.args = args; + + return clone( + CloneEntry, // entry + stack + sizeof(stack) - 16, + CLONE_PARENT | SIGCHLD, // flag + &info // data + ); +}; diff --git a/src/search_local/index_storage/watchdog/watchdog_helper.cc b/src/search_local/index_storage/watchdog/watchdog_helper.cc new file mode 100644 index 0000000..4412bcf --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_helper.cc @@ -0,0 +1,119 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_helper.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include "unix_socket.h" + +#include "dbconfig.h" +#include "thread.h" +#include "watchdog_helper.h" +#include "daemon.h" +#include "log.h" +#include "dtc_global.h" +#include + +WatchDogHelper::WatchDogHelper(WatchDog *o, int sec, const char *p, int g, int r, int b, int t, int c, int n) + : WatchDogDaemon(o, sec) +{ + /*snprintf(name, 20, "helper%u%c", g, MACHINEROLESTRING[r]);*/ + std::stringstream oss; + oss << "helper" << g << MACHINEROLESTRING[r]; + memcpy(name, oss.str().c_str(), oss.str().length()); + path = p; + backlog = b; + type = t; + conf = c; + num = n; +} + +WatchDogHelper::~WatchDogHelper(void) +{ +} + +const char *HelperName[] = + { + NULL, + NULL, + "rocksdb_helper", + "tdb_helper", + "custom_helper", +}; + +void WatchDogHelper::Exec(void) +{ + struct sockaddr_un unaddr; + int len = init_unix_socket_address(&unaddr, path); + int listenfd = socket(unaddr.sun_family, SOCK_STREAM, 0); + bind(listenfd, (sockaddr *)&unaddr, len); + listen(listenfd, backlog); + + // relocate listenfd to stdin + dup2(listenfd, 0); + close(listenfd); + + char *argv[9]; + int argc = 0; + + argv[argc++] = NULL; + argv[argc++] = (char *)"-d"; + if (strcmp(cacheFile, CACHE_CONF_NAME)) + { + argv[argc++] = (char *)"-f"; + argv[argc++] = cacheFile; + } + if (conf == DBHELPER_TABLE_NEW) + { + argv[argc++] = (char *)"-t"; + char tableName[64]; + snprintf(tableName, 64, "../conf/table%d.conf", num); + //argv[argc++] = (char *)"../conf/table2.conf"; + argv[argc++] = tableName; + } + else if (conf == DBHELPER_TABLE_ORIGIN && strcmp(tableFile, TABLE_CONF_NAME)) + { + argv[argc++] = (char *)"-t"; + argv[argc++] = tableFile; + } + argv[argc++] = name + 6; + argv[argc++] = (char *)"-"; + argv[argc++] = NULL; + + Thread *helperThread = new Thread(name, Thread::ThreadTypeProcess); + helperThread->initialize_thread(); + argv[0] = (char *)HelperName[type]; + execv(argv[0], argv); + log_error("helper[%s] execv error: %m", argv[0]); +} + +int WatchDogHelper::Verify(void) +{ + struct sockaddr_un unaddr; + int len = init_unix_socket_address(&unaddr, path); + + // delay 100ms and verify socket + usleep(100 * 1000); + int s = socket(unaddr.sun_family, SOCK_STREAM, 0); + if (connect(s, (sockaddr *)&unaddr, len) < 0) + { + close(s); + log_error("verify connect: %m"); + return -1; + } + close(s); + + return pid; +} diff --git a/src/search_local/index_storage/watchdog/watchdog_helper.h b/src/search_local/index_storage/watchdog/watchdog_helper.h new file mode 100644 index 0000000..bc18834 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_helper.h @@ -0,0 +1,40 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_helper.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_HELPER_H__ +#define __H_WATCHDOG_HELPER_H__ + +#include "watchdog_daemon.h" +#include "watchdog_listener.h" + +class WatchDogHelper : public WatchDogDaemon +{ +private: + const char *path; + int backlog; + int type; + int conf; + int num; + +public: + WatchDogHelper(WatchDog *o, int sec, const char *p, int, int, int, int t, int c = DBHELPER_TABLE_ORIGIN, int n = -1); + virtual ~WatchDogHelper(void); + virtual void Exec(void); + int Verify(void); +}; + +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_listener.cc b/src/search_local/index_storage/watchdog/watchdog_listener.cc new file mode 100644 index 0000000..bb6159d --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_listener.cc @@ -0,0 +1,129 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_listener.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include +#include +#include +#include + +#include "watchdog_listener.h" +#include "watchdog_helper.h" +#include "dbconfig.h" +#include "log.h" + +WatchDogListener::WatchDogListener(WatchDog *o, int sec) : owner(o), peerfd(-1), delay(sec){}; + +WatchDogListener::~WatchDogListener(void) +{ + if (peerfd > 0) + { + log_debug("watchdog listener fd: %d closed", peerfd); + close(peerfd); + } +}; + +int WatchDogListener::attach_watch_dog(void) +{ + int fd[2]; + socketpair(AF_UNIX, SOCK_DGRAM, 0, fd); + netfd = fd[0]; + peerfd = fd[1]; + + char buf[30]; + snprintf(buf, sizeof(buf), "%d", peerfd); + setenv(ENV_WATCHDOG_SOCKET_FD, buf, 1); + + int no; + setsockopt(netfd, SOL_SOCKET, SO_PASSCRED, (no = 1, &no), sizeof(int)); + fcntl(netfd, F_SETFD, FD_CLOEXEC); + enable_input(); + owner->set_listener(this); + return 0; +} + +void WatchDogListener::input_notify(void) +{ + char buf[16]; + int n; + struct msghdr msg = {0}; + char ucbuf[CMSG_SPACE(sizeof(struct ucred))]; + struct iovec iov[1]; + + iov[0].iov_base = (void *)&buf; + iov[0].iov_len = sizeof(buf); + msg.msg_control = ucbuf; + msg.msg_controllen = sizeof ucbuf; + msg.msg_iov = iov; + msg.msg_iovlen = 1; + msg.msg_flags = 0; + + while ((n = recvmsg(netfd, &msg, MSG_TRUNC | MSG_DONTWAIT)) > 0) + { + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + struct ucred *uc; + if (msg.msg_controllen < sizeof(ucbuf) || + cmsg->cmsg_level != SOL_SOCKET || + cmsg->cmsg_type != SCM_CREDENTIALS || + cmsg->cmsg_len != CMSG_LEN(sizeof(struct ucred)) || + msg.msg_controllen < cmsg->cmsg_len) + continue; + uc = (struct ucred *)CMSG_DATA(cmsg); + if (n != sizeof(buf)) + continue; + + log_debug("new watchdog object: %p, %d, %d, %s", owner, buf[0], uc->pid, buf + 1); + WatchDogObject *obj = NULL; + if (buf[0] == WATCHDOG_INPUT_OBJECT) + { + obj = new WatchDogObject(owner, buf, uc->pid); + if (obj == NULL) + { + log_error("new WatchDogObject error"); + return; + } + } + else if (buf[0] == WATCHDOG_INPUT_HELPER) + { + StartHelperPara *para = (StartHelperPara *)(buf + 1); + char path[32]; + if (!DbConfig::build_path(path, 32, getpid(), para->mach, para->role, para->type)) + { + log_error("build helper listen path error"); + return; + } + NEW(WatchDogHelper(owner, delay, path, para->mach, para->role, para->backlog, para->type, para->conf, para->num), obj); + if (obj == NULL) + { + log_error("new watchdog helper error"); + return; + } + WatchDogHelper *helper = (WatchDogHelper *)obj; + if (helper->Fork() < 0 || helper->Verify() < 0) + { + log_error("fork helper error"); + return; + } + } + else + { + log_error("unknown watchdog input type: %d, %s", buf[0], buf + 1); + return; + } + obj->attach_watch_dog(); + } +} diff --git a/src/search_local/index_storage/watchdog/watchdog_listener.h b/src/search_local/index_storage/watchdog/watchdog_listener.h new file mode 100644 index 0000000..3307d9e --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_listener.h @@ -0,0 +1,56 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_listener.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_LISTENER_H__ +#define __H_WATCHDOG_LISTENER_H__ + +#include "watchdog.h" + +#define ENV_WATCHDOG_SOCKET_FD "WATCHDOG_SOCKET_FD" + +#define WATCHDOG_INPUT_OBJECT 0 +#define WATCHDOG_INPUT_HELPER 1 + +#define DBHELPER_TABLE_ORIGIN 0 +#define DBHELPER_TABLE_NEW 1 + +struct StartHelperPara +{ + uint8_t type; + uint8_t mach; + uint8_t role; + uint8_t conf; + uint16_t num; + uint16_t backlog; +}; + +class WatchDogListener : public PollerObject +{ +private: + WatchDog *owner; + int peerfd; + int delay; + +public: + WatchDogListener(WatchDog *o, int sec); + virtual ~WatchDogListener(void); + int attach_watch_dog(void); + virtual void input_notify(void); +}; + +extern int watch_dog_fork(const char *name, int (*entry)(void *), void *args); +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_logger.cc b/src/search_local/index_storage/watchdog/watchdog_logger.cc new file mode 100644 index 0000000..f013f3f --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_logger.cc @@ -0,0 +1,142 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_logger.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "daemon.h" +#include "proc_title.h" +#include "config.h" +#include "gdb.h" +#include "log.h" +#include "watchdog.h" +#include "watchdog_unit.h" +#include "fault.h" +#include "thread.h" + +#define HOOKSO "../bin/faultlogger" + +static void Set_LD_PRELOAD(void) +{ + if (access(HOOKSO, R_OK) == 0) + { + char *preload = canonicalize_file_name(HOOKSO); + char *p = getenv("LD_PRELOAD"); + if (p == NULL) + { + setenv("LD_PRELOAD", preload, 1); + } + else + { + char *concat = NULL; + int unused; + unused = asprintf(&concat, "%s:%s", p, preload); + setenv("LD_PRELOAD", concat, 1); + } + } +} + +int start_fault_logger(WatchDog *wdog) +{ + // CarshProtect/CrashLog mode: + // 0 -- disabled + // 1 -- log-only + // 2 -- protect + // 3 -- screen + // 4 -- xterm + int mode = 2; // default protect + const char *display; + + display = gConfig->get_str_val("cache", "FaultLoggerMode"); + if (display == NULL || !display[0]) + { + mode = 2; // protect + } + else if (!strcmp(display, "log")) + { + mode = 1; // log + } + else if (!strcmp(display, "dump")) + { + mode = 1; // log + } + else if (!strcmp(display, "protect")) + { + mode = 2; // protect + } + else if (!strcmp(display, "screen")) + { + mode = 3; // screen + } + else if (!strcmp(display, "xterm")) + { + if (getenv("DISPLAY")) + { + mode = 4; // xterm + display = NULL; + } + else + { + log_warning("FaultLoggerTarget set to \"xterm\", but no DISPLAY found"); + mode = 2; // screen + } + } + else if (!strncmp(display, "xterm:", 6)) + { + mode = 4; // xterm + display += 6; + } + else if (!strcasecmp(display, "disable")) + { + mode = 0; + } + else if (!strcasecmp(display, "disabled")) + { + mode = 0; + } + else + { + log_warning("unknown FaultLoggerMode \"%s\"", display); + } + + log_info("FaultLoggerMode is %s\n", ((const char *[]){"disable", "log", "protect", "screen", "xterm"})[mode]); + if (mode == 0) + return 0; + + int pid = fork(); + + if (pid < 0) + return -1; + + if (pid == 0) + { + set_proc_title("FaultLogger"); + Thread *loggerThread = new Thread("faultlogger", Thread::ThreadTypeProcess); + loggerThread->initialize_thread(); + gdb_server(mode >= 3 /*debug*/, display); + exit(0); + } + + char buf[20]; + snprintf(buf, sizeof(buf), "%d", pid); + setenv(ENV_FAULT_LOGGER_PID, buf, 1); + FaultHandler::Initialize(mode >= 2); + Set_LD_PRELOAD(); + + WatchDogObject *obj = new WatchDogObject(wdog, "FaultLogger", pid); + obj->attach_watch_dog(); + return 0; +} diff --git a/src/search_local/index_storage/watchdog/watchdog_logger.h b/src/search_local/index_storage/watchdog/watchdog_logger.h new file mode 100644 index 0000000..7f96a02 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_logger.h @@ -0,0 +1,20 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_logger.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ + +class WatchDog; +int start_fault_logger(WatchDog *); diff --git a/src/search_local/index_storage/watchdog/watchdog_main.cc b/src/search_local/index_storage/watchdog/watchdog_main.cc new file mode 100644 index 0000000..dd1c0ce --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_main.cc @@ -0,0 +1,106 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_main.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "watchdog_main.h" +#include "daemon.h" +#include "log.h" +#include "proc_title.h" +extern void _set_log_thread_name_(const char *n); +/* + * recovery value: + * 0 -- don't refork + * 1 -- fork again if crashed + * 2 -- fork & enable core dump if crashed + * 3 -- fork if killed + * 4 -- fork again if exit != 0 + * 5 -- always fork again + */ +static const char *RecoveryInfoStr[] = { + "None", "Crash", "CrashDebug", "Killed", "Error", "Always"}; + +WatchDogEntry::WatchDogEntry(WatchDogUnit *o, int (*e)(void *), void *a, int r) + : WatchDogObject(o), entry(e), args(a), recovery(r), corecount(0) +{ + strncpy(name, "main", sizeof(name)); + _set_log_thread_name_("wdog"); + _set_remote_log_fd_(); + if (recovery > 3) + { + char comm[24] = "wdog-"; + get_proc_name(comm + 5); + set_proc_name(comm); + } + log_info("Main Process Recovery Mode: %s", RecoveryInfoStr[recovery]); +} + +WatchDogEntry::~WatchDogEntry(void) +{ +} + +int WatchDogEntry::Fork(int en) +{ + if ((pid = fork()) == 0) + { + if (en) + daemon_enable_core_dump(); + exit(entry(args)); + } + + if (pid < 0) + return -1; // cann't fork main process + + attach_watch_dog(); + return pid; +} + +void WatchDogEntry::killed_notify(int sig, int coredump) +{ + int corenable = 0; + + if (recovery == 0 || ((recovery <= 2 && sig == SIGKILL) || sig == SIGTERM)) + // main cache exited, stopping all children + stop = 1; + else + { + ++corecount; + if (corecount == 1 && recovery >= 2) + corenable = 1; + + sleep(2); + if (Fork(corenable) < 0) + // main fork error, stopping all children + stop = 1; + } + report_kill_alarm(sig, coredump); +} + +void WatchDogEntry::exited_notify(int retval) +{ + if (recovery < 4 || retval == 0) + // main cache exited, stopping all children + stop = 1; + else + { + sleep(2); // sync sleep + if (Fork() < 0) + // main fork error, stopping all children + stop = 1; + } +} diff --git a/src/search_local/index_storage/watchdog/watchdog_main.h b/src/search_local/index_storage/watchdog/watchdog_main.h new file mode 100644 index 0000000..12cf87f --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_main.h @@ -0,0 +1,42 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_main.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_MAIN_H__ +#define __H_WATCHDOG_MAIN_H__ + +#include "watchdog.h" + +class WatchDogEntry : private WatchDogObject +{ +private: + int (*entry)(void *); + void *args; + int recovery; + int corecount; + +private: + virtual void killed_notify(int sig, int cd); + virtual void exited_notify(int rv); + +public: + WatchDogEntry(WatchDogUnit *o, int (*entry)(void *), void *args, int r); + virtual ~WatchDogEntry(void); + + int Fork(int enCoreDump = 0); +}; + +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_run.cc b/src/search_local/index_storage/watchdog/watchdog_run.cc new file mode 100644 index 0000000..070a8f7 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_run.cc @@ -0,0 +1,134 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_run.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "watchdog.h" +#include "watchdog_main.h" +#include "watchdog_stattool.h" +#include "watchdog_listener.h" +#include "watchdog_helper.h" +#include "watchdog_logger.h" +#include "dbconfig.h" +#include "log.h" +#include "daemon.h" + +int start_watch_dog(int (*entry)(void *), void *args) +{ + int delay = 5; + dbConfig->set_helper_path(getpid()); + WatchDog *wdog = NULL; + WatchDogListener *srv = NULL; + + if (gConfig->get_int_val("cache", "DisableWatchDog", 0) == 0) + { + signal(SIGCHLD, SIG_DFL); + + delay = gConfig->get_int_val("cache", "WatchDogTime", 30); + if (delay < 5) + { + delay = 5; + } + + wdog = new WatchDog; + srv = new WatchDogListener(wdog, delay); + srv->attach_watch_dog(); + + start_fault_logger(wdog); + + if (gConfig->get_int_val("cache", "StartStatReporter", 0) > 0) + { + WatchDogStatTool *st = new WatchDogStatTool(wdog, delay); + if (st->Fork() < 0) + return -1; // cann't fork reporter + log_info("fork stat reporter"); + } + } + + if (gConfig->get_int_val("cache", "DisableDataSource", 0) == 0) + { + int nh = 0; + /* starting master helper */ + for (int g = 0; g < dbConfig->machineCnt; g++) + { + for (int r = 0; r < ROLES_PER_MACHINE; r++) + { + HELPERTYPE t = dbConfig->mach[g].helperType; + + log_debug("helper type = %d", t); + + //check helper type is dtc + if (DTC_HELPER >= t) + { + break; + } + + int i, n = 0; + for (i = 0; i < GROUPS_PER_ROLE && (r * GROUPS_PER_ROLE + i) < GROUPS_PER_MACHINE; i++) + { + n += dbConfig->mach[g].gprocs[r * GROUPS_PER_ROLE + i]; + } + + if (n <= 0) + { + continue; + } + + WatchDogHelper *h = NULL; + NEW(WatchDogHelper(wdog, delay, dbConfig->mach[g].role[r].path, g, r, n + 1, t), h); + if (NULL == h) + { + log_error("create WatchDogHelper object failed, msg:%m"); + return -1; + } + + if (h->Fork() < 0 || h->Verify() < 0) + { + return -1; + } + + nh++; + } + } + + log_info("fork %d helper groups", nh); + } + + if (wdog) + { + const int recovery = gConfig->get_idx_val("cache", "ServerRecovery", + ((const char *const[]){ + "none", + "crash", + "crashdebug", + "killed", + "error", + "always", + NULL}), + 2); + + WatchDogEntry *dtc = new WatchDogEntry(wdog, entry, args, recovery); + + if (dtc->Fork() < 0) + return -1; + + wdog->run_loop(); + exit(0); + } + + return 0; +} diff --git a/src/search_local/index_storage/watchdog/watchdog_stattool.cc b/src/search_local/index_storage/watchdog/watchdog_stattool.cc new file mode 100644 index 0000000..7591507 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_stattool.cc @@ -0,0 +1,47 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_statool.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include + +#include "watchdog_stattool.h" + +WatchDogStatTool::WatchDogStatTool(WatchDog *o, int sec) + : WatchDogDaemon(o, sec) +{ + strncpy(name, "stattool", sizeof(name)); +} + +WatchDogStatTool::~WatchDogStatTool(void) +{ +} + +void WatchDogStatTool::Exec(void) +{ + char *argv[3]; + + argv[1] = (char *)"reporter"; + argv[2] = NULL; + + argv[0] = (char *)"stattool32"; + execv(argv[0], argv); + argv[0] = (char *)"../bin/stattool32"; + execv(argv[0], argv); + argv[0] = (char *)"stattool"; + execv(argv[0], argv); + argv[0] = (char *)"../bin/stattool"; + execv(argv[0], argv); +} diff --git a/src/search_local/index_storage/watchdog/watchdog_stattool.h b/src/search_local/index_storage/watchdog/watchdog_stattool.h new file mode 100644 index 0000000..22e876e --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_stattool.h @@ -0,0 +1,31 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_statool.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_STATTOOL_H__ +#define __H_WATCHDOG_STATTOOL_H__ + +#include "watchdog_daemon.h" + +class WatchDogStatTool : public WatchDogDaemon +{ +public: + WatchDogStatTool(WatchDog *o, int sec); + virtual ~WatchDogStatTool(void); + virtual void Exec(void); +}; + +#endif diff --git a/src/search_local/index_storage/watchdog/watchdog_unit.cc b/src/search_local/index_storage/watchdog/watchdog_unit.cc new file mode 100644 index 0000000..f80045e --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_unit.cc @@ -0,0 +1,202 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_unit.cc + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#include +#include + +#include "config.h" +#include "watchdog_unit.h" +#include "log.h" +#include "stat_alarm_reporter.h" +#include +#include + +WatchDogObject::~WatchDogObject(void) +{ +} + +int WatchDogObject::attach_watch_dog(WatchDogUnit *u) +{ + if (u && owner == NULL) + owner = u; + + return owner ? owner->AttachProcess(this) : -1; +} + +void WatchDogObject::exited_notify(int retval) +{ + delete this; +} + +void WatchDogObject::killed_notify(int sig, int cd) +{ + report_kill_alarm(sig, cd); + delete this; +} +void WatchDogObject::report_kill_alarm(int signo, int coredumped) +{ + + if (!ALARM_REPORTER->init_alarm_cfg(std::string(ALARM_CONF_FILE), true)) + { + log_error("init alarm conf file fail"); + return; + } + ALARM_REPORTER->set_time_out(1); + std::stringstream oss; + oss << "child process[" << pid << "][ " << name << " ]killed by signal " << signo; + if (coredumped) + { + oss << " core dumped"; + } + ALARM_REPORTER->report_alarm(oss.str()); +} +WatchDogUnit::WatchDogUnit(void) + : pidCount(0){}; + +WatchDogUnit::~WatchDogUnit(void) +{ + pidmap_t::iterator i; + for (i = pid2obj.begin(); i != pid2obj.end(); i++) + { + WatchDogObject *obj = i->second; + delete obj; + } +}; + +int WatchDogUnit::CheckWatchDog(void) +{ + while (1) + { + int status; + int pid = waitpid(-1, &status, WNOHANG); + int err = errno; + + if (pid < 0) + { + switch (err) + { + case EINTR: + case ECHILD: + break; + default: + log_notice("wait() return pid %d errno %d", pid, err); + break; + } + break; + } + else if (pid == 0) + { + break; + } + else + { + pidmap_t::iterator itr = pid2obj.find(pid); + if (itr == pid2obj.end()) + { + log_notice("wait() return unknown pid %d status 0x%x", pid, status); + } + else + { + WatchDogObject *obj = itr->second; + const char *const name = obj->Name(); + + pid2obj.erase(itr); + + // special exit value return-ed by CrashProtector + if (WIFEXITED(status) && WEXITSTATUS(status) == 85 && strncmp(name, "main", 5) == 0) + { // treat it as a fake SIGSEGV + status = W_STOPCODE(SIGSEGV); + } + + if (WIFSIGNALED(status)) + { + const int sig = WTERMSIG(status); + const int core = WCOREDUMP(status); + + log_alert("%s: killed by signal %d", name, sig); + + log_error("child %.16s pid %d killed by signal %d%s", + name, pid, sig, + core ? " (core dumped)" : ""); + pidCount--; + obj->killed_notify(sig, core); + } + else if (WIFEXITED(status)) + { + const int retval = (signed char)WEXITSTATUS(status); + if (retval == 0) + log_debug("child %.16s pid %d exit status %d", + name, pid, retval); + else + log_info("child %.16s pid %d exit status %d", + name, pid, retval); + pidCount--; + obj->exited_notify(retval); + } + } + } + } + return pidCount; +} + +int WatchDogUnit::AttachProcess(WatchDogObject *obj) +{ + const int pid = obj->pid; + pidmap_t::iterator itr = pid2obj.find(pid); + if (itr != pid2obj.end() || pid <= 1) + return -1; + + pid2obj[pid] = obj; + pidCount++; + return 0; +} + +int WatchDogUnit::KillAll(void) +{ + pidmap_t::iterator i; + int n = 0; + for (i = pid2obj.begin(); i != pid2obj.end(); i++) + { + WatchDogObject *obj = i->second; + if (obj->Pid() > 1) + { + log_debug("killing child %.16s pid %d SIGTERM", + obj->Name(), obj->Pid()); + kill(obj->Pid(), SIGTERM); + n++; + } + } + return n; +} + +int WatchDogUnit::ForceKillAll(void) +{ + pidmap_t::iterator i; + int n = 0; + for (i = pid2obj.begin(); i != pid2obj.end(); i++) + { + WatchDogObject *obj = i->second; + if (obj->Pid() > 1) + { + log_error("child %.16s pid %d didn't exit in timely, sending SIGKILL.", + obj->Name(), obj->Pid()); + kill(obj->Pid(), SIGKILL); + n++; + } + } + return n; +} diff --git a/src/search_local/index_storage/watchdog/watchdog_unit.h b/src/search_local/index_storage/watchdog/watchdog_unit.h new file mode 100644 index 0000000..93c16d0 --- /dev/null +++ b/src/search_local/index_storage/watchdog/watchdog_unit.h @@ -0,0 +1,74 @@ +/* + * ===================================================================================== + * + * Filename: watchdog_unit.h + * + * Description: + * + * Version: 1.0 + * Created: 09/08/2020 10:02:05 PM + * Revision: none + * Compiler: gcc + * + * Author: linjinming, linjinming@jd.com + * Company: JD.com, Inc. + * + * ===================================================================================== + */ +#ifndef __H_WATCHDOG_UNIT_H__ +#define __H_WATCHDOG_UNIT_H__ +#include +#include +#include + +class WatchDogUnit; + +class WatchDogObject +{ +protected: + friend class WatchDogUnit; + WatchDogUnit *owner; + int pid; + char name[16]; + +public: + WatchDogObject(void) : owner(NULL), pid(0) { name[0] = '0'; } + WatchDogObject(WatchDogUnit *u) : owner(u), pid(0) { name[0] = '0'; } + WatchDogObject(WatchDogUnit *u, const char *n) : owner(u), pid(0) + { + strncpy(name, n, sizeof(name)); + } + WatchDogObject(WatchDogUnit *u, const char *n, int i) : owner(u), pid(i) + { + strncpy(name, n, sizeof(name)); + } + virtual ~WatchDogObject(); + virtual void exited_notify(int retval); + virtual void killed_notify(int signo, int coredumped); + const char *Name(void) const { return name; } + int Pid(void) const { return pid; } + int attach_watch_dog(WatchDogUnit *o = 0); + void report_kill_alarm(int signo, int coredumped); +}; + +class WatchDogUnit +{ +private: + friend class WatchDogObject; + typedef std::map pidmap_t; + int pidCount; + pidmap_t pid2obj; + +private: + int AttachProcess(WatchDogObject *); + +public: + WatchDogUnit(void); + virtual ~WatchDogUnit(void); + int CheckWatchDog(void); // return #pid monitored + int KillAll(void); + int ForceKillAll(void); + int ProcessCount(void) const { return pidCount; } +}; + +#endif