mirrors/isearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

storage

Browse Source
This commit is contained in:
yangshuang70 2021-03-19 18:51:46 +08:00
parent 86e7f9bd60
commit 0023b1b102
346 changed files with 75788 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/search_local/index_storage/Make.conf Normal file
View File

@ -0,0 +1,41 @@
#for memchecker only, developer-only
ifeq ($(LIB_PATH),)
LIB_PATH = ../..
endif
ifeq ($(MEMCHECK),1)
CFLAGS += -DMEMCHECK=1
endif
#for gprof only
ifeq ($(GPROF),1)
CFLAGS += -pg
LDFLAGS += -pg
endif
# slk10 don't accept: -fvisibility=hidden
CFLAGS += -g -fno-strict-aliasing -fpermissive
GCCVER := $(shell $(CC) -dumpversion | awk -F. '{ print $$1"."$$2}')
PLATFORM := $(shell $(CC) -dumpmachine)
ifeq ($(findstring x86_64,$(PLATFORM)),)
CFLAGS += -march=pentiumpro
endif
LD := $(CXX)
ifeq ($(findstring x86_64, $(PLATFORM)),)
ZINC := -I$(LIB_PATH)/3rdlib/zlib/include32/
Z_LIB := $(LIB_PATH)/3rdlib/zlib/lib/libz32.a
DTC_API_LIB = ../3rdlib/dtc_api/lib32/libdtc-gcc-4.4-r66b566b.so
CA_API_LIB = $(LIB_PATH)/3rdlib/CA_API/libapp-client.a
else
ZINC := -I$(LIB_PATH)/3rdlib/zlib/include64/
Z_LIB := $(LIB_PATH)/3rdlib/zlib/lib/libz64.a
DTC_API_LIB = ../3rdlib/dtc_api/lib64/libdtc-gcc-4.4-r66b566b.so
CA_API_LIB = $(LIB_PATH)/3rdlib/CA_API/libapp-client.a
endif
#custom-helper
CUSTOMHLP=1
all::

391
src/search_local/index_storage/Make.rules Normal file
View File

@ -0,0 +1,391 @@
.DEFAULT_GOAL := all
.PHONY: all clean install count FORCE
export MAKEFLAGS
MAKEFLAGS += --no-print-directory
ifneq (,$(DIRNAME))
target_external :=
endif
I=$(if $(filter install,$(MAKECMDGOALS)),1,0)
ifeq (,$(MAKEFILE_LIST))
MAKEFILE_LIST := Makefile
endif
_INDEX := 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
EXTDIRS := $(foreach p,$(target_external),$(if $(wildcard $p),,$(dir $p)))
EXTIDXS := $(addprefix make_extdir_,$(wordlist 1,$(words $(EXTDIRS)),$(_INDEX)))
all:: $(addprefix make_all_,$(SUBDIRS)) $(EXTIDXS) $(target) $(target_only);
install:: $(addprefix make_all_,$(SUBDIRS)) $(EXTIDXS) $(target) $(target_only);
ifeq ($I,1)
all:: $(addprefix install_,$(target_install));
install:: $(addprefix install_,$(target_install));
endif
FORCE:;
clean:: $(addprefix make_clean_,$(SUBDIRS));
#override CFLAGS += -Wall -D_GNU_SOURCE -D_REENTRANT -g -O2
override CFLAGS += -Wall -D_GNU_SOURCE -D_REENTRANT -g
ifeq ($(DEBUG), )
override CFLAGS += -O2
endif
ECHO = /bin/echo
ifneq (0,$C)
RED = \\e[1m\\e[31m
DARKRED = \\e[31m
GREEN = \\e[1m\\e[32m
DARKGREEN = \\e[32m
BLUE = \\e[1m\\e[34m
DARKBLUE = \\e[34m
YELLOW = \\e[1m\\e[33m
DARKYELLOW = \\e[33m
MAGENTA = \\e[1m\\e[35m
DARKMAGENTA = \\e[35m
CYAN = \\e[1m\\e[36m
DARKCYAN = \\e[36m
RESET = \\e[m
color:
@$(ECHO) -e $(RED)RED$(RESET)
@$(ECHO) -e $(GREEN)GREEN$(RESET)
@$(ECHO) -e $(BLUE)BLUE$(RESET)
@$(ECHO) -e $(YELLOW)YELLOW$(RESET)
@$(ECHO) -e $(MAGENTA)MAGENTA$(RESET)
@$(ECHO) -e $(CYAN)CYAN $(RESET)
@$(ECHO) -e $(DARKRED)DARKRED$(RESET)
@$(ECHO) -e $(DARKGREEN)DARKGREEN$(RESET)
@$(ECHO) -e $(DARKBLUE)DARKBLUE$(RESET)
@$(ECHO) -e $(DARKYELLOW)DARKYELLOW$(RESET)
@$(ECHO) -e $(DARKMAGENTA)DARKMAGENTA$(RESET)
@$(ECHO) -e $(DARKCYAN)DARKCYAN $(RESET)
endif
export P
PSX = $(if $(findstring -j,-$(MAKEFLAGS)),$(GREEN)$(EPREFIX) )
ifneq (,$(findstring clean,$(MAKECMDGOALS)))
override P := 0
EXTIDXS :=
endif
ifeq (0,$P)
_TOTAL :=
_TOTAL2 :=
override TOTAL :=
else
_TOTAL := $(foreach p,$(basename $(filter-out %.so,$(target))),$($p_objs))
_TOTAL2 := $(foreach p,$(basename $(filter %.so,$(target))),$($p_objs:.o=.pic.o))
_TOTAL2 := $(words $(sort $(target) $(_TOTAL) $(_TOTAL2)))
_TOTAL := $(SUBDIRS)
endif
$(foreach p,$(_TOTAL),$(eval $(shell $(MAKE) -s -C $p count DIRNAME=$p P=$P)))
$(foreach p,$(EXTIDXS),$(eval $(shell $(MAKE) -s -C $(word $(subst make_extdir_,,$p),$(EXTDIRS)) count DIRNAME=$p P=$P)))
_TOTAL += $(EXTIDXS)
_TOTAL := $(if $(_TOTAL),$(words $(_TOTAL))*2 $(patsubst %,+$$(TOTAL_%),$(sort $(_TOTAL))))
$(if $(_TOTAL),$(eval _TOTAL2:=$$(shell $(ECHO) $$$$(($(_TOTAL2)+$(_TOTAL))))))
$(if $(TOTAL),,$(eval TOTAL:=$(_TOTAL2)))
ifeq (0,$(TOTAL))
TOTAL :=
endif
ifeq (,$(TOTAL))
override PCT :=
override PCT0 :=
override PCT1 :=
override inc :=
else
ifneq (,$(findstring -j,-$(MAKEFLAGS)))
override PCT :=
override PCT0 :=
override PCT1 :=
override inc :=
else
ifeq ($(START),)
STEP := 0 00
else
STEP := $(START) $(shell B=$$(($(START)*100/$(TOTAL))); [ $$B -le 9 ] && B=0$$B; $(ECHO) $$B)
endif
START = $(firstword $(STEP))
PCT0 = $(if $(findstring -j,-$(MAKEFLAGS)),," ")
ifeq (2,$P)
PCT1 = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(TOTAL)/$(TOTAL)])
PCT = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(firstword $(STEP))/$(TOTAL)])
else
PCT1 = $(if $(findstring -j,-$(MAKEFLAGS)),,[100])
PCT = $(if $(findstring -j,-$(MAKEFLAGS)),,[$(word 2,$(STEP))%])
endif
define inc
$(if $(findstring -j,-$(MAKEFLAGS)),,STEP := $(shell A=$$(($(firstword $(STEP))+$(if $1,$1,1))); B=$$(($$A*995/$(TOTAL)/10)); [ $$B -le 9 ] && B=0$$B; $(ECHO) $$A $$B))
endef
ifeq (0,$P)
inc := 0 00
endif
endif
endif
ifeq (default,$(origin LD))
LD := $(CC)
endif
$(eval LD := $(CXX) -Wl,--as-needed)
$(eval MMD = -MMD -MF .dep.$$@)
define CC
@$(ECHO) -e $(PCT) $(eval $(inc)) " CC "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET)
@gcc
endef
define CCSO
@$(ECHO) -e $(PCT) $(eval $(inc)) " CC [PIC] "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET)
@gcc -fPIC
endef
define CXX
@$(ECHO) -e $(PCT) $(eval $(inc)) " CXX "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET)
@g++
endef
define CXXSO
@$(ECHO) -e $(PCT) $(eval $(inc)) " CXX [PIC] "$(PSX)$(CYAN)$(patsubst tmp.%,%,$<) $(RESET)
@g++ -fPIC
endef
define MLD
@$(ECHO) -e $(PCT) $(eval $(inc)) " LD "$(PSX)$(MAGENTA)$@ $(RESET)
@$(LD)
endef
define MLDSO
@$(ECHO) -e $(PCT) $(eval $(inc)) " LD [PIC] "$(PSX)$(MAGENTA)$@ $(RESET)
@$(LD) -shared
endef
define AR
@$(ECHO) -e $(PCT) $(eval $(inc)) " AR "$(PSX)$(MAGENTA)$@ $(RESET)
@ar
endef
define BIN2C
@$(ECHO) -e $(PCT) $(eval $(inc)) " BIN2C "$(PSX)$(YELLOW)$@ $(RESET)
@xxd -i
endef
ENTER = @$(ECHO) -e $(PCT) $(eval $(inc)) " ENTER ==> "$(GREEN)
LEAVE = @$(ECHO) -e $(PCT) $(eval $(inc)) " LEAVE <== "$(GREEN)
define RM
@$(ECHO) -e $(PCT0) " CLEAN "$(PSX)$(MAGENTA)$(if $(CLEAN),$(CLEAN),"<$(notdir $(CURDIR))>") $(RESET)
@rm -f
endef
define INST
@$(ECHO) -e $(PCT0) " INSTALL "$(PSX)$(YELLOW)$* $(RESET)
@install
endef
DONE = @$(ECHO) -e $(PCT1) " DONE "$(MAGENTA)"<$(notdir $(CURDIR))>" $(RESET)
BAD = ($(ECHO) -e $(PCT0) " BAD "$(PSX)$(RED)$(1) $(RESET); /bin/false)
BAD = ($(ECHO) -e $(PCT0) " BAD "$(PSX)$(RED)$(1) $(RESET); /bin/false)
MDEP = $(ECHO) -e $(PCT0) " MAKEDEP "$(YELLOW)$(EPREFIX)$(firstword $(MAKEFILE_LIST)) $(RESET) 1>&2
tmp.%.c: %.c
@ln -s $< $@
tmp.%.cc: %.cc
@ln -s $< $@
tmp.%.cpp: %.cpp
@ln -s $< $@
%.pic.o: %.c
$(CCSO) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $<
%_pic.o: %.c
$(CCSO) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $<
%.o: %.c
$(CC) $(MMD) $(CFLAGS) $($*_cflags) -c -o $@ $<
%.pic.o: %.cc
$(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%_pic.o: %.cc
$(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%.o: %.cc
$(CXX) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%.pic.o: %.cpp
$(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%_pic.o: %.cpp
$(CXXSO) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%.o: %.cpp
$(CXX) $(MMD) $(CFLAGS) $(CXXFLAGS) $($*_cflags) -c -o $@ $<
%.bin.c: %
$(BIN2C) $< >$@
make_all_%::
$(ENTER) $(EPREFIX)$* $(RESET)
@$(MAKE) -s -C $* all DONE= I=$I P=$P TOTAL=$(TOTAL) START=$(START) EPREFIX=$(EPREFIX)$*/
@$(eval $(call inc,$(TOTAL_$(strip $*))))
$(LEAVE) $(EPREFIX)$* $(RESET)
make_extdir_%::
$(ENTER) $(word $*,$(EXTDIRS)) $(RESET)
@$(MAKE) -s -C $(word $*,$(EXTDIRS)) all DONE= P=$P TOTAL=$(TOTAL) START=$(START)
@$(eval $(call inc,$(TOTAL_make_extdir_$(strip $*))))
$(LEAVE) $(word $*,$(EXTDIRS)) $(RESET)
make_clean_%:
@$(MAKE) -s -C $* clean CLEAN=$(if $(CLEAN),$(CLEAN)/)$*
install_%.so: %.so
@[ -f $(install_dir) ] || mkdir -p $(install_dir)
$(INST) $(if $($*_soname),$($*_soname),$*.so) $(install_dir)
$(if $($*_soname),@ln -sf $($*_soname) $(install_dir)/$*.so)
install_%: %
@[ -f $(install_dir) ] || mkdir -p $(install_dir)
$(INST) $* $(if $(install_mode),-m $(install_mode),) $(install_dir)
%: %.c
%: %.cc
%: %.cpp
%: %.o
_COMMA=,
_method :=
ifneq (,$(findstring second-expansion,$(.FEATURES)))
# for gmake v3.81 or above, second-expansion method
.SECONDEXPANSION:
_SUB=:.o=.pic.o
%.pic.a: $$(%_objs$$(_SUB))
$(AR) cr $@ $^
%.so: $$(%_objs$$(_SUB))
# $(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname),$($*_soname) -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS)
$(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname),$(if $($*_filename), $($*_filename), $($*_soname)) -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS)
$(if $($*_soname),@ln -sf $($*_soname) $@)
$(if $($*_soname),$(if $($*_filename), @ln -sf $($*_filename) $($*_soname)))
# libdtc.so: $$(%_objs$$(_SUB))
# $(MLDSO) -z defs $(LDFLAGS) -o $(if $($*_soname), $filename -Wl$(_COMMA)-soname=$($*_soname),$@) $^ $($*_libs) $(LIBPATH) $(LIBS)
# $(if $($*_soname),@ln -sf $(filename) libdtc.so.1)
# $(if $($*_soname),@ln -sf libdtc.so.1 $@)
%.a: $$(%_objs)
$(AR) cr $@ $^
%: $$(%_objs)
@test -n "$^" || $(call BAD,$@)
$(MLD) $(LDFLAGS) -o $@ $^ $($*_libs) $(LIBPATH) $(LIBS)
_method := second-expansion
endif
$(if $(_method),,$(if $(findstring i486,$(shell gcc -dumpmachine)),,$(eval _method := eval)))
ifeq (eval,$(_method))
# for gmake v3.80, $(eval) method
define rule-piclib
$p: $($(p:.pic.a=)_objs:.o=.pic.o)
$$(AR) cr $p $$^
endef
define rule-lib
$p: $($(p:.a=)_objs)
$$(AR) cr $p $$^
endef
define rule-so
$p: $($(p:.so=)_objs:.o=.pic.o)
$$(MLDSO) -z defs $$(LDFLAGS) -o $(if $($(p:.so=)_soname),$($(p:.so=)_soname) -Wl$(_COMMA)-soname=$($(p:.so=)_soname),$p) $$^ $$($(p:.so=)_libs) $$(LIBPATH) $$(LIBS)
$(if $($(p:.so=)_soname),@ln -s $($(p:.so=)_soname) $p)
endef
define rule-bin
$p: $($p_objs)
$$(MLD) $$(LDFLAGS) -o $p $$($p_objs) $$($p_libs) $$(LIBPATH) $$(LIBS)
endef
endif
# gmake 3.80 don't allow ($eval) inside if/endif block
$(foreach p,$(filter %.pic.a,$(target)),$(if $($(p:.pic.a=)_objs),$(eval $(rule-piclib))))
$(foreach p,$(filter %.a,$(target)),$(if $($(p:.a=)_objs),$(eval $(rule-lib))))
$(foreach p,$(filter %.so,$(target)),$(if $($(p:.so=)_objs),$(eval $(rule-so))))
$(foreach p,$(filter-out %.a %.o %.so,$(target)),$(if $($p_objs),$(eval $(rule-bin))))
ifneq (clean,$(MAKECMDGOALS))
ifneq (count,$(MAKECMDGOALS))
ifeq (,$(_method))
# without $(eval), fallback to temporary Makefile
_method := makefile
define cmd-piclib
$(ECHO) '$p: $($(p:.pic.a=)_objs:.o=.pic.o)' >>.make.dep; \
$(ECHO) ' $$(AR) cr $p $$^' >>.make.dep
endef
define cmd-lib
$(ECHO) '$p: $($(p:.a=)_objs)' >>.make.dep; \
$(ECHO) ' $$(AR) cr $p $$^' >>.make.dep
endef
define cmd-so
$(ECHO) '$p: $($(p:.so=)_objs:.o=.pic.o)' >>.make.dep; \
$(ECHO) ' $$(MLDSO) -z defs $$(LDFLAGS) -o $(if $($(p:.so=)_soname),$($(p:.so=)_soname) -Wl$(_COMMA)-soname=$($(p:.so=)_soname),$p) $$^ $$($(p:.so=)_libs) $$(LIBPATH) $$(LIBS)' >>.make.dep; \
$(ECHO) ' $(if $($(p:.so=)_soname),@ln -s $($(p:.so=)_soname) $p)' >>.make.dep
endef
define cmd-bin
$(ECHO) '$p: $($p_objs)' >>.make.dep; \
$(ECHO) ' $$(MLD) $$(LDFLAGS) -o $p $$($p_objs) $$($p_libs) $$(LIBPATH) $$(LIBS)' >>.make.dep
endef
ifeq (OK,$(shell [ $(firstword $(MAKEFILE_LIST)) -nt .make.dep ] && $(ECHO) OK))
$(shell $(MDEP))
$(shell >.make.dep)
$(foreach p,$(filter %.pic.a,$(target)),$(if $($(p:.pic.a=)_objs),$(shell $(cmd-piclib))))
$(foreach p,$(filter %.a,$(target)),$(if $($(p:.a=)_objs),$(shell $(cmd-lib))))
$(foreach p,$(filter %.so,$(target)),$(if $($(p:.so=)_objs),$(shell $(cmd-so))))
$(foreach p,$(filter-out %.a %.o %.so,$(target)),$(if $($p_objs),$(shell $(cmd-bin))))
endif
include .make.dep
endif
endif
endif
.SECONDARY:
all::
$(DONE)
install:: all;
clean::
$(foreach p,$(filter %.so,$(target)),@-rm -f $($(p:.so=)_soname))
clean::
$(RM) .make.* *.o *.a *.d *.so .dep.* tmp.* core core.* *.log $(target) $(TSTS)
count::
@$(ECHO) TOTAL_$(if $(DIRNAME),$(DIRNAME),all)=$(TOTAL)
method::
@$(ECHO) rule method is $(_method).
ifneq ($(wildcard .dep.*),)
-include $(wildcard .dep.*)
endif

12
src/search_local/index_storage/Makefile Normal file
View File

@ -0,0 +1,12 @@
include Make.conf
SUBDIRS := common api stat watchdog cache rocksdb_helper
make_all_stat:: make_all_common
make_all_cache:: make_all_common make_all_api make_all_stat make_all_watchdog
make_all_proxy:: make_all_common make_all_stat
include Make.rules

5
src/search_local/index_storage/api/Makefile Normal file
View File

@ -0,0 +1,5 @@
include ../Make.conf
SUBDIRS := c_api c_api_cc
include ../Make.rules

39
src/search_local/index_storage/api/c_api/Makefile Normal file
View File

@ -0,0 +1,39 @@
LIB_PATH = ../../../..
include ../../Make.conf
SVN_REVISION = $(shell test -d .svn && (svn info | grep "Last Changed Rev" | cut -d " " -f 4))
ifeq "$(SVN_REVISION)a" "a"
SVN_REVISION = "(unknown)"
endif
VPATH = ../../common
################compile#############
target = libdtc.a container_api.pic.o version.pic.o somain.pic.o libdtc.so libdtc.pic.a
# CLIENTAPI macro use for scope test only
CFLAGS += -DCLIENTAPI
CFLAGS += -pthread -I../../common -I../../stat $(ZINC)
LIBS = $(Z_LIB) -ldl -lpthread
filelist := dtc_req dtc_srv dtc_pool dtc_wrap dtc_wrapp \
poller timer_list key_list table_def \
log_client mem_check md5 value \
section decode encode field_api \
packet_base packet_client \
task_base task_const net_addr udp_pool compress buffer thread
# for auto ln -sf
libdtc_objs := $(patsubst %,%.o,$(filelist))
libdtc.so: LDFLAGS += -Wl,--version-script,dtc_api.lst -e _so_start container_api.pic.o version.pic.o somain.pic.o
libdtc_filename := libdtc-gcc-$(GCCVER)-r$(SVN_REVISION).so
###############install##############
target_install = libdtc.a libdtc.pic.a libdtc.so dtc_api.h
install_dir = ../../../bin
%.a: install_dir = ../../../lib
%.h: install_dir = ../../../include
include ../../Make.rules

123
src/search_local/index_storage/api/c_api/container_api.cc Normal file
View File

@ -0,0 +1,123 @@
/*
* =====================================================================================
*
* Filename: container_api.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdio.h>
#include <dlfcn.h>
#include "container.h"
#include "version.h"
#include "dtc_int.h"
typedef IInternalService *(*QueryInternalServiceFunctionType)(const char *name, const char *instance);
IInternalService *query_internal_service(const char *name, const char *instance)
{
QueryInternalServiceFunctionType entry = NULL;
entry = (QueryInternalServiceFunctionType)dlsym(RTLD_DEFAULT, "_QueryInternalService");
if (entry == NULL)
return NULL;
return entry(name, instance);
}
static inline int field_type_to_key_type(int t)
{
switch (t)
{
case DField::Signed:
case DField::Unsigned:
return DField::Signed;
case DField::String:
case DField::Binary:
return DField::String;
}
return DField::None;
}
void NCServer::check_internal_service(void)
{
if (NCResultInternal::verify_class() == 0)
return;
IInternalService *inst = query_internal_service("dtcd", this->tablename);
// not inside dtcd or tablename not found
if (inst == NULL)
return;
// version mismatch, internal service is unusable
const char *version = inst->query_version_string();
if (version == NULL || strcasecmp(version_detail, version) != 0)
return;
// cast to DTC service
IDTCService *inst1 = static_cast<IDTCService *>(inst);
DTCTableDefinition *tdef = inst1->query_table_definition();
// verify tablename etc
if (tdef->is_same_table(tablename) == 0)
return;
// verify and save key type
int kt = field_type_to_key_type(tdef->key_type());
if (kt == DField::None)
// bad key type
return;
if (keytype == DField::None)
{
keytype = kt;
}
else if (keytype != kt)
{
badkey = 1;
errstr = "Key Type Mismatch";
return;
}
if (keyinfo.key_fields() != 0)
{
// FIXME: checking keyinfo
// ZAP key info, use ordered name from server
keyinfo.Clear();
}
// add NCKeyInfo
for (int i = 0; i < tdef->key_fields(); i++)
{
kt = field_type_to_key_type(tdef->field_type(i));
if (kt == DField::None)
// bad key type
return;
keyinfo.add_key(tdef->field_name(i), kt);
}
// OK, save it.
// old tdef always NULL, because tablename didn't set, Server didn't complete
this->tdef = tdef;
this->admin_tdef = inst1->query_admin_table_definition();
// useless here, internal DTCTableDefinition don't maintent a usage count
tdef->INC();
this->iservice = inst1;
this->completed = 1;
if (get_address() && iservice->match_listening_ports(get_address(), NULL))
{
executor = iservice->query_task_executor();
}
}

974
src/search_local/index_storage/api/c_api/dtc_api.h Normal file
View File

@ -0,0 +1,974 @@
/*
* =====================================================================================
*
* Filename: dtc_api.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DistributedTableCache
{
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
//mem monior Request;2014/06/6;by seanzheng
const int RequestMonitor = 15;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int NodeHandleChange = 13;
const int Migrate = 14;
const int ReloadClusterNodeList = 15;
const int SetClusterNodeState = 16;
const int change_node_address = 17;
const int GetClusterState = 18;
const int PurgeForHit = 19;
const int ClearCache = 21;
const int MigrateDB = 22;
const int MigrateDBSwitch = 23;
const int ColExpandStatus = 24;
const int col_expand = 25;
const int ColExpandDone = 26;
const int ColExpandKey = 27;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log(const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
#ifndef WIN32
void write_log(int level,
const char *file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf, 5, 6)));
#endif
class Result;
class Server
{
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void clone_tab_def(const Server &source);
int set_address(const char *host, const char *port = 0);
int set_table_name(const char *);
//for compress
void set_compress_level(int);
//get address and tablename set by user
const char *get_address(void) const;
const char *get_table_name(void) const;
//get address and tablename set by dtc frame,for plugin only;
const char *get_server_address(void) const;
const char *get_server_table_name(void) const;
int int_key(void);
int binary_key(void);
int string_key(void);
int add_key(const char *name, int type);
int field_type(const char *name);
const char *error_message(void) const;
void set_timeout(int);
void set_m_timeout(int);
int Connect(void);
void Close(void);
int ping(void);
void auto_ping(void);
void SetFD(int); // UNSUPPORTED API
void set_auto_update_tab(bool autoUpdate);
void set_auto_reconnect(int autoReconnect);
int decode_packet(Result &, const char *, int);
int check_packet_size(const char *, int);
void set_access_key(const char *token);
};
class Request
{
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int attach_server(Server *srv);
void set_admin_code(int code);
void set_hot_backup_id(long long);
void set_master_hb_timestamp(long long);
void set_slave_hb_timestamp(long long);
#define _REDIR_(op, t) \
int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) \
int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int field_type(const char *);
void no_cache(void);
void no_next_server(void);
void limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int OR(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//just for compress,only support binary field
int compress_set(const char *, const char *, int);
//just compress and set. Don't need compressflag
int compress_set_force(const char *, const char *, int);
//bits op
int set_multi_bits(const char *, int, int, unsigned int);
int set_bit(const char *f, int o) { return set_multi_bits(f, o, 1, 1); }
int clear_bit(const char *f, int o) { return set_multi_bits(f, o, 1, 0); }
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(OR, unsigned long long);
_REDIR_(OR, long);
_REDIR_(OR, unsigned long);
_REDIR_(OR, int);
_REDIR_(OR, unsigned int);
_REDIR_(OR, short);
_REDIR_(OR, unsigned short);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void unset_key(void);
int set_key(long long);
int set_key(const char *);
int set_key(const char *, int);
#define _REDIR_(t) \
int set_key(t a) { return set_key((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int add_key_value(const char *name, long long v);
int add_key_value(const char *name, const char *str);
int add_key_value(const char *name, const char *ptr, int len);
#define _REDIR_(t) \
int add_key_value(const char *name, t a) { return add_key_value(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *execute(void);
Result *execute(long long);
Result *execute(const char *);
Result *execute(const char *, int);
#define _REDIR_(t) \
Result *execute(t a) { return execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int execute(Result &);
int execute(Result &, long long);
int execute(Result &, const char *);
int execute(Result &, const char *, int);
#define _REDIR_(t) \
int execute(Result &r, t a) { return execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int encode_packet(char *&, int &, long long &);
int encode_packet(char *&, int &, long long &, long long);
int encode_packet(char *&, int &, long long &, const char *);
int encode_packet(char *&, int &, long long &, const char *, int);
#define _REDIR_(t) \
int encode_packet(char *&p, int &l, long long &m, t a) { return encode_packet(p, l, m, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int set_cache_id(long long);
#define _REDIR_(t) \
int set_cache_id(t a) { return set_cache_id((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
const char *error_message(void) const;
//无源模式超时时间
int set_expire_time(const char *key, int time);
int get_expire_time(const char *key);
};
class GetRequest : public Request
{
public:
GetRequest(Server *srv) : Request(srv, RequestGet) {}
GetRequest() : Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request
{
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request
{
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request
{
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request
{
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest : public Request
{
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request
{
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest : public Request
{
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest : public Request
{
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class MonitorRequest : public Request
{
public:
MonitorRequest(Server *srv) : Request(srv, RequestMonitor) {}
};
class Result
{
private:
void *addr;
long check;
Result(const Result &);
char *server_info() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void set_error(int errcode, const char *from, const char *detail); // from will not dupped
int result_code(void) const;
const char *error_message(void) const;
const char *error_from(void) const;
long long hot_backup_id() const;
long long master_hb_timestamp() const;
long long slave_hb_timestamp() const;
long long binlog_id() const;
long long binlog_offset() const;
long long mem_size() const;
long long data_size() const;
int num_rows(void) const;
int total_rows(void) const;
int affected_rows(void) const;
int num_fields(void) const;
const char *field_name(int n) const;
int field_present(const char *name) const;
int field_type(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long server_timestamp(void) const;
long long insert_id(void) const;
long long int_key(void) const;
const char *binary_key(void) const;
const char *binary_key(int *) const;
const char *binary_key(int &) const;
const char *string_key(void) const;
const char *string_key(int *) const;
const char *string_key(int &) const;
long long int_value(const char *) const;
double float_value(const char *) const;
const char *string_value(const char *) const;
const char *string_value(const char *, int *) const;
const char *string_value(const char *, int &) const;
const char *binary_value(const char *) const;
const char *binary_value(const char *, int *) const;
const char *binary_value(const char *, int &) const;
int uncompress_binary_value(const char *name, char **buf, int *lenp);
//Uncompress Binary Value without check compressflag
int uncompress_binary_value_force(const char *name, char **buf, int *lenp);
const char *uncompress_error_message() const;
long long int_value(int) const;
double float_value(int) const;
const char *string_value(int) const;
const char *string_value(int, int *) const;
const char *string_value(int, int &) const;
const char *binary_value(int) const;
const char *binary_value(int, int *) const;
const char *binary_value(int, int &) const;
int fetch_row(void);
int rewind(void);
};
class ServerPool
{
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int get_epoll_fd(int size);
int add_server(Server *srv, int mReq = 1, int mConn = 0);
int add_request(Request *, long long);
int add_request(Request *, long long, long long);
int add_request(Request *, long long, const char *);
int add_request(Request *, long long, const char *, int);
int add_request(Request *, void *);
int add_request(Request *, void *, long long);
int add_request(Request *, void *, const char *);
int add_request(Request *, void *, const char *, int);
int execute(int msec);
int execute_all(int msec);
int cancel_request(int);
int cancel_all_request(int type);
int abort_request(int);
int abort_all_request(int type);
Result *get_result(void);
Result *get_result(int);
int get_result(Result &);
int get_result(Result &, int);
int server_count(void) const;
int request_count(int type) const;
int request_state(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT | SEND | RECV | DONE;
enum
{
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
EC_TASK_TIMEOUT,
EC_BUSINESS_WITHOUT_EXPIRETIME, //62
EC_EMPTY_TBDEF, //63
EC_INVALID_KEY_VALUE, //64
EC_INVALID_EXPIRETIME, //65
EC_GET_EXPIRETIME_END_OF_RESULT, //66
EC_ERR_MIGRATEDB_ILLEGAL,
EC_ERR_MIGRATEDB_DUPLICATE,
EC_ERR_MIGRATEDB_HELPER,
EC_ERR_MIGRATEDB_MIGRATING, // 70
EC_ERR_MIGRATEDB_NOT_START,
EC_ERR_MIGRATEDB_DISTINCT,
EC_ERR_COL_EXPANDING,
EC_ERR_COL_EXPAND_DUPLICATE,
EC_ERR_COL_EXPAND_DONE_DUPLICATE, // 75
EC_ERR_COL_EXPAND_DONE_DISTINCT,
EC_ERR_COL_EXPAND_NO_MEM,
EC_ERR_COL_EXPAND_COLD,
};
enum
{
ER_HASHCHK = 1000,
ER_NISAMCHK = 1001,
ER_NO = 1002,
ER_YES = 1003,
ER_CANT_CREATE_FILE = 1004,
ER_CANT_CREATE_TABLE = 1005,
ER_CANT_CREATE_DB = 1006,
ER_DB_CREATE_EXISTS = 1007,
ER_DB_DROP_EXISTS = 1008,
ER_DB_DROP_DELETE = 1009,
ER_DB_DROP_RMDIR = 1010,
ER_CANT_DELETE_FILE = 1011,
ER_CANT_FIND_SYSTEM_REC = 1012,
ER_CANT_GET_STAT = 1013,
ER_CANT_GET_WD = 1014,
ER_CANT_LOCK = 1015,
ER_CANT_OPEN_FILE = 1016,
ER_FILE_NOT_FOUND = 1017,
ER_CANT_READ_DIR = 1018,
ER_CANT_SET_WD = 1019,
ER_CHECKREAD = 1020,
ER_DISK_FULL = 1021,
ER_DUP_KEY = 1022,
ER_ERROR_ON_CLOSE = 1023,
ER_ERROR_ON_READ = 1024,
ER_ERROR_ON_RENAME = 1025,
ER_ERROR_ON_WRITE = 1026,
ER_FILE_USED = 1027,
ER_FILSORT_ABORT = 1028,
ER_FORM_NOT_FOUND = 1029,
ER_GET_ERRNO = 1030,
ER_ILLEGAL_HA = 1031,
ER_KEY_NOT_FOUND = 1032,
ER_NOT_FORM_FILE = 1033,
ER_NOT_KEYFILE = 1034,
ER_OLD_KEYFILE = 1035,
ER_OPEN_AS_READONLY = 1036,
ER_OUTOFMEMORY = 1037,
ER_OUT_OF_SORTMEMORY = 1038,
ER_UNEXPECTED_EOF = 1039,
ER_CON_COUNT_ERROR = 1040,
ER_OUT_OF_RESOURCES = 1041,
ER_BAD_HOST_ERROR = 1042,
ER_HANDSHAKE_ERROR = 1043,
ER_DBACCESS_DENIED_ERROR = 1044,
ER_ACCESS_DENIED_ERROR = 1045,
ER_NO_DB_ERROR = 1046,
ER_UNKNOWN_COM_ERROR = 1047,
ER_BAD_NULL_ERROR = 1048,
ER_BAD_DB_ERROR = 1049,
ER_TABLE_EXISTS_ERROR = 1050,
ER_BAD_TABLE_ERROR = 1051,
ER_NON_UNIQ_ERROR = 1052,
ER_SERVER_SHUTDOWN = 1053,
ER_BAD_FIELD_ERROR = 1054,
ER_WRONG_FIELD_WITH_GROUP = 1055,
ER_WRONG_GROUP_FIELD = 1056,
ER_WRONG_SUM_SELECT = 1057,
ER_WRONG_VALUE_COUNT = 1058,
ER_TOO_LONG_IDENT = 1059,
ER_DUP_FIELDNAME = 1060,
ER_DUP_KEYNAME = 1061,
ER_DUP_ENTRY = 1062,
ER_WRONG_FIELD_SPEC = 1063,
ER_PARSE_ERROR = 1064,
ER_EMPTY_QUERY = 1065,
ER_NONUNIQ_TABLE = 1066,
ER_INVALID_DEFAULT = 1067,
ER_MULTIPLE_PRI_KEY = 1068,
ER_TOO_MANY_KEYS = 1069,
ER_TOO_MANY_KEY_PARTS = 1070,
ER_TOO_LONG_KEY = 1071,
ER_KEY_COLUMN_DOES_NOT_EXITS = 1072,
ER_BLOB_USED_AS_KEY = 1073,
ER_TOO_BIG_FIELDLENGTH = 1074,
ER_WRONG_AUTO_KEY = 1075,
ER_READY = 1076,
ER_NORMAL_SHUTDOWN = 1077,
ER_GOT_SIGNAL = 1078,
ER_SHUTDOWN_COMPLETE = 1079,
ER_FORCING_CLOSE = 1080,
ER_IPSOCK_ERROR = 1081,
ER_NO_SUCH_INDEX = 1082,
ER_WRONG_FIELD_TERMINATORS = 1083,
ER_BLOBS_AND_NO_TERMINATED = 1084,
ER_TEXTFILE_NOT_READABLE = 1085,
ER_FILE_EXISTS_ERROR = 1086,
ER_LOAD_INFO = 1087,
ER_ALTER_INFO = 1088,
ER_WRONG_SUB_KEY = 1089,
ER_CANT_REMOVE_ALL_FIELDS = 1090,
ER_CANT_DROP_FIELD_OR_KEY = 1091,
ER_INSERT_INFO = 1092,
ER_INSERT_TABLE_USED = 1093,
ER_NO_SUCH_THREAD = 1094,
ER_KILL_DENIED_ERROR = 1095,
ER_NO_TABLES_USED = 1096,
ER_TOO_BIG_SET = 1097,
ER_NO_UNIQUE_LOGFILE = 1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE = 1099,
ER_TABLE_NOT_LOCKED = 1100,
ER_BLOB_CANT_HAVE_DEFAULT = 1101,
ER_WRONG_DB_NAME = 1102,
ER_WRONG_TABLE_NAME = 1103,
ER_TOO_BIG_SELECT = 1104,
ER_UNKNOWN_ERROR = 1105,
ER_UNKNOWN_PROCEDURE = 1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE = 1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE = 1108,
ER_UNKNOWN_TABLE = 1109,
ER_FIELD_SPECIFIED_TWICE = 1110,
ER_INVALID_GROUP_FUNC_USE = 1111,
ER_UNSUPPORTED_EXTENSION = 1112,
ER_TABLE_MUST_HAVE_COLUMNS = 1113,
ER_RECORD_FILE_FULL = 1114,
ER_UNKNOWN_CHARACTER_SET = 1115,
ER_TOO_MANY_TABLES = 1116,
ER_TOO_MANY_FIELDS = 1117,
ER_TOO_BIG_ROWSIZE = 1118,
ER_STACK_OVERRUN = 1119,
ER_WRONG_OUTER_JOIN = 1120,
ER_NULL_COLUMN_IN_INDEX = 1121,
ER_CANT_FIND_UDF = 1122,
ER_CANT_INITIALIZE_UDF = 1123,
ER_UDF_NO_PATHS = 1124,
ER_UDF_EXISTS = 1125,
ER_CANT_OPEN_LIBRARY = 1126,
ER_CANT_FIND_DL_ENTRY = 1127,
ER_FUNCTION_NOT_DEFINED = 1128,
ER_HOST_IS_BLOCKED = 1129,
ER_HOST_NOT_PRIVILEGED = 1130,
ER_PASSWORD_ANONYMOUS_USER = 1131,
ER_PASSWORD_NOT_ALLOWED = 1132,
ER_PASSWORD_NO_MATCH = 1133,
ER_UPDATE_INFO = 1134,
ER_CANT_CREATE_THREAD = 1135,
ER_WRONG_VALUE_COUNT_ON_ROW = 1136,
ER_CANT_REOPEN_TABLE = 1137,
ER_INVALID_USE_OF_NULL = 1138,
ER_REGEXP_ERROR = 1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS = 1140,
ER_NONEXISTING_GRANT = 1141,
ER_TABLEACCESS_DENIED_ERROR = 1142,
ER_COLUMNACCESS_DENIED_ERROR = 1143,
ER_ILLEGAL_GRANT_FOR_TABLE = 1144,
ER_GRANT_WRONG_HOST_OR_USER = 1145,
ER_NO_SUCH_TABLE = 1146,
ER_NONEXISTING_TABLE_GRANT = 1147,
ER_NOT_ALLOWED_COMMAND = 1148,
ER_SYNTAX_ERROR = 1149,
ER_DELAYED_CANT_CHANGE_LOCK = 1150,
ER_TOO_MANY_DELAYED_THREADS = 1151,
ER_ABORTING_CONNECTION = 1152,
ER_NET_PACKET_TOO_LARGE = 1153,
ER_NET_READ_ERROR_FROM_PIPE = 1154,
ER_NET_FCNTL_ERROR = 1155,
ER_NET_PACKETS_OUT_OF_ORDER = 1156,
ER_NET_UNCOMPRESS_ERROR = 1157,
ER_NET_READ_ERROR = 1158,
ER_NET_READ_INTERRUPTED = 1159,
ER_NET_ERROR_ON_WRITE = 1160,
ER_NET_WRITE_INTERRUPTED = 1161,
ER_TOO_LONG_STRING = 1162,
ER_TABLE_CANT_HANDLE_BLOB = 1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164,
ER_DELAYED_INSERT_TABLE_LOCKED = 1165,
ER_WRONG_COLUMN_NAME = 1166,
ER_WRONG_KEY_COLUMN = 1167,
ER_WRONG_MRG_TABLE = 1168,
ER_DUP_UNIQUE = 1169,
ER_BLOB_KEY_WITHOUT_LENGTH = 1170,
ER_PRIMARY_CANT_HAVE_NULL = 1171,
ER_TOO_MANY_ROWS = 1172,
ER_REQUIRES_PRIMARY_KEY = 1173,
ER_NO_RAID_COMPILED = 1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175,
ER_KEY_DOES_NOT_EXITS = 1176,
ER_CHECK_NO_SUCH_TABLE = 1177,
ER_CHECK_NOT_IMPLEMENTED = 1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION = 1179,
ER_ERROR_DURING_COMMIT = 1180,
ER_ERROR_DURING_ROLLBACK = 1181,
ER_ERROR_DURING_FLUSH_LOGS = 1182,
ER_ERROR_DURING_CHECKPOINT = 1183,
ER_NEW_ABORTING_CONNECTION = 1184,
ER_DUMP_NOT_IMPLEMENTED = 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED = 1186,
ER_INDEX_REBUILD = 1187,
ER_MASTER = 1188,
ER_MASTER_NET_READ = 1189,
ER_MASTER_NET_WRITE = 1190,
ER_FT_MATCHING_KEY_NOT_FOUND = 1191,
ER_LOCK_OR_ACTIVE_TRANSACTION = 1192,
ER_UNKNOWN_SYSTEM_VARIABLE = 1193,
ER_CRASHED_ON_USAGE = 1194,
ER_CRASHED_ON_REPAIR = 1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK = 1196,
ER_TRANS_CACHE_FULL = 1197,
ER_SLAVE_MUST_STOP = 1198,
ER_SLAVE_NOT_RUNNING = 1199,
ER_BAD_SLAVE = 1200,
ER_MASTER_INFO = 1201,
ER_SLAVE_THREAD = 1202,
ER_TOO_MANY_USER_CONNECTIONS = 1203,
ER_SET_CONSTANTS_ONLY = 1204,
ER_LOCK_WAIT_TIMEOUT = 1205,
ER_LOCK_TABLE_FULL = 1206,
ER_READ_ONLY_TRANSACTION = 1207,
ER_DROP_DB_WITH_READ_LOCK = 1208,
ER_CREATE_DB_WITH_READ_LOCK = 1209,
ER_WRONG_ARGUMENTS = 1210,
ER_NO_PERMISSION_TO_CREATE_USER = 1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR = 1212,
ER_LOCK_DEADLOCK = 1213,
ER_TABLE_CANT_HANDLE_FULLTEXT = 1214,
ER_CANNOT_ADD_FOREIGN = 1215,
ER_NO_REFERENCED_ROW = 1216,
ER_ROW_IS_REFERENCED = 1217,
ER_CONNECT_TO_MASTER = 1218,
ER_QUERY_ON_MASTER = 1219,
ER_ERROR_WHEN_EXECUTING_COMMAND = 1220,
ER_WRONG_USAGE = 1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222,
ER_CANT_UPDATE_WITH_READLOCK = 1223,
ER_MIXING_NOT_ALLOWED = 1224,
ER_DUP_ARGUMENT = 1225,
ER_USER_LIMIT_REACHED = 1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR = 1227,
ER_LOCAL_VARIABLE = 1228,
ER_GLOBAL_VARIABLE = 1229,
ER_NO_DEFAULT = 1230,
ER_WRONG_VALUE_FOR_VAR = 1231,
ER_WRONG_TYPE_FOR_VAR = 1232,
ER_VAR_CANT_BE_READ = 1233,
ER_CANT_USE_OPTION_HERE = 1234,
ER_NOT_SUPPORTED_YET = 1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG = 1236,
ER_SLAVE_IGNORED_TABLE = 1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR = 1238,
CR_UNKNOWN_ERROR = 1900,
CR_SOCKET_CREATE_ERROR = 1901,
CR_CONNECTION_ERROR = 1902,
CR_CONN_HOST_ERROR = 1903,
CR_IPSOCK_ERROR = 1904,
CR_UNKNOWN_HOST = 1905,
CR_SERVER_GONE_ERROR = 1906,
CR_VERSION_ERROR = 1907,
CR_OUT_OF_MEMORY = 1908,
CR_WRONG_HOST_INFO = 1909,
CR_LOCALHOST_CONNECTION = 1910,
CR_TCP_CONNECTION = 1911,
CR_SERVER_HANDSHAKE_ERR = 1912,
CR_SERVER_LOST = 1913,
CR_COMMANDS_OUT_OF_SYNC = 1914,
CR_NAMEDPIPE_CONNECTION = 1915,
CR_NAMEDPIPEWAIT_ERROR = 1916,
CR_NAMEDPIPEOPEN_ERROR = 1917,
CR_NAMEDPIPESETSTATE_ERROR = 1918,
CR_CANT_READ_CHARSET = 1919,
CR_NET_PACKET_TOO_LARGE = 1920,
CR_EMBEDDED_CONNECTION = 1921,
CR_PROBE_SLAVE_STATUS = 1922,
CR_PROBE_SLAVE_HOSTS = 1923,
CR_PROBE_SLAVE_CONNECT = 1924,
CR_PROBE_MASTER_CONNECT = 1925,
CR_SSL_CONNECTION_ERROR = 1926,
CR_MALFORMED_PACKET = 1927,
CR_WRONG_LICENSE = 1928,
};
}; // namespace DistributedTableCache
namespace DTC = DistributedTableCache;
#endif

12
src/search_local/index_storage/api/c_api/dtc_api.lst Normal file
View File

@ -0,0 +1,12 @@
{
global:
*DistributedTableCache*;
__invoke_dynamic_linker__;
set_network_mode;
set_server_address;
set_server_tablename;
local:
*;
};

494
src/search_local/index_storage/api/c_api/dtc_int.h Normal file
View File

@ -0,0 +1,494 @@
/*
* =====================================================================================
*
* Filename: dtc_int.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CHC_CLI_H
#define __CHC_CLI_H
#include <table_def.h>
#include <field_api.h>
#include <packet.h>
#include <unistd.h>
#include <buffer_error.h>
#include <log.h>
#include "key_list.h"
#include <task_request.h>
#include <container.h>
#include <net_addr.h>
#include "udp_pool.h"
#include "compress.h"
#include "poll_thread.h"
#include "lock.h"
#include <map>
/*
* Goal:
* single transation (*)
* batch transation (*)
* async transation
*/
class DTCTask;
class Packet;
class NCResult;
class NCResultInternal;
class NCPool;
struct NCTransation;
class IDTCTaskExecutor;
class IDTCService;
class NCBase
{
private:
AtomicU32 _count;
public:
NCBase(void) {}
~NCBase(void) {}
int INC(void) { return ++_count; }
int DEC(void) { return --_count; }
int CNT(void) { return _count.get(); };
};
class NCRequest;
class DataConnector;
class NCServer : public NCBase
{
public: // global server state
NCServer();
NCServer(const NCServer &);
~NCServer(void);
// base settings
SocketAddress addr;
char *tablename;
char *appname;
static DataConnector *dc;
//for compress
void set_compress_level(int level) { compressLevel = level; }
int get_compress_level(void) { return compressLevel; }
//dtc set _server_address and _server_tablename for plugin
static char *_server_address;
static char *_server_tablename;
int keytype;
int autoUpdateTable;
int autoReconnect;
static int _network_mode;
NCKeyInfo keyinfo;
void clone_tab_def(const NCServer &source);
int set_address(const char *h, const char *p = NULL);
int set_table_name(const char *);
const char *get_address(void) const { return addr.Name(); } //this addres is set by user
const char *get_server_address(void) const { return _server_address; } //this address is set by dtc
const char *get_server_table_name(void) const { return _server_tablename; }
int is_dgram(void) const { return addr.socket_type() == SOCK_DGRAM; }
int is_udp(void) const { return addr.socket_type() == SOCK_DGRAM && addr.socket_family() == AF_INET; }
const char *get_table_name(void) const { return tablename; }
int int_key(void);
int string_key(void);
int field_type(const char *);
int is_completed(void) const { return completed; }
int add_key(const char *name, uint8_t type);
int key_field_cnt(void) const { return keyinfo.key_fields() ?: keytype != DField::None ? 1 : 0; }
int allow_batch_key(void) const { return keyinfo.key_fields(); }
int simple_batch_key(void) const { return keyinfo.key_fields() == 1 && keytype == keyinfo.key_type(0); }
void set_auto_update_tab(bool autoUpdate) { autoUpdateTable = autoUpdate ? 1 : 0; }
void set_auto_reconnect(int reconnect) { autoReconnect = reconnect; }
// error state
unsigned completed : 1;
unsigned badkey : 1;
unsigned badname : 1;
unsigned autoping : 1;
const char *errstr;
const char *error_message(void) const { return errstr; }
// table definition
DTCTableDefinition *tdef;
DTCTableDefinition *admin_tdef;
void save_definition(NCResult *);
DTCTableDefinition *get_tab_def(int cmd) const;
std::string accessToken;
int set_access_key(const char *token);
private: // serialNr manupulation
uint64_t lastSN;
public:
uint64_t NextSerialNr(void)
{
++lastSN;
if (lastSN == 0)
lastSN++;
return lastSN;
}
uint64_t LastSerialNr(void) { return lastSN; }
private: // timeout settings
int timeout;
int realtmo;
public:
void set_m_timeout(int n);
int get_timeout(void) const { return timeout; }
private:
int compressLevel;
private: // sync execution
IDTCService *iservice;
IDTCTaskExecutor *executor;
int netfd;
time_t lastAct;
NCRequest *pingReq;
private:
uint64_t agentTime;
public:
void set_agent_time(int t) { agentTime = t; }
uint64_t get_agent_time(void) { return agentTime; }
public:
int Connect(void);
int Reconnect(void);
void Close(void);
void SetFD(int fd)
{
Close();
netfd = fd;
update_timeout();
}
// stream, connected
int send_packet_stream(Packet &);
int decode_result_stream(NCResult &);
// dgram, connected or connectless
int send_packet_dgram(SocketAddress *peer, Packet &);
int decode_result_dgram(SocketAddress *peer, NCResult &);
// connectless
NCUdpPort *get_global_port(void);
void put_global_port(NCUdpPort *);
void TryPing(void);
int ping(void);
void auto_ping(void)
{
if (!is_dgram())
autoping = 1;
}
NCResultInternal *execute_internal(NCRequest &rq, const DTCValue *kptr) { return executor->task_execute(rq, kptr); }
int has_internal_executor(void) const { return executor != 0; }
private:
int bind_temp_unix_socket(void);
void update_timeout(void);
// this method is weak, and don't exist in libdtc.a
__attribute__((__weak__)) void check_internal_service(void);
public: // transation manager, impl at dtcpool.cc
int async_connect(int &);
NCPool *ownerPool;
int ownerId;
void set_owner(NCPool *, int);
NCResult *decode_buffer(const char *, int);
static int check_packet_size(const char *, int);
};
class NCRequest
{
public:
NCServer *server;
uint8_t cmd;
uint8_t haskey;
uint8_t flags;
int err;
DTCValue key;
NCKeyValueList kvl;
FieldValueByName ui;
FieldValueByName ci;
FieldSetByName fs;
DTCTableDefinition *tdef;
char *tablename;
int keytype;
unsigned int limitStart;
unsigned int limitCount;
int adminCode;
uint64_t hotBackupID;
uint64_t master_hb_timestamp;
uint64_t slave_hb_timestamp;
public:
NCRequest(NCServer *, int op);
~NCRequest(void);
int attach_server(NCServer *);
void enable_no_cache(void) { flags |= DRequest::Flag::no_cache; }
void enable_no_next_server(void) { flags |= DRequest::Flag::no_next_server; }
void enable_no_result(void) { flags |= DRequest::Flag::NoResult; }
int add_condition(const char *n, uint8_t op, uint8_t t, const DTCValue &v);
int add_operation(const char *n, uint8_t op, uint8_t t, const DTCValue &v);
int compress_set(const char *n, const char *v, int len);
int compress_set_force(const char *n, const char *v, int len);
int add_value(const char *n, uint8_t t, const DTCValue &v);
int Need(const char *n, int);
void limit(unsigned int st, unsigned int cnt)
{
if (cnt == 0)
st = 0;
limitStart = st;
limitCount = cnt;
}
int set_key(int64_t);
int set_key(const char *, int);
int unset_key(void);
int unset_key_value(void);
int field_type(const char *name) { return server ? server->field_type(name) : DField::None; }
int add_key_value(const char *name, const DTCValue &v, uint8_t type);
int set_cache_id(int dummy) { return err = -EINVAL; }
void set_admin_code(int code) { adminCode = code; }
void set_hot_backup_id(uint64_t v) { hotBackupID = v; }
void set_master_hb_timestamp(uint64_t t) { master_hb_timestamp = t; }
void set_slave_hb_timestamp(uint64_t t) { slave_hb_timestamp = t; }
// never return NULL
NCResult *execute(const DTCValue *key = NULL);
NCResult *execute_stream(const DTCValue *key = NULL);
NCResult *execute_dgram(SocketAddress *peer, const DTCValue *key = NULL);
NCResult *execute_network(const DTCValue *key = NULL);
NCResult *execute_internal(const DTCValue *key = NULL);
NCResult *execute(int64_t);
NCResult *execute(const char *, int);
NCResult *pre_check(const DTCValue *key); // return error result, NULL==success
int set_compress_field_name(void); //Need compress flag for read,or set compressFlag for write
int Encode(const DTCValue *key, Packet *);
// return 1 if tdef changed...
int set_tab_def(void);
int encode_buffer(char *&ptr, int &len, int64_t &magic, const DTCValue *key = NULL);
int encode_buffer(char *&ptr, int &len, int64_t &magic, int64_t);
int encode_buffer(char *&ptr, int &len, int64_t &magic, const char *, int);
const char *error_message(void) const
{
return _errmsg;
}
int set_expire_time(const char *key, int t);
int get_expire_time(const char *key);
private:
int set_compress_flag(const char *name)
{
if (tdef == NULL)
return -EC_NOT_INITIALIZED;
if (tdef->field_id(name) >= 64)
{
snprintf(_errmsg, sizeof(_errmsg), "compress error:field id must less than 64");
return -EC_COMPRESS_ERROR;
}
compressFlag |= (1 << tdef->field_id(name));
return 0;
}
uint64_t compressFlag; //field flag
DTCCompress *gzip;
int init_compress(void);
char _errmsg[1024];
};
class NCResultLocal
{
public:
const uint8_t *vidmap;
long long apiTag;
int maxvid;
DTCCompress *gzip;
public:
NCResultLocal(DTCTableDefinition *tdef) : vidmap(NULL),
apiTag(0),
maxvid(0),
gzip(NULL),
_tdef(tdef),
compressid(-1)
{
}
virtual ~NCResultLocal(void)
{
FREE_CLEAR(vidmap);
DELETE(gzip);
}
virtual int field_id_virtual(int id) const
{
return id > 0 && id <= maxvid ? vidmap[id - 1] : 0;
}
virtual void set_api_tag(long long t) { apiTag = t; }
virtual long long get_api_tag(void) const { return apiTag; }
void set_virtual_map(FieldSetByName &fs)
{
if (fs.max_virtual_id())
{
fs.Resolve(_tdef, 0);
vidmap = fs.virtual_map();
maxvid = fs.max_virtual_id();
}
}
virtual int init_compress()
{
if (NULL == _tdef)
{
return -EC_CHECKSUM_MISMATCH;
}
int iret = 0;
compressid = _tdef->compress_field_id();
if (compressid < 0)
return 0;
if (gzip == NULL)
NEW(DTCCompress, gzip);
if (gzip == NULL)
return -ENOMEM;
iret = gzip->set_buffer_len(_tdef->max_field_size());
if (iret)
return iret;
return 0;
}
virtual const int compress_id(void) const { return compressid; }
private:
DTCTableDefinition *_tdef;
uint64_t compressid;
};
class NCResult : public NCResultLocal, public DTCTask
{
public:
NCResult(DTCTableDefinition *tdef) : NCResultLocal(tdef), DTCTask(tdef, TaskRoleClient, 0)
{
if (tdef)
tdef->INC();
mark_allow_remote_table();
}
NCResult(int err, const char *from, const char *msg) : NCResultLocal(NULL), DTCTask(NULL, TaskRoleClient, 1)
{
resultInfo.set_error_dup(err, from, msg);
}
virtual ~NCResult()
{
DTCTableDefinition *tdef = table_definition();
DEC_DELETE(tdef);
}
};
class NCResultInternal : public NCResultLocal, public TaskRequest
{
public:
NCResultInternal(DTCTableDefinition *tdef = NULL) : NCResultLocal(tdef)
{
}
virtual ~NCResultInternal()
{
}
static inline int verify_class(void)
{
NCResultInternal *ir = 0;
NCResult *er = reinterpret_cast<NCResult *>(ir);
NCResultLocal *il = (NCResultLocal *)ir;
NCResultLocal *el = (NCResultLocal *)er;
DTCTask *it = (DTCTask *)ir;
DTCTask *et = (DTCTask *)er;
long dl = reinterpret_cast<char *>(il) - reinterpret_cast<char *>(el);
long dt = reinterpret_cast<char *>(it) - reinterpret_cast<char *>(et);
return dl == 0 && dt == 0;
}
};
class DataConnector
{
struct businessStatistics
{
uint64_t TotalTime; // 10s内请求总耗时
uint32_t TotalRequests; // 10s内请求总次数
public:
businessStatistics()
{
TotalTime = 0;
TotalRequests = 0;
}
};
struct bidCurve
{
uint32_t bid;
uint32_t curve;
bool operator<(const bidCurve &that) const
{
int sum1 = bid * 10 + curve;
int sum2 = that.bid * 10 + that.curve;
return sum1 < sum2;
}
};
private:
std::map<bidCurve, businessStatistics> mapBi;
Mutex _lock; // 读写 TotalTime、TotalRequests时，加锁，防止脏数据
static DataConnector *pDataConnector;
DataConnector();
~DataConnector();
pthread_t threadid;
public:
static DataConnector *getInstance()
{
if (pDataConnector == NULL)
pDataConnector = new DataConnector();
return pDataConnector;
};
public:
int send_data();
int set_report_info(const std::string str, const uint32_t curve, const uint64_t t);
void get_report_info(std::map<bidCurve, businessStatistics> &mapData);
int set_bussiness_id(std::string str);
};
#endif

1075
src/search_local/index_storage/api/c_api/dtc_pool.cc Normal file
View File

File diff suppressed because it is too large Load Diff

315
src/search_local/index_storage/api/c_api/dtc_pool.h Normal file
View File

@ -0,0 +1,315 @@
/*
* =====================================================================================
*
* Filename: dtc_pool.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CHC_CLI_POOL_H
#define __CHC_CLI_POOL_H
#include <sys/poll.h>
#include "list.h"
#include "poller.h"
#include "timer_list.h"
#include "dtc_int.h"
#include "buffer_error.h"
class NCServerInfo;
class NCPool;
class NCConnection;
// transation is a internal async request
class NCTransation : public ListObject<NCTransation>
{
public:
// constructor/destructor equiv
NCTransation(void);
~NCTransation(void);
/* clear transation state */
void Clear(void);
/* abort current transation, zero means succ */
void Abort(int);
/* transation succ with result */
void Done(NCResult *res)
{
result = res;
Abort(0);
}
/* get transation result */
NCResult *get_result(void);
/* adjust generation id */
void round_gen_id(int m)
{
if (genId >= m)
genId = 0;
}
/* state & info management */
int State(void) const { return state; }
int gen_id(void) const { return genId; }
int MatchSN(NCResult *res) const { return SN == res->versionInfo.serial_nr(); }
/* send packet management */
int Send(int fd) { return packet->Send(fd); }
int attach_request(NCServerInfo *s, long long tag, NCRequest *req, DTCValue *key);
void attach_connection(NCConnection *c);
void SendOK(ListObject<NCTransation> *);
void RecvOK(ListObject<NCTransation> *);
public: // constant member declare as static
// owner info, associated server
NCServerInfo *server;
// attached connection, SEND, RECV
NCConnection *conn;
private: // transient members is private
// current transation state, WAIT, SEND, RECV, DONE
int state;
// internal transation generation id
int genId;
// associated request tag
long long reqTag;
// associated request SN, state SEND, RECV
uint64_t SN;
// sending packet
Packet *packet;
// execute result
NCResult *result;
};
class NCConnection : public ListObject<NCConnection>,
public PollerObject
{
public:
NCConnection(NCPool *, NCServerInfo *);
~NCConnection(void);
int is_dgram(void) const;
int is_async(void) const;
/* starting state machine, by NCServerInfo::Connect */
int Connect(void);
/* attach transation to connection */
void process_request(NCTransation *);
/* flush send channel */
void send_request(void);
/* flush recv channel */
int RecvResult(void);
/* check connection hangup, recv zero bytes */
int check_hangup(void);
/* abort all associated request */
void abort_requests(int err);
/* close connection */
void Close(int err);
/* abort current sending request, linger recv channel */
void abort_send_side(int err);
/* a valid result received */
void done_result(void);
/* connection is idle, usable for transation */
void switch_to_idle(void);
/* connection is async connecting */
void switch_to_connecting(void);
virtual void input_notify(void);
virtual void output_notify(void);
virtual void hangup_notify(void);
virtual void timer_notify(void);
private:
/* associated server */
NCServerInfo *serverInfo;
/* queued transation in RECV state */
ListObject<NCTransation> reqList;
/* sending transation */
NCTransation *sreq;
/* decoding/decoded result */
NCResult *result;
/* connection state */
int state;
int NETFD(void) const { return netfd; }
private:
TimerMember<NCConnection> timer;
SimpleReceiver receiver;
};
typedef ListObject<NCConnection> NCConnectionList;
class NCPool : public PollerUnit,
public TimerUnit
{
public:
NCPool(int maxServers, int maxRequests);
~NCPool();
int initialize_poller_unit(void);
int get_epoll_fd(int maxpoller);
int add_server(NCServer *srv, int maxReq = 1, int maxConn = 0);
int add_request(NCRequest *req, long long tag, DTCValue *key = 0);
void execute_one_loop(int timeout);
int execute(int timeout);
int execute_all(int timeout);
NCTransation *Id2Req(int) const;
int cancel_request(int);
int cancel_all_request(int);
int abort_request(NCTransation *);
int abort_request(int);
int abort_all_request(int);
NCResult *get_result(void);
NCResult *get_result(int);
int count_request_state(int) const;
int request_state(int) const;
public:
NCTransation *get_transation_slot(void);
void transation_finished(NCTransation *);
NCResult *get_transation_result(NCTransation *);
int get_transation_state(NCTransation *);
int server_count(void) const { return numServers; }
int request_count(void) const { return numRequests; }
int done_request_count(void) const { return doneRequests; }
private:
int initFlag;
int maxServers;
int maxRequests;
int numServers;
int numRequests;
int maxRequestId;
int doneRequests;
ListObject<NCTransation> freeList;
ListObject<NCTransation> doneList;
NCServerInfo *serverList;
NCTransation *transList;
public:
char *buf;
};
class NCServerInfo : private TimerObject
{
public:
NCServerInfo(void);
~NCServerInfo(void);
void Init(NCServer *, int, int);
/* prepare wake timer_notify */
void mark_as_ready() { attach_ready_timer(owner); }
virtual void timer_notify(void);
/* four reason server has more work to do */
/* more transation attached */
void more_request_and_ready(void)
{
reqWait++;
mark_as_ready();
}
/* more close connection, should reconnecting */
void more_closed_connection_and_ready(void)
{
connRemain++;
mark_as_ready();
}
/* more idle connection available */
void connection_idle_and_ready(void) { mark_as_ready(); }
/* more request can assign to idle pool */
void request_done_and_ready(int oldstate);
/* one request scheduled to SEND state */
void request_scheduled(void)
{
reqRemain--;
reqSend++;
}
void request_sent(void)
{
reqSend--;
reqRecv++;
}
/* queue transation to this server */
void queue_request(NCTransation *req)
{
req->ListAddTail(&reqList);
more_request_and_ready();
}
/* one connecting aborted */
void connecting_failed(void) { connError++; }
void connecting_done(void) { connConnecting--; }
/* abort all waiting transations */
void abort_wait_queue(int err);
int count_request_state(int type) const;
/* get a waiting transation */
NCTransation *get_request_from_queue(void)
{
NCTransation *trans = reqList.NextOwner();
trans->list_del();
reqWait--;
return trans;
}
private:
int Connect(void);
ListObject<NCTransation> reqList;
public: // constant member declare as public
/* associated ServerPool */
NCPool *owner;
/* basic server info */
NCServer *info;
TimerList *timerList;
int mode; // 0--TCP 1--ASYNC 2--UDP
/* total connection */
int connTotal;
private: // transient member is private
/* transation in state WAIT */
int reqWait;
/* transation in state SEND */
int reqSend;
/* transation in state RECV */
int reqRecv;
/* remain requests can assign to connection pool */
int reqRemain;
/* remain connections can connect */
int connRemain;
/* number connections in connecting state */
int connConnecting;
/* number of connect error this round */
int connError;
/* busy connection count */
inline int conn_working(void) const { return connTotal - connConnecting - connRemain; }
public: //except idleList for code conventional
/* idle connection list */
NCConnectionList idleList;
/* busy connection list SEND,RECV,LINGER */
NCConnectionList busyList;
};
inline int NCConnection::is_dgram(void) const { return serverInfo->mode == 2; }
inline int NCConnection::is_async(void) const { return serverInfo->mode; }
#endif

1184
src/search_local/index_storage/api/c_api/dtc_req.cc Normal file
View File

File diff suppressed because it is too large Load Diff

1056
src/search_local/index_storage/api/c_api/dtc_srv.cc Normal file
View File

File diff suppressed because it is too large Load Diff

1427
src/search_local/index_storage/api/c_api/dtc_wrap.cc Normal file
View File

File diff suppressed because it is too large Load Diff

250
src/search_local/index_storage/api/c_api/dtc_wrapp.cc Normal file
View File

@ -0,0 +1,250 @@
/*
* =====================================================================================
*
* Filename: dtc_wrapp.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <inttypes.h>
#include <value.h>
#include <compiler.h>
#include "dtc_api.h"
#include "dtc_pool.h"
using namespace DistributedTableCache;
#define CAST(type, var) type *var = (type *)addr
#define CAST2(type, var, src) type *var = (type *)src->addr
#define CAST3(type, var, src) type *var = (type *)src.addr
// WRAPPER for Server
__EXPORT
ServerPool::ServerPool(int ms, int mr)
{
NCPool *pl = new NCPool(ms, mr);
addr = pl;
}
__EXPORT
ServerPool::~ServerPool(void)
{
CAST(NCPool, pl);
delete pl;
}
__EXPORT
int ServerPool::add_server(Server *srv, int mreq, int mconn)
{
CAST(NCPool, pl);
CAST2(NCServer, s, srv);
return pl->add_server(s, mreq, mconn);
}
__EXPORT
int ServerPool::add_request(Request *req, long long tag)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if (r->server == NULL)
return -EC_NOT_INITIALIZED;
return pl->add_request(r, tag, NULL);
}
__EXPORT
int ServerPool::add_request(Request *req, void *tag)
{
return add_request(req, (long long)(long)tag);
}
__EXPORT
int ServerPool::add_request(Request *req, long long tag, long long k)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if (r->server == NULL)
return -EC_NOT_INITIALIZED;
if (r->server->keytype != DField::Signed)
return -EC_BAD_KEY_TYPE;
DTCValue v(k);
return pl->add_request(r, tag, &v);
}
__EXPORT
int ServerPool::add_request(Request *req, void *tag, long long k)
{
return add_request(req, (long long)(long)tag, k);
}
__EXPORT
int ServerPool::add_request(Request *req, long long tag, const char *k)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if (r->server == NULL)
return -EC_NOT_INITIALIZED;
if (r->server->keytype != DField::String)
return -EC_BAD_KEY_TYPE;
DTCValue v(k);
return pl->add_request(r, tag, &v);
}
__EXPORT
int ServerPool::add_request(Request *req, void *tag, const char *k)
{
return add_request(req, (long long)(long)tag, k);
}
__EXPORT
int ServerPool::add_request(Request *req, long long tag, const char *k, int l)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if (r->server == NULL)
return -EC_NOT_INITIALIZED;
if (r->server->keytype != DField::String)
return -EC_BAD_KEY_TYPE;
DTCValue v(k, l);
return pl->add_request(r, tag, &v);
}
__EXPORT
int ServerPool::add_request(Request *req, void *tag, const char *k, int l)
{
return add_request(req, (long long)(long)tag, k, l);
}
__EXPORT
int ServerPool::execute(int msec)
{
CAST(NCPool, pl);
return pl->execute(msec);
}
__EXPORT
int ServerPool::execute_all(int msec)
{
CAST(NCPool, pl);
return pl->execute_all(msec);
}
__EXPORT
int ServerPool::cancel_request(int id)
{
CAST(NCPool, pl);
return pl->cancel_request(id);
}
__EXPORT
int ServerPool::cancel_all_request(int type)
{
CAST(NCPool, pl);
return pl->cancel_all_request(type);
}
__EXPORT
int ServerPool::abort_request(int id)
{
CAST(NCPool, pl);
return pl->abort_request(id);
}
__EXPORT
int ServerPool::abort_all_request(int type)
{
CAST(NCPool, pl);
return pl->abort_all_request(type);
}
__EXPORT
Result *ServerPool::get_result(void)
{
return get_result(0);
}
__EXPORT
Result *ServerPool::get_result(int id)
{
CAST(NCPool, pl);
NCResult *a = pl->get_result(id);
if ((long)a < 4096 && (long)a > -4095)
return NULL;
Result *s = new Result();
s->addr = (void *)a;
return s;
}
__EXPORT
int ServerPool::get_result(Result &s)
{
return get_result(s, 0);
}
__EXPORT
int ServerPool::get_result(Result &s, int id)
{
CAST(NCPool, pl);
s.Reset();
NCResult *a = pl->get_result(id);
long iv = (long)a;
if (iv < 4096 && iv > -4095)
{
s.addr = (void *)(new NCResult(iv, "API::executing", iv > 0 ? "Request not completed" : id ? "Invalid request" : "No more result"));
return iv;
}
s.addr = (void *)a;
return 0;
}
__EXPORT
int ServerPool::server_count(void) const
{
CAST(NCPool, pl);
return pl->server_count();
}
__EXPORT
int ServerPool::request_count(int type) const
{
CAST(NCPool, pl);
if (type == DONE)
return pl->done_request_count();
if (type == (WAIT | SEND | RECV | DONE))
return pl->request_count();
return pl->count_request_state(type);
}
__EXPORT
int ServerPool::request_state(int reqId) const
{
CAST(NCPool, pl);
return pl->request_state(reqId);
}
__EXPORT
int ServerPool::get_epoll_fd(int size)
{
CAST(NCPool, pl);
return pl->get_epoll_fd(size);
}
const int ALL_STATE = WAIT | SEND | RECV | DONE;

154
src/search_local/index_storage/api/c_api/key_list.cc Normal file
View File

@ -0,0 +1,154 @@
/*
* =====================================================================================
*
* Filename: key_list.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <errno.h>
#include "mem_check.h"
#include "protocol.h"
#include "key_list.h"
#include "buffer_error.h"
int NCKeyValueList::KeyValueMax = 32;
int NCKeyInfo::KeyIndex(const char *n) const
{
namemap_t::const_iterator i = keyMap.find(n);
return i == keyMap.end() ? -1 : i->second;
}
int NCKeyInfo::add_key(const char *name, int type)
{
switch (type)
{
case DField::Signed:
case DField::String:
break;
default:
return -EC_BAD_KEY_TYPE;
}
if (KeyIndex(name) >= 0)
{
//return -EC_DUPLICATE_FIELD;
// ignore duplicate key field name
// because NCKeyInfo may be initialized by check_internal_service()
// add again must be allowed for code compatibility
return 0;
}
int cnt = key_fields();
if (cnt >= (int)(sizeof(keyType) / sizeof(keyType[0])))
return -EC_BAD_MULTIKEY;
keyName[cnt] = name;
keyMap[name] = cnt;
keyType[cnt] = type;
keyCount++;
return 0;
}
int NCKeyInfo::Equal(const NCKeyInfo &other) const
{
int n = key_fields();
// key field count mis-match
if (other.key_fields() != n)
return 0;
// key type mis-match
if (memcmp(keyType, other.keyType, n) != 0)
return 0;
for (int i = 0; i < n; i++)
{
// key name mis-match
if (strcasecmp(keyName[i], other.keyName[i]) != 0)
return 0;
}
return 1;
}
void NCKeyValueList::Unset(void)
{
if (kcount)
{
const int kn = key_fields();
for (int i = 0; i < kcount; i++)
{
for (int j = 0; j < kn; j++)
if (key_type(j) == DField::String || key_type(j) == DField::Binary)
FREE(Value(i, j).bin.ptr);
}
FREE_CLEAR(val);
memset(fcount, 0, sizeof(fcount));
kcount = 0;
}
}
int NCKeyValueList::add_value(const char *name, const DTCValue &v, int type)
{
const int kn = key_fields();
int col = keyinfo->KeyIndex(name);
if (col < 0 || col >= kn)
return -EC_BAD_KEY_NAME;
switch (key_type(col))
{
case DField::Signed:
case DField::Unsigned:
if (type != DField::Signed && type != DField::Unsigned)
return -EC_BAD_VALUE_TYPE;
break;
case DField::String:
case DField::Binary:
if (type != DField::String && type != DField::Binary)
return -EC_BAD_VALUE_TYPE;
if (v.bin.len > 255)
return -EC_KEY_OVERFLOW;
break;
default:
return -EC_BAD_KEY_TYPE;
}
int row = fcount[col];
if (row >= KeyValueMax)
return -EC_TOO_MANY_KEY_VALUE; // key值太多
if (row >= kcount)
{
if (REALLOC(val, (kcount + 1) * kn * sizeof(DTCValue)) == NULL)
throw std::bad_alloc();
memset(&Value(row, 0), 0, kn * sizeof(DTCValue));
kcount++;
}
DTCValue &slot = Value(row, col);
switch (key_type(col))
{
case DField::Signed:
case DField::Unsigned:
slot = v;
break;
case DField::String:
case DField::Binary:
slot.bin.len = v.bin.len;
slot.bin.ptr = (char *)MALLOC(v.bin.len + 1);
if (slot.bin.ptr == NULL)
throw std::bad_alloc();
memcpy(slot.bin.ptr, v.bin.ptr, v.bin.len);
slot.bin.ptr[v.bin.len] = '\0';
break;
}
fcount[col]++;
return 0;
}

117
src/search_local/index_storage/api/c_api/key_list.h Normal file
View File

@ -0,0 +1,117 @@
/*
* =====================================================================================
*
* Filename: key_list.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CHC_KEYLIST_H
#define __CHC_KEYLIST_H
#include <string.h>
#include <map>
#include <value.h>
class NCKeyInfo
{
private:
struct nocase
{
bool operator()(const char *const &a, const char *const &b) const
{
return strcasecmp(a, b) < 0;
}
};
typedef std::map<const char *, int, nocase> namemap_t;
private:
namemap_t keyMap;
const char *keyName[8];
uint8_t keyType[8];
int keyCount;
public:
// zero is KeyField::None
void Clear(void)
{
keyCount = 0;
memset(keyType, 0, sizeof(keyType));
memset(keyName, 0, sizeof(keyName));
keyMap.clear();
}
NCKeyInfo()
{
keyCount = 0;
memset(keyType, 0, sizeof(keyType));
memset(keyName, 0, sizeof(keyName));
}
NCKeyInfo(const NCKeyInfo &that)
{
keyCount = that.keyCount;
memcpy(keyType, that.keyType, sizeof(keyType));
memcpy(keyName, that.keyName, sizeof(keyName));
for (int i = 0; i < keyCount; i++)
keyMap[keyName[i]] = i;
}
~NCKeyInfo() {}
int add_key(const char *name, int type);
int Equal(const NCKeyInfo &other) const;
int KeyIndex(const char *n) const;
const char *key_name(int n) const { return keyName[n]; }
int key_type(int id) const { return keyType[id]; }
int key_fields(void) const { return keyCount; }
};
class NCKeyValueList
{
public:
static int KeyValueMax;
public:
NCKeyInfo *keyinfo;
DTCValue *val;
int fcount[8];
int kcount;
public:
NCKeyValueList(void) : keyinfo(NULL), val(NULL), kcount(0)
{
memset(fcount, 0, sizeof(fcount));
}
~NCKeyValueList()
{
FREE_CLEAR(val);
}
int key_fields(void) const { return keyinfo->key_fields(); }
int key_type(int id) const { return keyinfo->key_type(id); }
int key_count(void) const { return kcount; }
const char *key_name(int id) const { return keyinfo->key_name(id); }
void Unset(void);
int add_value(const char *, const DTCValue &, int);
DTCValue &Value(int row, int col) { return val[row * keyinfo->key_fields() + col]; }
const DTCValue &Value(int row, int col) const { return val[row * keyinfo->key_fields() + col]; }
DTCValue &operator()(int row, int col) { return Value(row, col); }
const DTCValue &operator()(int row, int col) const { return Value(row, col); }
int IsFlat(void) const
{
for (int i = 1; i < keyinfo->key_fields(); i++)
if (fcount[0] != fcount[i])
return 0;
return 1;
}
};
#endif

26
src/search_local/index_storage/api/c_api/log_client.cc Normal file
View File

@ -0,0 +1,26 @@
/*
* =====================================================================================
*
* Filename: log_client.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "dtc_api.h"
#include <log.cc>
__EXPORT
void DistributedTableCache::write_log(int level,
const char *file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf, 5, 6)))
__attribute__((__alias__("_write_log_")));

3
src/search_local/index_storage/api/c_api/readme.txt Normal file
View File

@ -0,0 +1,3 @@
注意事项：
1.auto_dtcc++.sh 该脚本是需要root权限去执行的，它的作用是将dtc sdk 的动态库
放置在 /usr/lib 目录下，并且执行命令 ldconfig 使配置生效

49
src/search_local/index_storage/api/c_api/somain.c Normal file
View File

@ -0,0 +1,49 @@
/*
* =====================================================================================
*
* Filename: somain.c
*
* Description: entry function.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#undef _GNU_SOURCE
#define _GNU_SOURCE
#include <stdlib.h>
#include <unistd.h>
#include <version.h>
#include <compiler.h>
const char __invoke_dynamic_linker__[]
__attribute__((section(".interp")))
__HIDDEN =
#if __x86_64__
"/lib64/ld-linux-x86-64.so.2"
#else
"/lib/ld-linux.so.2"
#endif
;
__HIDDEN
void _so_start(char *arg1, ...)
{
#define BANNER "DTC client API v" DTC_VERSION_DETAIL "\n" \
" - TCP connection supported\n" \
" - UDP connection supported\n" \
" - UNIX stream connection supported\n" \
" - embeded threading connection supported\n" \
" - protocol packet encode/decode interface supported\n" \
" - async execute (except embeded threading) supported \n"
int unused;
unused = write(1, BANNER, sizeof(BANNER) - 1);
_exit(0);
}

257
src/search_local/index_storage/api/c_api/udp_pool.cc Normal file
View File

@ -0,0 +1,257 @@
/*
* =====================================================================================
*
* Filename: udp_pool.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <netinet/in.h>
#include <net/if.h>
#include <pthread.h>
#include <arpa/inet.h>
#include <string.h>
#include <unistd.h>
#include <alloca.h>
#include "udp_pool.h"
#include "log.h"
static unsigned int bindip = 0xFFFFFFFF;
unsigned int GetBindIp(void)
{
const char *name = getenv("DTCAPI_UDP_INTERFACE");
if (name == NULL || name[0] == 0 || strcmp(name, "*") == 0)
{
return 0;
}
int fd = socket(AF_INET, SOCK_DGRAM, 0);
struct ifconf ifc;
struct ifreq *ifr = NULL;
int n = 0;
int i;
if (fd < 0)
return 0;
ifc.ifc_len = 0;
ifc.ifc_req = NULL;
if (ioctl(fd, SIOCGIFCONF, &ifc) == 0)
{
ifr = (struct ifreq *) alloca(ifc.ifc_len > 128 ? ifc.ifc_len : 128);
ifc.ifc_req = ifr;
if (ioctl(fd, SIOCGIFCONF, &ifc) == 0)
n = ifc.ifc_len / sizeof(struct ifreq);
}
close(fd);
for (i = 0; i < n; i++)
{
if (strncmp(ifr[i].ifr_name, name, sizeof(ifr[i].ifr_name)) != 0)
continue;
if (ifr[i].ifr_addr.sa_family == AF_INET)
return ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr;
}
return 0;
}
static int CreatePortIpv4(void)
{
int fd = socket(AF_INET, SOCK_DGRAM, 0);
if (bindip == 0xFFFFFFFF)
bindip = GetBindIp();
if (bindip != 0)
{
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = bindip;
if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0)
{
close(fd);
return -1;
}
}
return fd;
}
static int CreatePortIpv6(void)
{
int fd = socket(AF_INET6, SOCK_DGRAM, 0);
return fd;
}
static int CreatePortUnix(void)
{
int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
snprintf(addr.sun_path, sizeof(addr.sun_path),
"@dtcapi-global-%d-%d-%d", getpid(), fd, (int)time(NULL));
socklen_t len = SUN_LEN(&addr);
addr.sun_path[0] = 0;
if (bind(fd, (const struct sockaddr *)&addr, len) < 0)
{
close(fd);
return -1;
}
return fd;
}
struct NCUdpPortList {
public:
int stopped;
int family;
int (*newport)(void);
pthread_mutex_t lock;
NCUdpPort *list;
public:
NCUdpPort *Get(void);
void Put(NCUdpPort *);
public:
~NCUdpPortList(void);
};
// this destructor only called when unloading libdtc.so
NCUdpPortList::~NCUdpPortList(void)
{
if (pthread_mutex_lock(&lock) == 0)
{
stopped = 1;
while (list != NULL)
{
NCUdpPort *port = list;
list = port->next;
delete port;
}
pthread_mutex_unlock(&lock);
}
}
NCUdpPort *NCUdpPortList::Get(void)
{
NCUdpPort *port = NULL;
if (pthread_mutex_lock(&lock) == 0)
{
if (list != NULL)
{
port = list;
list = port->next;
}
pthread_mutex_unlock(&lock);
}
else
{
log_error("api mutex_lock error,may have fd leak");
}
if (port != NULL)
{
if (getpid() == port->pid)
{
port->sn++;
}
else
{
delete port;
port = NULL;
}
}
if (port == NULL)
{
int fd = newport();
if (fd > 0)
{
port = new NCUdpPort;
port->fd = fd;
unsigned int seed = fd + (long)port + (long) & port + (long)pthread_self() + (long)port;
port->sn = rand_r(&seed);
port->pid = getpid();
port->timeout = -1;
port->family = family;
}
}
return port;
}
void NCUdpPortList::Put(NCUdpPort *port)
{
if (this != NULL && pthread_mutex_lock(&lock) == 0)
{
if(stopped) {
// always delete port after unloading process
port->Eat();
} else {
port->next = list;
list = port;
}
pthread_mutex_unlock(&lock);
}
else
{
port->Eat();
}
}
static NCUdpPortList ipv4List = { 0, AF_INET, CreatePortIpv4, PTHREAD_MUTEX_INITIALIZER, NULL };
static NCUdpPortList ipv6List = { 0, AF_INET6, CreatePortIpv6, PTHREAD_MUTEX_INITIALIZER, NULL };
static NCUdpPortList unixList = { 0, AF_UNIX, CreatePortUnix, PTHREAD_MUTEX_INITIALIZER, NULL };
struct NCUdpPortList *GetPortList(int family) {
switch(family) {
case AF_INET:
return &ipv4List;
case AF_INET6:
return &ipv6List;
case AF_UNIX:
return &unixList;
}
return NULL;
}
NCUdpPort *NCUdpPort::Get(int family)
{
NCUdpPortList *portList = GetPortList(family);
if(portList == NULL) {
return NULL;
}
return portList->Get();
}
void NCUdpPort::Put()
{
NCUdpPortList *portList = GetPortList(family);
portList->Put(this);
}

76
src/search_local/index_storage/api/c_api/udp_pool.h Normal file
View File

@ -0,0 +1,76 @@
/*
* =====================================================================================
*
* Filename: udp_pool.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: Norton, yangshuang68@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdlib.h>
class NCPort
{
public:
uint64_t sn;
int fd;
int timeout; //the timeout this socket is.
public:
NCPort()
{
sn = 0;
fd = -1;
timeout = -1;
}
NCPort(const NCPort &that)
{
sn = that.sn;
fd = that.fd;
timeout = that.timeout;
}
~NCPort()
{
if (fd >= 0)
{
close(fd);
}
}
};
class NCUdpPortList;
class NCUdpPort : public NCPort
{
public:
int family;
pid_t pid;
private:
NCUdpPort *next;
private:
NCUdpPort()
{
pid = -1;
};
~NCUdpPort(){};
public:
friend class NCUdpPortList;
static NCUdpPort *Get(int family); // get from cache
void Put(); // put back cache
void Eat()
{
delete this;
} // eat and delete
};

42
src/search_local/index_storage/api/c_api_cc/Makefile Normal file
View File

@ -0,0 +1,42 @@
LIB_PATH = ../../../..
include ../../Make.conf
GIT_VERSION=$(shell git log . | head -n 4 | grep "commit" | cut -d " " -f 2 | cut -c 1-7)
SVN_REVISION = $(shell test -d .svn && (svn info | grep "Last Changed Rev" | cut -d " " -f 4))
ifeq "$(SVN_REVISION)a" "a"
SVN_REVISION = "(unknown)"
endif
VPATH = ../../common
################compile#############
target = libdtc.a container_api.pic.o version.pic.o somain.pic.o libdtc.so libdtc.pic.a
# CLIENTAPI macro use for scope test only
CFLAGS += -DCLIENTAPI
CFLAGS += -pthread -I../../common -I../../stat -I../../../../3rdlib/CA_API/ $(ZINC)
LIBS = ${Z_LIB} -ldl -lpthread $(CA_API_LIB)
filelist := dtcreq dtcsvr dtcpool dtcwrap dtcwrapp dtcqossvr \
poller timerlist keylist table_def \
log_client memcheck md5 value \
section decode encode field_api \
packet_base packet_client \
task_base task_const sockaddr udppool compress buffer thread
#liddtc.so: container_api.pic.o version.pic.o
# for auto ln -sf
#libdtc_objs := $(patsubst %,%.o,$(filelist))
#libdtc.so: LDFLAGS += -Wl,--version-script,dtcapi.lst -e _so_start container_api.pic.o version.pic.o somain.pic.o
#libdtc_filename := libdtc-gcc-$(GCCVER)-r$(GIT_VERSION).so
#libdtc_soname := libdtc.so.1
###############install##############
target_install = libdtc.a libdtc.pic.a libdtc.so dtcapi.h
install_dir = ../../../bin
%.a: install_dir = ../../../lib
%.h: install_dir = ../../../include
include ../../Make.rules

101
src/search_local/index_storage/api/c_api_cc/container_api.cc Normal file
View File

@ -0,0 +1,101 @@
#include <stdio.h>
#include <dlfcn.h>
#include "container.h"
#include "version.h"
#include "dtcint.h"
typedef IInternalService *(*QueryInternalServiceFunctionType)(const char *name, const char *instance);
IInternalService *QueryInternalService(const char *name, const char *instance)
{
QueryInternalServiceFunctionType entry = NULL;
entry = (QueryInternalServiceFunctionType)dlsym(RTLD_DEFAULT, "_QueryInternalService");
if(entry == NULL)
return NULL;
return entry(name, instance);
}
static inline int fieldtype2keytype(int t)
{
switch(t) {
case DField::Signed:
case DField::Unsigned:
return DField::Signed;
case DField::String:
case DField::Binary:
return DField::String;
}
return DField::None;
}
void NCServer::CheckInternalService(void)
{
if(NCResultInternal::VerifyClass()==0)
return;
IInternalService *inst = QueryInternalService("dtcd", this->tablename);
// not inside dtcd or tablename not found
if(inst == NULL)
return;
// version mismatch, internal service is unusable
const char *version = inst->query_version_string();
if(version==NULL || strcasecmp(version_detail, version) != 0)
return;
// cast to DTC service
IDTCService *inst1 = static_cast<IDTCService *>(inst);
DTCTableDefinition *tdef = inst1->query_table_definition();
// verify tablename etc
if(tdef->is_same_table(tablename)==0)
return;
// verify and save key type
int kt = fieldtype2keytype(tdef->key_type());
if(kt == DField::None)
// bad key type
return;
if(keytype == DField::None) {
keytype = kt;
} else if(keytype != kt) {
badkey = 1;
errstr = "Key Type Mismatch";
return;
}
if(keyinfo.KeyFields()!=0) {
// FIXME: checking keyinfo
// ZAP key info, use ordered name from server
keyinfo.Clear();
}
// add NCKeyInfo
for(int i=0; i<tdef->key_fields(); i++) {
kt = fieldtype2keytype(tdef->field_type(i));
if(kt == DField::None)
// bad key type
return;
keyinfo.AddKey(tdef->field_name(i), kt);
}
// OK, save it.
// old tdef always NULL, because tablename didn't set, Server didn't complete
this->tdef = tdef;
this->admin_tdef = inst1->query_admin_table_definition();
// useless here, internal DTCTableDefinition don't maintent a usage count
tdef->INC();
this->iservice = inst1;
this->completed = 1;
if(GetAddress() && iservice->match_listening_ports(GetAddress(), NULL)) {
executor = iservice->query_task_executor();
}
}

1021
src/search_local/index_storage/api/c_api_cc/dtcapi.h Normal file
View File

File diff suppressed because it is too large Load Diff

12
src/search_local/index_storage/api/c_api_cc/dtcapi.lst Normal file
View File

@ -0,0 +1,12 @@
{
global:
*DTC*;
__invoke_dynamic_linker__;
set_network_mode;
set_server_address;
set_server_tablename;
local:
*;
};

617
src/search_local/index_storage/api/c_api_cc/dtcint.h Normal file
View File

@ -0,0 +1,617 @@
#ifndef __CHC_CLI_H
#define __CHC_CLI_H
#include <table_def.h>
#include <field_api.h>
#include <packet.h>
#include <unistd.h>
#include <buffer_error.h>
#include <log.h>
#include "keylist.h"
#include <task_request.h>
#include <container.h>
#include <net_addr.h>
#include "udppool.h"
#include "compress.h"
#include "poll_thread.h"
#include "lock.h"
#include <map>
/*
* Goal:
* single transation (*)
* batch transation (*)
* async transation
*/
#define HUNDRED 100
#define THOUSAND 1000
#define TENTHOUSAND 10000
#define HUNDREDTHOUSAND 100000
#define MILLION 1000000
enum E_REPORT_TYPE
{
RT_MIN,
RT_SHARDING,
RT_ALL,
RT_MAX
};
//tp99 统计区间，单位us
static const uint32_t kpi_sample_count[] =
{
200, 500, 1000,
2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000,
20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000, 100000,
200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000, 1000000,
2000000, 500000, 10000000
};
class DTCTask;
class CPacket;
class NCResult;
class NCResultInternal;
class NCPool;
struct NCTransation;
class IDTCTaskExecutor;
class IDTCService;
class NCBase {
private:
AtomicU32 _count;
public:
NCBase(void) {}
~NCBase(void) {}
int INC(void) { return ++_count; }
int DEC(void) { return --_count; }
int CNT(void) { return _count.get(); };
};
class NCRequest;
class DataConnector;
class NCServer: public NCBase {
public: // global server state
NCServer();
NCServer(const NCServer &);
~NCServer(void);
// base settings
SocketAddress addr;
char *tablename;
char *appname;
static DataConnector *dc;
//for compress
void SetCompressLevel(int level){compressLevel=level;}
int GetCompressLevel(void){return compressLevel;}
//dtc set _server_address and _server_tablename for plugin
static char * _server_address;
static char * _server_tablename;
int keytype;
int autoUpdateTable;
int autoReconnect;
static int _network_mode;
NCKeyInfo keyinfo;
void CloneTabDef(const NCServer& source);
int SetAddress(const char *h, const char *p=NULL);
int SetTableName(const char *);
const char * GetAddress(void) const { return addr.Name(); }//this addres is set by user
const char * GetServerAddress(void) const { return _server_address; }//this address is set by dtc
const char * GetServerTableName(void) const { return _server_tablename;}
int IsDgram(void) const { return addr.socket_type()==SOCK_DGRAM; }
int IsUDP(void) const { return addr.socket_type()==SOCK_DGRAM && addr.socket_family()==AF_INET; }
const char * GetTableName(void) const { return tablename; }
int IntKey(void);
int StringKey(void);
int FieldType(const char*);
int IsCompleted(void) const { return completed; }
int AddKey(const char* name, uint8_t type);
int KeyFieldCnt(void) const { return keyinfo.KeyFields() ? : keytype != DField::None ? 1 : 0; }
int AllowBatchKey(void) const { return keyinfo.KeyFields(); }
int SimpleBatchKey(void) const { return keyinfo.KeyFields()==1 && keytype == keyinfo.KeyType(0); }
void SetAutoUpdateTab(bool autoUpdate){ autoUpdateTable = autoUpdate?1:0; }
void SetAutoReconnect(int reconnect){ autoReconnect = reconnect; }
// error state
unsigned completed:1;
unsigned badkey:1;
unsigned badname:1;
unsigned autoping:1;
const char *errstr;
const char *ErrorMessage(void) const { return errstr; }
// table definition
DTCTableDefinition *tdef;
DTCTableDefinition *admin_tdef;
void SaveDefinition(NCResult *);
DTCTableDefinition* GetTabDef(int cmd) const;
/*date:2014/06/04, author:xuxinxin*/
std::string accessToken;
int SetAccessKey(const char *token);
private: // serialNr manupulation
uint64_t lastSN;
public:
uint64_t NextSerialNr(void) { ++lastSN; if(lastSN==0) lastSN++; return lastSN; }
uint64_t LastSerialNr(void) { return lastSN; }
private: // timeout settings
int timeout;
int realtmo;
public:
void SetMTimeout(int n);
int GetTimeout(void) const { return timeout; }
private:
int compressLevel;
private: // sync execution
IDTCService *iservice;
IDTCTaskExecutor *executor;
int netfd;
time_t lastAct;
NCRequest *pingReq;
private:
uint64_t agentTime;
public:
void SetAgentTime(int t){agentTime=t;}
uint64_t GetAgentTime(void){return agentTime;}
public:
int Connect(void);
int Reconnect(void);
void Close(void);
void SetFD(int fd) { Close(); netfd = fd; UpdateTimeout(); }
// stream, connected
int SendPacketStream(CPacket &);
int DecodeResultStream(NCResult &);
// dgram, connected or connectless
int SendPacketDgram(SocketAddress *peer, CPacket &);
int DecodeResultDgram(SocketAddress *peer, NCResult &);
// connectless
NCUdpPort *GetGlobalPort(void);
void PutGlobalPort(NCUdpPort *);
void TryPing(void);
int Ping(void);
void AutoPing(void) { if(!IsDgram()) autoping = 1; }
NCResultInternal *ExecuteInternal(NCRequest &rq, const DTCValue *kptr) { return executor->task_execute(rq, kptr); }
int HasInternalExecutor(void) const { return executor != 0; }
private:
int BindTempUnixSocket(void);
void UpdateTimeout(void);
void UpdateTimeoutAnyway(void);
// this method is weak, and don't exist in libdtc.a
__attribute__((__weak__)) void CheckInternalService(void);
public: // transation manager, impl at dtcpool.cc
int AsyncConnect(int &);
NCPool *ownerPool;
int ownerId;
void SetOwner(NCPool *, int);
NCResult *DecodeBuffer(const char *, int);
static int CheckPacketSize(const char *, int);
//add by neolv to QOS
private:
//原有的不变增加三个参数用于做QOS
uint64_t m_iErrCount;
//用于统计请求总数， 请求总数只对一个周期有效
uint64_t m_iTotalCount;
//用于统计请求总耗时， 请求总数只对一个周期有效
uint64_t m_iTotalElaps;
//被摘次数
int m_RemoveCount;
public:
void IncErrCount()
{
++m_iErrCount;
}
uint64_t GetErrCount()
{
return m_iErrCount;
}
void ClearErrCount()
{
m_iErrCount = 0;
}
void IncRemoveCount()
{
++m_RemoveCount;
}
int GetRemoveCount()
{
return m_RemoveCount;
}
void ClearRemoveCount()
{
m_RemoveCount = 0;
}
void IncTotalReq()
{
++m_iTotalCount;
}
uint64_t GetTotalReq()
{
return m_iTotalCount;
}
void ClearTotalReq()
{
m_iTotalCount = 0;
}
void AddTotalElaps(uint64_t iElaps)
{
m_iTotalElaps += iElaps;
}
uint64_t GetTotalElaps()
{
return m_iTotalElaps;
}
void ClearTotalElaps()
{
m_iTotalElaps = 0;
}
};
class NCRequest {
public:
NCServer *server;
uint8_t cmd;
uint8_t haskey;
uint8_t flags;
int err;
DTCValue key;
NCKeyValueList kvl;
FieldValueByName ui;
FieldValueByName ci;
FieldSetByName fs;
DTCTableDefinition *tdef;
char *tablename;
int keytype;
unsigned int limitStart;
unsigned int limitCount;
int adminCode;
uint64_t hotBackupID;
uint64_t MasterHBTimestamp;
uint64_t SlaveHBTimestamp;
public:
NCRequest(NCServer *, int op);
~NCRequest(void);
int AttachServer(NCServer *);
void EnableNoCache(void) { flags |= DRequest::Flag::no_cache; }
void EnableNoNextServer(void) { flags |= DRequest::Flag::no_next_server; }
void EnableNoResult(void) { flags |= DRequest::Flag::NoResult; }
int AddCondition(const char *n, uint8_t op, uint8_t t, const DTCValue &v);
int AddOperation(const char *n, uint8_t op, uint8_t t, const DTCValue &v);
int CompressSet(const char *n,const char * v,int len);
int CompressSetForce(const char *n,const char * v,int len);
int AddValue(const char *n, uint8_t t, const DTCValue &v);
int Need(const char *n, int);
void Limit(unsigned int st, unsigned int cnt) {
if(cnt==0) st = 0;
limitStart = st;
limitCount = cnt;
}
int SetKey(int64_t);
int SetKey(const char *, int);
int UnsetKey(void);
int UnsetKeyValue(void);
int FieldType(const char* name){ return server?server->FieldType(name):DField::None; }
int AddKeyValue(const char* name, const DTCValue &v, uint8_t type);
int SetCacheID(int dummy) { return err = -EINVAL; }
void SetAdminCode(int code){ adminCode = code; }
void SetHotBackupID(uint64_t v){ hotBackupID=v; }
void SetMasterHBTimestamp(uint64_t t){ MasterHBTimestamp=t; }
void SetSlaveHBTimestamp(uint64_t t){ SlaveHBTimestamp=t; }
// never return NULL
NCResult *Execute(const DTCValue *key=NULL);
NCResult *ExecuteStream(const DTCValue *key=NULL);
NCResult *ExecuteDgram(SocketAddress *peer, const DTCValue *key = NULL);
NCResult *ExecuteNetwork(const DTCValue *key=NULL);
NCResult *ExecuteInternal(const DTCValue *key=NULL);
NCResult *Execute(int64_t);
NCResult *Execute(const char *, int);
NCResult *PreCheck(const DTCValue *key); // return error result, NULL==success
int SetCompressFieldName(void);//Need compress flag for read,or set compressFlag for write
int Encode(const DTCValue *key, CPacket *);
// return 1 if tdef changed...
int SetTabDef(void);
int EncodeBuffer(char *&ptr, int&len, int64_t &magic, const DTCValue *key=NULL);
int EncodeBuffer(char *&ptr, int&len, int64_t &magic, int64_t);
int EncodeBuffer(char *&ptr, int&len, int64_t &magic, const char *, int);
const char* ErrorMessage(void) const
{
return _errmsg;
}
int SetExpireTime(const char* key, int t);
int GetExpireTime(const char* key);
private:
int CheckKey(const DTCValue *kptr);
int setCompressFlag(const char * name)
{
if (tdef==NULL)
return -EC_NOT_INITIALIZED;
if (tdef->field_id(name) >= 64)
{
snprintf(_errmsg, sizeof(_errmsg), "compress error:field id must less than 64");
return -EC_COMPRESS_ERROR;
}
compressFlag|=(1<<tdef->field_id(name));
return 0;
}
uint64_t compressFlag;//field flag
DTCCompress *gzip;
int initCompress(void);
char _errmsg[1024];
};
class NCResultLocal {
public:
const uint8_t *vidmap;
long long apiTag;
int maxvid;
DTCCompress *gzip;
public:
NCResultLocal(DTCTableDefinition* tdef) :
vidmap(NULL),
apiTag(0),
maxvid(0),
gzip (NULL),
_tdef(tdef),
compressid(-1)
{
}
virtual ~NCResultLocal(void) {
FREE_CLEAR(vidmap);
DELETE (gzip);
}
virtual int FieldIdVirtual(int id) const {
return id > 0 && id <= maxvid ? vidmap[id-1] : 0;
}
virtual void SetApiTag(long long t) { apiTag = t; }
virtual long long GetApiTag(void) const { return apiTag; }
void SetVirtualMap(FieldSetByName &fs)
{
if(fs.max_virtual_id()){
fs.Resolve(_tdef, 0);
vidmap = fs.virtual_map();
maxvid = fs.max_virtual_id();
}
}
virtual int initCompress()
{
if (NULL == _tdef)
{
return -EC_CHECKSUM_MISMATCH;
}
int iret = 0;
compressid = _tdef->compress_field_id();
if (compressid<0) return 0;
if (gzip==NULL)
NEW(DTCCompress,gzip);
if (gzip==NULL)
return -ENOMEM;
iret = gzip->set_buffer_len(_tdef->max_field_size());
if (iret) return iret;
return 0;
}
virtual const int CompressId (void)const {return compressid;}
private:
DTCTableDefinition* _tdef;
uint64_t compressid;
};
class NCResult: public NCResultLocal, public DTCTask
{
public:
NCResult(DTCTableDefinition *tdef) : NCResultLocal(tdef), DTCTask(tdef, TaskRoleClient, 0) {
if(tdef) tdef->INC();
mark_allow_remote_table();
}
NCResult(int err, const char *from, const char *msg) : NCResultLocal(NULL), DTCTask(NULL, TaskRoleClient, 1) {
resultInfo.set_error_dup(err, from, msg);
}
virtual ~NCResult() {
DTCTableDefinition *tdef = table_definition();
DEC_DELETE(tdef);
}
};
class NCResultInternal: public NCResultLocal, public TaskRequest
{
public:
NCResultInternal(DTCTableDefinition* tdef=NULL) : NCResultLocal (tdef)
{
}
virtual ~NCResultInternal()
{
}
static inline int VerifyClass(void)
{
NCResultInternal *ir = 0;
NCResult *er = reinterpret_cast<NCResult *>(ir);
NCResultLocal *il = (NCResultLocal *) ir;
NCResultLocal *el = (NCResultLocal *) er;
DTCTask *it = (DTCTask *) ir;
DTCTask *et = (DTCTask *) er;
long dl = reinterpret_cast<char *>(il) - reinterpret_cast<char *>(el);
long dt = reinterpret_cast<char *>(it) - reinterpret_cast<char *>(et);
return dl==0 && dt==0;
}
};
/*date:2014/06/09, author:xuxinxin 模调上报 */
class DataConnector
{
struct businessStatistics
{
uint64_t TotalTime; // 10s内请求总耗时
uint32_t TotalRequests; // 10s内请求总次数
public:
businessStatistics(){ TotalTime = 0; TotalRequests = 0; }
};
struct bidCurve{
uint32_t bid;
uint32_t curve;
bool operator < (const bidCurve &that) const{
int sum1 = bid * 10 + curve;
int sum2 = that.bid * 10 + that.curve;
return sum1 < sum2;
}
};
struct top_percentile_statistics
{
uint32_t uiBid;
uint32_t uiAgentIP;
uint16_t uiAgentPort;
uint32_t uiTotalRequests; //10s内请求总次数
uint64_t uiTotalTime; //10s内请求总耗时
uint32_t uiFailCount; //10s内错误次数
uint64_t uiMaxTime; //10s内的最大执行时间
uint64_t uiMinTime; //10s内的最小执行时间
uint32_t statArr[sizeof(kpi_sample_count) / sizeof(kpi_sample_count[0])]; //统计值
public:
top_percentile_statistics()
{
uiBid = 0;
uiAgentIP = 0;
uiAgentPort = 0;
uiTotalRequests = 0;
uiTotalTime = 0;
uiFailCount = 0;
uiMaxTime = 0;
uiMinTime = 0;
memset(statArr, 0, sizeof(statArr));
}
top_percentile_statistics(const top_percentile_statistics &that)
{
this->uiBid = that.uiBid;
this->uiAgentIP = that.uiAgentIP;
this->uiAgentPort = that.uiAgentPort;
this->uiTotalRequests = that.uiTotalRequests;
this->uiTotalTime = that.uiTotalTime;
this->uiFailCount = that.uiFailCount;
this->uiMaxTime = that.uiMaxTime;
this->uiMinTime = that.uiMinTime;
memcpy(this->statArr, that.statArr, sizeof(this->statArr));
}
top_percentile_statistics &operator =(const top_percentile_statistics &that)
{
this->uiBid = that.uiBid;
this->uiAgentIP = that.uiAgentIP;
this->uiAgentPort = that.uiAgentPort;
this->uiTotalRequests = that.uiTotalRequests;
this->uiTotalTime = that.uiTotalTime;
this->uiFailCount = that.uiFailCount;
this->uiMaxTime = that.uiMaxTime;
this->uiMinTime = that.uiMinTime;
memcpy(this->statArr, that.statArr, sizeof(this->statArr));
return *this;
}
};
private:
std::map<bidCurve, businessStatistics> mapBi;
Mutex _lock; // 读写 TotalTime、TotalRequests时，加锁，防止脏数据
std::map<uint64_t, top_percentile_statistics> mapTPStat;
Mutex m_tp_lock; //读写 tp99 数据时，加锁，防止脏数据
static DataConnector *pDataConnector;
DataConnector();
~DataConnector();
pthread_t threadid;
public:
static DataConnector* getInstance()
{
if( pDataConnector == NULL)
pDataConnector = new DataConnector();
return pDataConnector;
};
public:
int SendData();
int SetReportInfo(const std::string str, const uint32_t curve, const uint64_t t);
void GetReportInfo(std::map<bidCurve, businessStatistics> &mapData);
int SetBussinessId(std::string str);
int SendTopPercentileData();
int SetTopPercentileData(const std::string strAccessKey, const std::string strAgentAddr, const uint64_t elapse, const int status);
void GetTopPercentileData(std::map<uint64_t, top_percentile_statistics> &mapStat);
int SetTopPercentileConfig(std::string strAccessKey, const std::string strAgentAddr);
private:
int ParseAddr(const std::string strAddr, uint64_t *uipIP = NULL, uint64_t *uipPort = NULL);
};
class CTopPercentileSection
{
public:
static int16_t GetTPSection(uint64_t elapse);
private:
//执行时间小于1000us的
static int16_t GetLTThousand(uint64_t elapse);
//执行时间大于等于1000us 小于 10000us的
static int16_t GetGTEThousandLTTenThousand(uint64_t elapse);
//执行时间大于等于10000us 小于 100000us的
static int16_t GetGTETenThousandLTHundredThousand(uint64_t elapse);
//执行时间大于等于100000us 小于 1000000us的
static int16_t GetGTEHundredThousandLTMillion(uint64_t elapse);
//执行时间大于等于1000000us的
static int16_t GetGTEMillion(uint64_t elapse);
};
#endif

1009
src/search_local/index_storage/api/c_api_cc/dtcpool.cc Normal file
View File

File diff suppressed because it is too large Load Diff

275
src/search_local/index_storage/api/c_api_cc/dtcpool.h Normal file
View File

@ -0,0 +1,275 @@
#ifndef __CHC_CLI_POOL_H
#define __CHC_CLI_POOL_H
#include <sys/poll.h>
#include "list.h"
#include "poller.h"
#include "timerlist.h"
#include "dtcint.h"
#include "cache_error.h"
class NCServerInfo;
class NCPool;
class NCConnection;
// transation is a internal async request
class NCTransation :
public CListObject<NCTransation>
{
public:
// constructor/destructor equiv
NCTransation(void);
~NCTransation(void);
/* clear transation state */
void Clear(void);
/* abort current transation, zero means succ */
void Abort(int);
/* transation succ with result */
void Done(NCResult *res) { result = res; Abort(0); }
/* get transation result */
NCResult * GetResult(void);
/* adjust generation id */
void RoundGenId(int m) { if(genId >= m) genId = 0; }
/* state & info management */
int State(void) const { return state; }
int GenId(void) const { return genId; }
int MatchSN(NCResult *res) const { return SN == res->versionInfo.SerialNr(); }
/* send packet management */
int Send(int fd) { return packet->Send(fd); }
int AttachRequest(NCServerInfo *s, long long tag, NCRequest *req, DTCValue *key);
void AttachConnection(NCConnection *c);
void SendOK(CListObject<NCTransation>*);
void RecvOK(CListObject<NCTransation>*);
public: // constant member declare as static
// owner info, associated server
NCServerInfo *server;
// attached connection, SEND, RECV
NCConnection *conn;
private:// transient members is private
// current transation state, WAIT, SEND, RECV, DONE
int state;
// internal transation generation id
int genId;
// associated request tag
long long reqTag;
// associated request SN, state SEND, RECV
uint64_t SN;
// sending packet
CPacket *packet;
// execute result
NCResult *result;
};
class NCConnection :
public CListObject<NCConnection>,
public CPollerObject
{
public:
NCConnection(NCPool *, NCServerInfo *);
~NCConnection(void);
int IsDgram(void) const;
int IsAsync(void) const;
/* starting state machine, by NCServerInfo::Connect */
int Connect(void);
/* attach transation to connection */
void ProcessRequest(NCTransation *);
/* flush send channel */
void SendRequest(void);
/* flush recv channel */
int RecvResult(void);
/* check connection hangup, recv zero bytes */
int CheckHangup(void);
/* abort all associated request */
void AbortRequests(int err);
/* close connection */
void Close(int err);
/* abort current sending request, linger recv channel */
void AbortSendSide(int err);
/* a valid result received */
void DoneResult(void);
/* connection is idle, usable for transation */
void SwitchToIdle(void);
/* connection is async connecting */
void SwitchToConnecting(void);
virtual void InputNotify(void);
virtual void OutputNotify(void);
virtual void HangupNotify(void);
virtual void TimerNotify(void);
private:
/* associated server */
NCServerInfo *serverInfo;
/* queued transation in RECV state */
CListObject<NCTransation> reqList;
/* sending transation */
NCTransation *sreq;
/* decoding/decoded result */
NCResult *result;
/* connection state */
int state;
int NETFD(void) const { return netfd; }
private:
CTimerMember<NCConnection> timer;
CSimpleReceiver receiver;
};
typedef CListObject<NCConnection> NCConnectionList;
class NCPool :
public CPollerUnit,
public CTimerUnit
{
public:
NCPool(int maxServers, int maxRequests);
~NCPool();
int InitializePollerUnit(void);
int GetEpollFD(int maxpoller);
int AddServer(NCServer *srv, int maxReq=1, int maxConn=0);
int AddRequest(NCRequest *req, long long tag, DTCValue *key=0);
void ExecuteOneLoop(int timeout);
int Execute(int timeout);
int ExecuteAll(int timeout);
NCTransation *Id2Req(int) const;
int CancelRequest(int);
int CancelAllRequest(int);
int AbortRequest(NCTransation *);
int AbortRequest(int);
int AbortAllRequest(int);
NCResult *GetResult(void);
NCResult *GetResult(int);
int CountRequestState(int) const;
int RequestState(int) const;
public:
NCTransation * GetTransationSlot(void);
void TransationFinished(NCTransation *);
NCResult * GetTransationResult(NCTransation *);
int GetTransationState(NCTransation *);
int ServerCount(void) const { return numServers; }
int RequestCount(void) const { return numRequests; }
int DoneRequestCount(void) const { return doneRequests; }
private:
int initFlag;
int maxServers;
int maxRequests;
int numServers;
int numRequests;
int maxRequestId;
int doneRequests;
CListObject<NCTransation> freeList;
CListObject<NCTransation> doneList;
NCServerInfo *serverList;
NCTransation *transList;
public:
char *buf;
};
class NCServerInfo :
private CTimerObject
{
public:
NCServerInfo(void);
~NCServerInfo(void);
void Init(NCServer *, int, int);
/* prepare wake TimerNotify */
void MarkAsReady() { AttachReadyTimer(owner); }
virtual void TimerNotify(void);
/* four reason server has more work to do */
/* more transation attached */
void MoreRequestAndReady(void) { reqWait++; MarkAsReady(); }
/* more close connection, should reconnecting */
void MoreClosedConnectionAndReady(void) { connRemain++; MarkAsReady(); }
/* more idle connection available */
void ConnectionIdleAndReady(void) { MarkAsReady(); }
/* more request can assign to idle pool */
void RequestDoneAndReady(int oldstate);
/* one request scheduled to SEND state */
void RequestScheduled(void) { reqRemain--; reqSend++; }
void RequestSent(void) { reqSend--; reqRecv++; }
/* queue transation to this server */
void QueueRequest(NCTransation *req)
{
req->ListAddTail(&reqList);
MoreRequestAndReady();
}
/* one connecting aborted */
void ConnectingFailed(void) { connError++; }
void ConnectingDone(void) { connConnecting--; }
/* abort all waiting transations */
void AbortWaitQueue(int err);
int CountRequestState(int type) const;
/* get a waiting transation */
NCTransation * GetRequestFromQueue(void)
{
NCTransation *trans = reqList.NextOwner();
trans->ListDel();
reqWait--;
return trans;
}
private:
int Connect(void);
CListObject<NCTransation> reqList;
public: // constant member declare as public
/* associated ServerPool */
NCPool *owner;
/* basic server info */
NCServer *info;
CTimerList *timerList;
int mode; // 0--TCP 1--ASYNC 2--UDP
/* total connection */
int connTotal;
private:// transient member is private
/* transation in state WAIT */
int reqWait;
/* transation in state SEND */
int reqSend;
/* transation in state RECV */
int reqRecv;
/* remain requests can assign to connection pool */
int reqRemain;
/* remain connections can connect */
int connRemain;
/* number connections in connecting state */
int connConnecting;
/* number of connect error this round */
int connError;
/* busy connection count */
inline int connWorking(void) const { return connTotal - connConnecting - connRemain; }
public: //except idleList for code conventional
/* idle connection list */
NCConnectionList idleList;
/* busy connection list SEND,RECV,LINGER */
NCConnectionList busyList;
};
inline int NCConnection::IsDgram(void) const { return serverInfo->mode==2; }
inline int NCConnection::IsAsync(void) const { return serverInfo->mode; }
#endif

834
src/search_local/index_storage/api/c_api_cc/dtcqossvr.cc Normal file
View File

@ -0,0 +1,834 @@
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <time.h>
#include <fcntl.h>
#include "protocol.h"
#include "dtcapi.h"
#include "dtcint.h"
#include "dtcqossvr.h"
using namespace DTC;
/* linux-2.6.38.8/include/linux/compiler.h */
//# define likely(x) __builtin_expect(!!(x), 1)
//# define unlikely(x) __builtin_expect(!!(x), 0)
#define random(x) (rand()%x)
const uint32_t ACCESS_KEY_LEN = 40;
DTCQosServer::DTCQosServer():m_Status(0), m_Weight(0), m_RemoveTimeStamp(0),
m_LastRemoveTime(0), m_MaxTimeStamp(0), m_Server(NULL)
{
}
DTCQosServer::~DTCQosServer()
{
if (this->m_Server != NULL){
delete this->m_Server;
this->m_Server = NULL;
}
}
DTCServers::DTCServers()
:m_TimeOut(50), m_AgentTime(0), m_KeyType(0), m_TableName(NULL),
m_SetRoute(false), m_ConstructedBySetIPs(false), m_Bid(0),
m_IDCNo(0), m_BucketsPos(0), m_BalanceBucketSize(0), m_BidVersion(0),
m_LastGetCaTime(0), m_RefreshBucketsTime(0), m_RemoveBucketsTime(0),
m_LoadBalanceBuckets(NULL), m_QOSSevers(NULL)
{
}
DTCServers::~DTCServers()
{
if(this->m_TableName)
{
free(this->m_TableName);
this->m_TableName = NULL;
}
if(this->m_LoadBalanceBuckets)
{
free(this->m_LoadBalanceBuckets);
this->m_LoadBalanceBuckets = NULL;
}
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
}
void PrintIPNode(ROUTE_NODE RouteNode)
{
printf("\n");
printf("\t bid = %d \n", RouteNode.bid);
printf("\t ip = %s \n", RouteNode.ip);
printf("\t port = %d \n", RouteNode.port);
printf("\t weight = %d \n", RouteNode.weight);
printf("\t status = %d \n", RouteNode.status);
printf("\n");
}
void DTCServers::SetErrorMsg(int err, std::string from, std::string msg)
{
char buffer[1000];
memset(buffer, 0, 1000);
sprintf(buffer, "ERROR CODE %d, FROM %s ERRMSG %s", err, from.c_str(), msg.c_str());
std::string ErrMsg(buffer);
this->m_ErrMsg.clear();
this->m_ErrMsg = ErrMsg;
}
std::string DTCServers::GetErrorMsg()
{
return this->m_ErrMsg;
}
int InvalidIpaddr(char *str)
{
if(str == NULL || *str == '\0')
return 1;
union
{
struct sockaddr addr;
struct sockaddr_in6 addr6;
struct sockaddr_in addr4;
} a;
memset(&a, 0, sizeof(a));
if(1 == inet_pton(AF_INET, str, &a.addr4.sin_addr))
return 0;
else if(1 == inet_pton(AF_INET6, str, &a.addr6.sin6_addr))
return 0;
return 1;
}
int DTCServers::SetRouteList(std::vector<ROUTE_NODE>& IPList)
{
if(IPList.empty())
{
return -ER_SET_IPLIST_NULL;
}
std::vector<ROUTE_NODE>().swap(this->m_IPList);
std::vector<ROUTE_NODE>::iterator it = IPList.begin();
for(; it != IPList.end(); ++it)
{
ROUTE_NODE ip;
ip.bid = it->bid;
if(it->port > 0 && it->port <= 65535){
ip.port = it->port;
}else{
SetErrorMsg(ER_PORT_OUT_RANGE, "DTCServers::SetRouteList", "port is out of range!");
return -ER_PORT_OUT_RANGE;
}
if(it->status == 0 || it->status == 1){
ip.status = it->status;
}else{
SetErrorMsg(ER_STATUS_ERROR_VALUE, "DTCServers::SetRouteList", "status is error value!");
return -ER_STATUS_ERROR_VALUE;
}
if(it->weight > 0){
ip.weight = it->weight;
}else{
SetErrorMsg(ER_WEIGHT_ERROR_VALUE, "DTCServers::SetRouteList", "weight is error value!");
return -ER_WEIGHT_ERROR_VALUE;
}
if(InvalidIpaddr(it->ip) == 0){
memcpy(ip.ip, it->ip, IP_LEN);
}else{
SetErrorMsg(ER_IP_ERROR_VALUE, "DTCServers::SetAccessKey", "ip is error value!");
return -ER_IP_ERROR_VALUE;
}
#ifdef DEBUG_INFO
PrintIPNode(ip);
#endif
this->m_IPList.push_back(ip);
}
this->m_SetRoute = true;
return 0;
}
void DTCServers::SetIDCNo(int IDCNo)
{
this->m_IDCNo = IDCNo;
}
int DTCServers::SetAccessKey(const char *token)
{
std::string str;
if(token == NULL)
return -EC_BAD_ACCESS_KEY;
else
str = token;
if(str.length() != ACCESS_KEY_LEN)
{
log_error("Invalid accessKey!");
this->m_AccessToken = "";
SetErrorMsg(EC_BAD_ACCESS_KEY, "DTCServers::SetAccessKey", "Invalid accessKey!");
return -EC_BAD_ACCESS_KEY;
}
else
this->m_AccessToken = str;
std::string stemp = str.substr(0, 8);
sscanf(stemp.c_str(), "%d", &(this->m_Bid));
return 0;
}
int DTCServers::SetTableName(const char *tableName)
{
if(tableName==NULL) return -DTC::EC_BAD_TABLE_NAME;
if(this->m_TableName)
return mystrcmp(tableName, this->m_TableName, 256)==0 ? 0 : -DTC::EC_BAD_TABLE_NAME;
this->m_TableName = STRDUP(tableName);
return 0;
}
int DTCServers::SetKeyType(int type)
{
switch(type)
{
case DField::Signed:
case DField::Unsigned:
case DField::Float:
case DField::String:
case DField::Binary:
this->m_KeyType = type;
break;
default:
return -DTC::EC_BAD_KEY_TYPE;
break;
}
return 0;
}
/*
int DTCServers::AddKey(const char* name, uint8_t type)
{
}
*/
void DTCServers::SetAgentTime(int t)
{
this->m_AgentTime = t;
}
void DTCServers::SetMTimeout(int n)
{
this->m_TimeOut = n<=0 ? 5000 : n;
}
int DTCServers::ConstructServers()
{
if(this->m_IPList.empty())
{
log_error("ip list is empty!");
SetErrorMsg(ER_SET_IPLIST_NULL, "DTCServers::ConstructServers", "ip list is empty!");
return -ER_SET_IPLIST_NULL;
}
if(this->m_AccessToken.empty() || (this->m_TableName == NULL) || (this->m_KeyType == 0))
{
log_error("m_AccessToken m_TableName or m_KeyType is unset");
SetErrorMsg(ER_SET_INSTANCE_PROPERTIES_ERR, "DTCServers::ConstructServers", "m_AccessToken m_TableName or m_KeyType is unset!");
return -ER_SET_INSTANCE_PROPERTIES_ERR;
}
int i = 0;
int ret = 0;
char tmpPort[7];
memset(tmpPort, 0, sizeof(tmpPort));
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
int IPCount = this->m_IPList.size();
this->m_QOSSevers = new DTCQosServer[IPCount];
for( ; i < IPCount; ++i)
{
this->m_QOSSevers[i].CreateDTCServer();
Server *server = this->m_QOSSevers[i].GetDTCServer();
sprintf(tmpPort, "%d", this->m_IPList[i].port);
ret = server->SetAddress(this->m_IPList[i].ip, tmpPort);
if(ret < 0)
{
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
return -ret;
}
this->m_QOSSevers[i].setWeight(this->m_IPList[i].weight);
this->m_QOSSevers[i].SetStatus(this->m_IPList[i].status);
ret = server->SetTableName(this->m_TableName);
if(ret < 0)
{
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
return -ret;
}
server->SetAccessKey(this->m_AccessToken.c_str());
switch(this->m_KeyType)
{
case DField::Signed:
case DField::Unsigned:
server->IntKey();
break;
case DField::String:
server->StringKey();
break;
case DField::Binary:
server->StringKey();
break;
case DField::Float:
default:
{
log_error("key type is wrong!");
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
SetErrorMsg(ER_KEY_TYPE, "DTCServers::ConstructServers", "key type is wrong!");
return -ER_KEY_TYPE;
}
break;
}
if(this->m_TimeOut > 0)
{
server->SetMTimeout(this->m_TimeOut);
}
}
return 0;
}
int DTCServers::IsServerHasExisted(ROUTE_NODE& ip)
{
int i;
for (i = 0; i < this->m_IPList.size(); i++)
{
if((ip.port == this->m_IPList[i].port) && (strncmp(ip.ip, this->m_IPList[i].ip, IP_LENGHT) == 0))
return i;
}
return -1;
}
int DTCServers::ConstructServers2(std::vector<ROUTE_NODE>& IPList)
{
if(IPList.empty())
{
log_error("ip list is empty!");
SetErrorMsg(ER_SET_IPLIST_NULL, "DTCServers::ConstructServers2", "ip list is empty!");
return -ER_SET_IPLIST_NULL;
}
if(this->m_AccessToken.empty() || (this->m_TableName == NULL) || (this->m_KeyType == 0))
{
log_error("m_AccessToken m_TableName or m_KeyType is unset");
SetErrorMsg(ER_SET_INSTANCE_PROPERTIES_ERR, "DTCServers::ConstructServers2", "m_AccessToken m_TableName or m_KeyType is unset!");
return -ER_SET_INSTANCE_PROPERTIES_ERR;
}
int i = 0;
int ret = 0;
char tmpPort[7];
memset(tmpPort, 0, sizeof(tmpPort));
int IPCount = IPList.size();
DTCQosServer* tmpQosServer = new DTCQosServer[IPCount];
for ( ; i < IPCount; i++)
{
int idx = IsServerHasExisted(IPList[i]);
if (idx >= 0){
tmpQosServer[i].setWeight(IPList[i].weight);
tmpQosServer[i].SetStatus(IPList[i].status);
tmpQosServer[i].SetRemoveTimeStamp(this->m_QOSSevers[idx].GetRemoveTimeStamp());
tmpQosServer[i].SetLastRemoveTime(this->m_QOSSevers[idx].GetLastRemoveTime());
tmpQosServer[i].SetMaxTimeStamp(this->m_QOSSevers[idx].GetMaxTimeStamp());
tmpQosServer[i].SetDTCServer(this->m_QOSSevers[idx].GetDTCServer());
this->m_QOSSevers[idx].ResetDTCServer();
} else {
tmpQosServer[i].CreateDTCServer();
Server *server = tmpQosServer[i].GetDTCServer();
sprintf(tmpPort, "%d", IPList[i].port);
ret = server->SetAddress(IPList[i].ip, tmpPort);
if(ret < 0)
{
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
delete[] tmpQosServer;
return -ret;
}
tmpQosServer[i].setWeight(IPList[i].weight);
tmpQosServer[i].SetStatus(IPList[i].status);
ret = server->SetTableName(this->m_TableName);
if(ret < 0)
{
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
delete[] tmpQosServer;
return -ret;
}
server->SetAccessKey(this->m_AccessToken.c_str());
switch(this->m_KeyType)
{
case DField::Signed:
case DField::Unsigned:
server->IntKey();
break;
case DField::String:
server->StringKey();
break;
case DField::Binary:
server->StringKey();
break;
case DField::Float:
default:
{
log_error("key type is wrong!");
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
delete[] tmpQosServer;
SetErrorMsg(ER_KEY_TYPE, "DTCServers::ConstructServers2", "key type is wrong!");
return -ER_KEY_TYPE;
}
break;
}
if(this->m_TimeOut > 0)
{
server->SetMTimeout(this->m_TimeOut);
}
}
}
std::vector<ROUTE_NODE>().swap(this->m_IPList);
if(this->m_QOSSevers)
{
delete[] this->m_QOSSevers;
this->m_QOSSevers = NULL;
}
this->m_QOSSevers = tmpQosServer;
tmpQosServer = NULL;
return 0;
}
void init_random()
{
uint64_t ticks;
struct timeval tv;
int fd;
gettimeofday(&tv, NULL);
ticks = tv.tv_sec + tv.tv_usec;
fd = open("/dev/urandom", O_RDONLY);
if (fd > 0)
{
uint64_t r;
int i;
for (i = 0; i <100 ; i++)
{
read(fd, &r, sizeof(r));
ticks += r;
}
close(fd);
}
srand(ticks);
}
unsigned int new_rand()
{
int fd;
unsigned int n = 0;
fd = open("/dev/urandom", O_RDONLY);
if(fd > 0)
{
read(fd, &n, sizeof(n));
}
close (fd);
return n;
}
void DTCServers::DisorderList(int *tmpBuckets, int size)
{
int randCount = 0;// 索引
unsigned int position = 0;// 位置
int k = 0;
// srand((int)time(0));
do{
int r = size - randCount;
// position = random(r);
init_random();
unsigned int tmp = new_rand();
position = tmp%r;
this->m_LoadBalanceBuckets[k++] = tmpBuckets[position];
randCount++;
// 将最后一位数值赋值给已经被使用的position
tmpBuckets[position] = tmpBuckets[r - 1];
}while(randCount < size);
return ;
}
int DTCServers::ConstructBalanceBuckets()
{
if(!this->m_QOSSevers || this->m_IPList.empty())
{
log_error("QOSSevers is null or ip count <0 ");
SetErrorMsg(ER_ROUTE_INFO_NULL, "DTCServers::ConstructBalanceBuckets", "QOSSevers is null or ip count <0!");
return -ER_ROUTE_INFO_NULL;
}
int i = 0;
int totalCount = 0;
int IPCount = this->m_IPList.size();
for( ; i < IPCount; ++i)
{
totalCount += this->m_QOSSevers[i].m_Weight;
}
FREE_IF(this->m_LoadBalanceBuckets);
this->m_LoadBalanceBuckets = (int *)malloc(sizeof(int) * totalCount);
this->m_BalanceBucketSize = totalCount;
int* tmpBuckets = (int *)malloc(sizeof(int) * totalCount);
int j = 0;
int pos = 0;
for( ; j < IPCount; ++j)
{
int k = 0;
for( ; k < this->m_QOSSevers[j].m_Weight; ++k)
{
tmpBuckets[pos] = j;
pos ++;
}
}
this->m_BucketsPos = 0;
DisorderList(tmpBuckets, totalCount);
// FREE_IF(tmpBuckets);
FREE_CLEAR(tmpBuckets);
return 0;
}
void DTCServers::RemoveServerFromBuckets(uint64_t now)
{
int i = 0;
int IPCount = this->m_IPList.size();
for( ; i < IPCount; ++i)
{
if(0 == this->m_QOSSevers[i].GetStatus())
{
continue;
}
uint64_t errorCount = this->m_QOSSevers[i].GetDTCServer()->GetErrCount();
if(errorCount >= DEFAULT_REMOVE_ERROR_COUNT)
{
this->m_QOSSevers[i].GetDTCServer()->IncRemoveCount();
uint64_t removeTime = this->m_QOSSevers[i].GetDTCServer()->GetRemoveCount() * DEFAULT_ROUTE_INTERVAL_TIME;
if(removeTime >= DEFAULT_MAX_REMOVE_THRESHOLD_TIME)
{
removeTime = DEFAULT_MAX_REMOVE_THRESHOLD_TIME;
}
this->m_QOSSevers[i].SetRemoveTimeStamp(removeTime);
this->m_QOSSevers[i].SetLastRemoveTime(now);
this->m_QOSSevers[i].SetStatus(0);
log_debug("bid=[%d], remove time=[%lu], now=[%lu], address=[%s], pos=[%d]!", \
this->m_Bid, removeTime, now, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), i);
#ifdef DEBUG_INFO
printf("remove bid=[%d],now=[%lu],remvoe timeStamp=[%lu],remove count=[%d],address=[%s]!\n", \
this->m_Bid, now, removeTime, this->m_QOSSevers[i].GetDTCServer()->GetRemoveCount(), \
this->m_QOSSevers[i].GetDTCServer()->GetAddress());
#endif
}
}
}
int DTCServers::RefreshBalanceBuckets(uint64_t now)
{
int i = 0;
int IPCount = this->m_IPList.size();
for( ; i < IPCount; ++i)
{
if(this->m_QOSSevers[i].GetStatus() == 0)
{
if(this->m_QOSSevers[i].GetLastRemoveTime() != 0)
{
if(now - this->m_QOSSevers[i].GetLastRemoveTime() > this->m_QOSSevers[i].GetRemoveTimeStamp())
{
this->m_QOSSevers[i].SetStatus(1);
this->m_QOSSevers[i].SetRemoveTimeStamp(0);
this->m_QOSSevers[i].SetLastRemoveTime(0);
this->m_QOSSevers[i].GetDTCServer()->ClearErrCount();
log_debug("bid=[%d], address=[%s], pos=[%d]!", \
this->m_Bid, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), i);
#ifdef DEBUG_INFO
printf("refresh bid=[%d],address=[%s],now=[%lu]!\n", \
this->m_Bid, this->m_QOSSevers[i].GetDTCServer()->GetAddress(), now);
#endif
}
}
}
}
return 0;
}
DTC::Server* DTCServers::GetOneServerFromBuckets()
{
int serverPos = 0;
int lastBucketsPos = this->m_BucketsPos;
do{
++this->m_BucketsPos;
if(this->m_BucketsPos >= this->m_BalanceBucketSize)
this->m_BucketsPos = 0;
if(unlikely(!this->m_LoadBalanceBuckets))
return NULL;
serverPos = this->m_LoadBalanceBuckets[this->m_BucketsPos];
if(this->m_QOSSevers[serverPos].GetStatus())
{
#ifdef DEBUG_INFO
printf("get server address=[%s]\n", this->m_QOSSevers[serverPos].GetDTCServer()->GetAddress());
#endif
return this->m_QOSSevers[serverPos].GetDTCServer();
}else{
//整租服务皆不可用，返回默认位置的server
if(lastBucketsPos == this->m_BucketsPos)
{
int tmpPos = this->m_LoadBalanceBuckets[DEFAULT_SERVER_POS];
return this->m_QOSSevers[tmpPos].GetDTCServer();
}
}
}while(1);
}
DTC::Server* DTCServers::GetServer()
{
int ret = 0;
struct timeval tv;
gettimeofday(&tv, NULL);
uint64_t now = tv.tv_sec;
if(this->m_SetRoute && (!this->m_IPList.empty()) && (!this->m_ConstructedBySetIPs))
{
#ifdef DEBUG_INFO
printf("construct server by set route\n");
#endif
ret = ConstructServers();
if(ret < 0)
{
log_error("construct servers by set route failed! error code=[%d]", ret);
SetErrorMsg(ret, "DTCServers::GetServer", "construct servers by set route failed!");
return NULL;
}
ret = ConstructBalanceBuckets();
if(ret < 0)
{
log_error("construct balance bucket failed! error code=[%d]", ret);
SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed");
return NULL;
}
this->m_ConstructedBySetIPs = true;
#ifdef DEBUG_INFO
printf("get one server from set route\n");
#endif
goto GetOneServer;
}
if(this->m_SetRoute && (!this->m_IPList.empty()) && this->m_ConstructedBySetIPs)
{
#ifdef DEBUG_INFO
printf("get one server from set route\n");
#endif
goto GetOneServer;
}
if(0 == this->m_BidVersion)
{
uint64_t BidVersion = 0;
int ret = get_version(&BidVersion);
if(BidVersion <= 0)
{
log_info("get version from CC error!");
// SetErrorMsg(ER_BID_VERSION_ERR, "DTCServers::GetServer", "get version from CC error!");
SetErrorMsg(ret, "DTCServers::GetServer", "get version from CC error!");
return NULL;
}
IP_ROUTE IPRoute;
memset(&IPRoute, 0, sizeof(IP_ROUTE));
ret = get_ip_route(this->m_Bid, &IPRoute);
if(ret < 0 || IPRoute.ip_num <= 0 || IPRoute.ip_list == NULL)
{
log_error("get ip list by bid failed! ip list is null!");
SetErrorMsg(0, "DTCServers::GetServer", "get ip list by bid failed! ip list is null!");
free_ip_route(&IPRoute);
return NULL;
}
int i = 0;
std::vector<ROUTE_NODE>().swap(this->m_IPList);
for(; i < IPRoute.ip_num; ++i)
{
ROUTE_NODE RouteNode;
RouteNode.bid = IPRoute.ip_list[i].bid;
memcpy(RouteNode.ip, IPRoute.ip_list[i].ip, IP_LEN);
RouteNode.port = IPRoute.ip_list[i].port;
RouteNode.status = IPRoute.ip_list[i].status;
RouteNode.weight = IPRoute.ip_list[i].weight;
#ifdef DEBUG_INFO
PrintIPNode(RouteNode);
#endif
this->m_IPList.push_back(RouteNode);
}
free_ip_route(&IPRoute);
#ifdef DEBUG_INFO
printf("construct server by cc\n");
#endif
ret = ConstructServers();
if(ret < 0)
{
log_error("construct servers failed!");
SetErrorMsg(ret, "DTCServers::GetServer", "construct servers failed!!");
return NULL;
}
//TODO cc version管理
ret = ConstructBalanceBuckets();
if(ret < 0)
{
log_error("construct balance bucket failed! error code=[%d]", ret);
SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed!");
return NULL;
}
#ifdef DEBUG_INFO
printf("get one server from cc\n");
#endif
this->m_LastGetCaTime = now;
this->m_BidVersion = BidVersion;
goto GetOneServer;
}
if(now - this->m_LastGetCaTime >= DEFAULT_ROUTE_EXPIRE_TIME)
{
uint64_t BidVersion = 0;
get_version(&BidVersion);
this->m_LastGetCaTime = now;
if(this->m_BidVersion >= BidVersion)
{
log_info("the version is lastest!");
#ifdef DEBUG_INFO
printf("get one server from cc\n");
#endif
goto GetOneServer;
}
IP_ROUTE IPRoute;
memset(&IPRoute, 0, sizeof(IP_ROUTE));
ret = get_ip_route(this->m_Bid, &IPRoute);
if(ret < 0 || IPRoute.ip_num <= 0 || IPRoute.ip_list == NULL)
{
log_error("get ip list by bid failed! ip list is null!");
SetErrorMsg(0, "DTCServers::GetServer", "get ip list by bid failed! ip list is null!");
free_ip_route(&IPRoute);
return NULL;
}
int i = 0;
std::vector<ROUTE_NODE> IPList;
for(; i < IPRoute.ip_num; ++i)
{
ROUTE_NODE RouteNode;
RouteNode.bid = IPRoute.ip_list[i].bid;
memcpy(RouteNode.ip, IPRoute.ip_list[i].ip, IP_LEN);
RouteNode.port = IPRoute.ip_list[i].port;
RouteNode.status = IPRoute.ip_list[i].status;
RouteNode.weight = IPRoute.ip_list[i].weight;
#ifdef DEBUG_INFO
PrintIPNode(RouteNode);
#endif
IPList.push_back(RouteNode);
}
free_ip_route(&IPRoute);
#ifdef DEBUG_INFO
printf("construct server by cc\n");
#endif
ret = ConstructServers2(IPList);
if(ret < 0)
{
std::vector<ROUTE_NODE>().swap(this->m_IPList);
this->m_IPList = IPList;
log_error("construct servers failed!");
SetErrorMsg(ret, "DTCServers::GetServer", "construct servers failed!");
return NULL;
}
this->m_IPList = IPList;
ret = ConstructBalanceBuckets();
if(ret < 0)
{
log_error("construct balance bucket failed! error code=[%d]", ret);
SetErrorMsg(ret, "DTCServers::GetServer", "construct balance bucket failed!");
return NULL;
}
this->m_BidVersion = BidVersion;
#ifdef DEBUG_INFO
printf("get one server from cc\n");
#endif
goto GetOneServer;
}else{
#ifdef DEBUG_INFO
printf("get one server from cc\n");
#endif
goto GetOneServer;
}
log_error("Exception process!");
SetErrorMsg(0, "DTCServers::GetServer", "Exception process!");
return NULL;
GetOneServer:
/* if(now - this->m_RefreshBucketsTime >= DEFAULT_REFRESH_BUCKETS_TIME)
{
RefreshBalanceBuckets(now);
this->m_RefreshBucketsTime = now;
}
if(now - this->m_RemoveBucketsTime >= DEFAULT_REMOVE_BUCKETS_TIME)
{
RemoveServerFromBuckets(now);
this->m_RemoveBucketsTime = now;
}
*/
RefreshBalanceBuckets(now);
RemoveServerFromBuckets(now);
return GetOneServerFromBuckets();
}

108
src/search_local/index_storage/api/c_api_cc/dtcqossvr.h Normal file
View File

@ -0,0 +1,108 @@
#ifndef __CHC_CLI_SERVERS_H
#define __CHC_CLI_SERVERS_H
#include "dtcint.h"
extern "C"
{
#include "app_client_set.h"
}
#define DEFAULT_ROUTE_EXPIRE_TIME 120
#define DEFAULT_MAX_REMOVE_THRESHOLD_TIME 1800
#define DEFAULT_REMOVE_ERROR_COUNT 3
#define DEFAULT_ROUTE_INTERVAL_TIME 5
#define DEFAULT_SERVER_POS 0
namespace DTC {
class DTCQosServer
{
public:
DTCQosServer();
~DTCQosServer(void);
private:
DTCQosServer(const DTCQosServer& qosServer);
DTCQosServer& operator=(const DTCQosServer& qosServer);
public:
int GetStatus()
{
return this->m_Status;
}
void SetStatus(int iFlag)
{
this->m_Status = iFlag;
}
int GetWeight()
{
return this->m_Weight;
}
void setWeight(int iFlag)
{
this->m_Weight = iFlag;
}
uint64_t GetRemoveTimeStamp()
{
return this->m_RemoveTimeStamp;
}
void SetRemoveTimeStamp(uint64_t iFlag)
{
this->m_RemoveTimeStamp = iFlag;
}
uint64_t GetLastRemoveTime()
{
return this->m_LastRemoveTime;
}
void SetLastRemoveTime(uint64_t iFlag)
{
this->m_LastRemoveTime = iFlag;
}
uint64_t GetMaxTimeStamp()
{
return this->m_MaxTimeStamp;
}
void SetMaxTimeStamp(uint64_t iFlag)
{
this->m_MaxTimeStamp = iFlag;
}
Server* GetDTCServer()
{
return this->m_Server;
}
void SetDTCServer(Server* ser)
{
this->m_Server = ser;
}
void ResetDTCServer()
{
this->m_Server = NULL;
}
void CreateDTCServer()
{
this->m_Server = new Server();
}
public:
friend class DTCServers;
private:
int m_Status;
int m_Weight;
uint64_t m_RemoveTimeStamp;//需要被摘除的时间
uint64_t m_LastRemoveTime; //上次被移除的时间点
uint64_t m_MaxTimeStamp;//最大指数阀值
Server *m_Server;
};
};
#endif

1120
src/search_local/index_storage/api/c_api_cc/dtcreq.cc Normal file
View File

File diff suppressed because it is too large Load Diff

1468
src/search_local/index_storage/api/c_api_cc/dtcsvr.cc Normal file
View File

File diff suppressed because it is too large Load Diff

1295
src/search_local/index_storage/api/c_api_cc/dtcwrap.cc Normal file
View File

File diff suppressed because it is too large Load Diff

231
src/search_local/index_storage/api/c_api_cc/dtcwrapp.cc Normal file
View File

@ -0,0 +1,231 @@
#include <inttypes.h>
#include <value.h>
#include <compiler.h>
#include "dtcapi.h"
#include "dtcpool.h"
using namespace DTC;
#define CAST(type, var) type *var = (type *)addr
#define CAST2(type, var, src) type *var = (type *)src->addr
#define CAST3(type, var, src) type *var = (type *)src.addr
// WRAPPER for Server
__EXPORT
ServerPool::ServerPool(int ms, int mr) {
NCPool *pl = new NCPool(ms, mr);
addr = pl;
}
__EXPORT
ServerPool::~ServerPool(void) {
CAST(NCPool, pl);
delete pl;
}
__EXPORT
int ServerPool::AddServer(Server *srv, int mreq, int mconn)
{
CAST(NCPool, pl);
CAST2(NCServer, s, srv);
return pl->AddServer(s, mreq, mconn);
}
__EXPORT
int ServerPool::AddRequest(Request *req, long long tag)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if(r->server == NULL)
return -EC_NOT_INITIALIZED;
return pl->AddRequest(r, tag, NULL);
}
__EXPORT
int ServerPool::AddRequest(Request *req, void *tag)
{
return AddRequest(req, (long long)(long)tag);
}
__EXPORT
int ServerPool::AddRequest(Request *req, long long tag, long long k)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if(r->server == NULL)
return -EC_NOT_INITIALIZED;
if(r->server->keytype != DField::Signed)
return -EC_BAD_KEY_TYPE;
DTCValue v(k);
return pl->AddRequest(r, tag, &v);
}
__EXPORT
int ServerPool::AddRequest(Request *req, void *tag, long long k)
{
return AddRequest(req, (long long)(long)tag, k);
}
__EXPORT
int ServerPool::AddRequest(Request *req, long long tag, const char *k)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if(r->server == NULL)
return -EC_NOT_INITIALIZED;
if(r->server->keytype != DField::String)
return -EC_BAD_KEY_TYPE;
DTCValue v(k);
return pl->AddRequest(r, tag, &v);
}
__EXPORT
int ServerPool::AddRequest(Request *req, void *tag, const char *k)
{
return AddRequest(req, (long long)(long)tag, k);
}
__EXPORT
int ServerPool::AddRequest(Request *req, long long tag, const char *k, int l)
{
CAST(NCPool, pl);
CAST2(NCRequest, r, req);
if(r->server == NULL)
return -EC_NOT_INITIALIZED;
if(r->server->keytype != DField::String)
return -EC_BAD_KEY_TYPE;
DTCValue v(k, l);
return pl->AddRequest(r, tag, &v);
}
__EXPORT
int ServerPool::AddRequest(Request *req, void *tag, const char *k, int l)
{
return AddRequest(req, (long long)(long)tag, k, l);
}
__EXPORT
int ServerPool::Execute(int msec)
{
CAST(NCPool, pl);
return pl->Execute(msec);
}
__EXPORT
int ServerPool::ExecuteAll(int msec)
{
CAST(NCPool, pl);
return pl->ExecuteAll(msec);
}
__EXPORT
int ServerPool::CancelRequest(int id)
{
CAST(NCPool, pl);
return pl->CancelRequest(id);
}
__EXPORT
int ServerPool::CancelAllRequest(int type)
{
CAST(NCPool, pl);
return pl->CancelAllRequest(type);
}
__EXPORT
int ServerPool::AbortRequest(int id)
{
CAST(NCPool, pl);
return pl->AbortRequest(id);
}
__EXPORT
int ServerPool::AbortAllRequest(int type)
{
CAST(NCPool, pl);
return pl->AbortAllRequest(type);
}
__EXPORT
Result * ServerPool::GetResult(void)
{
return GetResult(0);
}
__EXPORT
Result * ServerPool::GetResult(int id)
{
CAST(NCPool, pl);
NCResult *a = pl->GetResult(id);
if( (long)a < 4096 && (long)a > -4095)
return NULL;
Result *s = new Result();
s->addr = (void *)a;
return s;
}
__EXPORT
int ServerPool::GetResult(Result &s)
{
return GetResult(s, 0);
}
__EXPORT
int ServerPool::GetResult(Result &s, int id)
{
CAST(NCPool, pl);
s.Reset();
NCResult *a = pl->GetResult(id);
long iv = (long)a;
if( iv < 4096 && iv > -4095)
{
s.addr = (void *)(new NCResult(iv, "API::executing", iv>0 ? "Request not completed" : id ? "Invalid request" : "No more result"));
return iv;
}
s.addr = (void *)a;
return 0;
}
__EXPORT
int ServerPool::ServerCount(void) const
{
CAST(NCPool, pl);
return pl->ServerCount();
}
__EXPORT
int ServerPool::RequestCount(int type) const
{
CAST(NCPool, pl);
if(type == DONE)
return pl->DoneRequestCount();
if(type == (WAIT|SEND|RECV|DONE))
return pl->RequestCount();
return pl->CountRequestState(type);
}
__EXPORT
int ServerPool::RequestState(int reqId) const
{
CAST(NCPool, pl);
return pl->RequestState(reqId);
}
__EXPORT
int ServerPool::GetEpollFD(int size)
{
CAST(NCPool, pl);
return pl->GetEpollFD(size);
}
const int ALL_STATE = WAIT|SEND|RECV|DONE;

34
src/search_local/index_storage/api/c_api_cc/example/deletetest.cpp Normal file
View File

@ -0,0 +1,34 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
DTC::DeleteRequest deleteReq(&stServer);
uid = atoi(argv[1]);
deleteReq.SetKey(uid);
DTC::Result stResult;
retCode = deleteReq.Execute(stResult);
printf("retCode:%d\n", retCode);
if(retCode == 0)
{
printf("delete success!\n");
}
return 0;
}

77
src/search_local/index_storage/api/c_api_cc/example/inserttest.cpp Normal file
View File

@ -0,0 +1,77 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
unsigned int age;
std::string name;
std::string city;
std::string descr;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
DTC::InsertRequest insertReq(&stServer);
//retCode = insertReq.SetKey(key);
uid = atoi(argv[1]);
name = std::string(argv[2]);
city = std::string(argv[3]);
descr = std::string(argv[4]);
age = atoi(argv[5]);
insertReq.SetKey(uid);
//insertReq.Set("key", 100003);
insertReq.Set("uid", uid);
insertReq.Set("name", name.c_str());
insertReq.Set("city", city.c_str());
insertReq.Set("descr", descr.c_str());
insertReq.Set("age", age);
DTC::Result stResult;
retCode = insertReq.Execute(stResult);
printf("retCode:%d\n", retCode);
if(retCode == 0)
{
DTC::GetRequest getReq(&stServer);
getReq.SetKey(uid);
if(retCode == 0)
retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(retCode == 0)
retCode = getReq.Need("name");
if(retCode == 0)
retCode = getReq.Need("city");
if(retCode == 0)
retCode = getReq.Need("descr");
if(retCode == 0)
retCode = getReq.Need("age");
if(retCode != 0)
{
printf("get-req set key or need error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
retCode = getReq.Execute(stResult);
retCode = stResult.FetchRow();//开始获取数据
printf("uid:%d\n", stResult.IntValue("uid"));
printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据
printf("city: %s\n", stResult.StringValue("city"));
printf("descr: %s\n", stResult.BinaryValue("descr"));
printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据
}
return 0;
}

91
src/search_local/index_storage/api/c_api_cc/example/selecttest.cpp Normal file
View File

@ -0,0 +1,91 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int iRet;
unsigned int uid;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与在dtc.jd.com网站上申请dtc时指定的表名即tablename应该一样
stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
//下面三行代码是Get数据
unsigned int age;
std::string name;
std::string city;
std::string descr;
DTC::GetRequest stGetReq(&stServer);
uid = atoi(argv[1]);
iRet = stGetReq.SetKey(uid);
if(iRet == 0)
iRet = stGetReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(iRet == 0)
iRet = stGetReq.Need("name");
if(iRet == 0)
iRet = stGetReq.Need("city");
if(iRet == 0)
iRet = stGetReq.Need("descr");
if(iRet == 0)
iRet = stGetReq.Need("age");
if(iRet == 0)
iRet = stGetReq.GT("age", 24);
//iRet = stGetReq.OR("age", 27);
if(iRet != 0)
{
printf("get-req set key or need error: %d", iRet);
fflush(stdout);
return(-1);
}
// execute & get result
DTC::Result stResult;
iRet = stGetReq.Execute(stResult);
if(iRet != 0)
//如果出错,则输出错误码、错误阶段，错误信息，stResult.ErrorFrom(), stResult.ErrorMessage() 这两个错误信息很重要，一定要打印出来，方便定位问题
{
printf ("uin[%u] dtc execute get error: %d, error_from:%s, msg:%s\n",
uid,//出错的key是多少
iRet,//错误码为多少
stResult.ErrorFrom(),//返回错误阶段
stResult.ErrorMessage()//返回错误信息
);
fflush(stdout);
return(-2);
}
if(stResult.NumRows() <= 0)
{
// 数据不存在
printf("uin[%u] data not exist.\n", uid);
return(0);
}
printf("nubrow:%d\n", stResult.NumRows());
for(int i=0;i<=stResult.NumRows();++i)
{
iRet = stResult.FetchRow();//开始获取数据
if(iRet < 0)
{
printf ("uid[%lu] dtc fetch row error: %d\n", uid, iRet);
fflush(stdout);
return(-3);
}
//如果一切正确，则可以输出数据了
printf("uid: %d\n", stResult.IntValue("uid"));//输出int类型的数据
printf("name: %s\n", stResult.StringValue("name"));//输出string类型的数据
printf("city: %s\n", stResult.StringValue("city"));//输出string类型的数据
printf("descr: %s\n", stResult.BinaryValue("descr"));//输出binary类型的数据
printf("age:%d\n",stResult.IntValue("age"));
}
return(0);
}

890
src/search_local/index_storage/api/c_api_cc/example/ttcapi.h Normal file
View File

@ -0,0 +1,890 @@
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DTC {
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
//mem monior Request;2014/06/6;by seanzheng
const int RequestMonitor = 15;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int NodeHandleChange = 13;
const int Migrate = 14;
const int ReloadClusterNodeList = 15;
const int SetClusterNodeState = 16;
const int ChangeNodeAddress = 17;
const int GetClusterState = 18;
const int PurgeForHit = 19;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log (const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
void write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)));
class Result;
class Server {
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void CloneTabDef(const Server& source);
int SetAddress(const char *host, const char *port=0);
int SetTableName(const char *);
//for compress
void SetCompressLevel(int);
//get address and tablename set by user
const char * GetAddress(void) const;
const char * GetTableName(void) const;
//get address and tablename set by dtc frame,for plugin only;
const char * GetServerAddress(void) const;
const char * GetServerTableName(void) const;
int IntKey(void);
int BinaryKey(void);
int StringKey(void);
int AddKey(const char* name, int type);
int FieldType(const char* name);
const char *ErrorMessage(void) const;
void SetTimeout(int);
void SetMTimeout(int);
int Connect(void);
void Close(void);
int Ping(void);
void AutoPing(void);
void SetFD(int); // UNSUPPORTED API
void SetAutoUpdateTab(bool autoUpdate);
void SetAutoReconnect(int autoReconnect);
int DecodePacket(Result &, const char *, int);
int CheckPacketSize(const char *, int);
void SetAccessKey(const char *token);
};
class Request {
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int AttachServer(Server *srv);
void SetAdminCode(int code);
void SetHotBackupID(long long);
void SetMasterHBTimestamp(long long);
void SetSlaveHBTimestamp(long long);
#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int FieldType(const char*);
void NoCache(void);
void NoNextServer(void);
void Limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int OR(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//just for compress,only support binary field
int CompressSet(const char *, const char *, int);
//just compress and set. Don't need compressflag
int CompressSetForce(const char *, const char *, int);
//bits op
int SetMultiBits(const char *, int, int, unsigned int);
int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);}
int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);}
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(OR, unsigned long long);
_REDIR_(OR, long);
_REDIR_(OR, unsigned long);
_REDIR_(OR, int);
_REDIR_(OR, unsigned int);
_REDIR_(OR, short);
_REDIR_(OR, unsigned short);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void UnsetKey(void);
int SetKey(long long);
int SetKey(const char *);
int SetKey(const char *, int);
#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int AddKeyValue(const char* name, long long v);
int AddKeyValue(const char* name, const char *str);
int AddKeyValue(const char* name, const char *ptr, int len);
#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *Execute(void);
Result *Execute(long long);
Result *Execute(const char *);
Result *Execute(const char *, int);
#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int Execute(Result&);
int Execute(Result&, long long);
int Execute(Result&, const char *);
int Execute(Result&, const char *, int);
#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int EncodePacket(char *&, int&, long long&);
int EncodePacket(char *&, int&, long long&, long long);
int EncodePacket(char *&, int&, long long&, const char *);
int EncodePacket(char *&, int&, long long&, const char *, int);
#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int SetCacheID(long long);
#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);}
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
const char *ErrorMessage(void) const;
};
class GetRequest : public Request {
public:
GetRequest(Server *srv): Request(srv, RequestGet) {}
GetRequest(): Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request {
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request {
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request {
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request {
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest:public Request {
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request {
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest: public Request {
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest: public Request {
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class MonitorRequest: public Request {
public:
MonitorRequest(Server *srv) : Request(srv, RequestMonitor){}
};
class Result {
private:
void *addr;
long check;
Result(const Result &);
char *ServerInfo() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void SetError(int errcode, const char *from, const char *detail); // from will not dupped
int ResultCode(void) const;
const char *ErrorMessage(void) const;
const char *ErrorFrom(void) const;
long long HotBackupID() const;
long long MasterHBTimestamp() const;
long long SlaveHBTimestamp() const;
long long BinlogID() const;
long long BinlogOffset() const;
long long MemSize() const;
long long DataSize() const;
int NumRows(void) const;
int TotalRows(void) const;
int AffectedRows(void) const;
int NumFields(void) const;
const char* FieldName(int n) const;
int FieldPresent(const char* name) const;
int FieldType(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long ServerTimestamp(void) const;
long long InsertID(void) const;
long long IntKey(void) const;
const char *BinaryKey(void) const;
const char *BinaryKey(int *) const;
const char *BinaryKey(int &) const;
const char *StringKey(void) const;
const char *StringKey(int *) const;
const char *StringKey(int &) const;
long long IntValue(const char *) const;
double FloatValue(const char *) const;
const char *StringValue(const char *) const;
const char *StringValue(const char *, int*) const;
const char *StringValue(const char *, int&) const;
const char *BinaryValue(const char *) const;
const char *BinaryValue(const char *, int*) const;
const char *BinaryValue(const char *, int&) const;
int UnCompressBinaryValue(const char *name,char **buf,int *lenp);
//Uncompress Binary Value without check compressflag
int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp);
const char * UnCompressErrorMessage() const;
long long IntValue(int) const;
double FloatValue(int) const;
const char *StringValue(int) const;
const char *StringValue(int, int*) const;
const char *StringValue(int, int&) const;
const char *BinaryValue(int) const;
const char *BinaryValue(int, int*) const;
const char *BinaryValue(int, int&) const;
int FetchRow(void);
int Rewind(void);
};
class ServerPool {
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int GetEpollFD(int size);
int AddServer(Server *srv, int mReq=1, int mConn=0);
int AddRequest(Request *, long long);
int AddRequest(Request *, long long, long long);
int AddRequest(Request *, long long, const char *);
int AddRequest(Request *, long long, const char *, int);
int AddRequest(Request *, void *);
int AddRequest(Request *, void *, long long);
int AddRequest(Request *, void *, const char *);
int AddRequest(Request *, void *, const char *, int);
int Execute(int msec);
int ExecuteAll(int msec);
int CancelRequest(int);
int CancelAllRequest(int type);
int AbortRequest(int);
int AbortAllRequest(int type);
Result *GetResult(void);
Result *GetResult(int);
int GetResult(Result&);
int GetResult(Result&, int);
int ServerCount(void) const;
int RequestCount(int type) const;
int RequestState(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT|SEND|RECV|DONE;
enum {
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
};
enum {
ER_HASHCHK=1000,
ER_NISAMCHK=1001,
ER_NO=1002,
ER_YES=1003,
ER_CANT_CREATE_FILE=1004,
ER_CANT_CREATE_TABLE=1005,
ER_CANT_CREATE_DB=1006,
ER_DB_CREATE_EXISTS=1007,
ER_DB_DROP_EXISTS=1008,
ER_DB_DROP_DELETE=1009,
ER_DB_DROP_RMDIR=1010,
ER_CANT_DELETE_FILE=1011,
ER_CANT_FIND_SYSTEM_REC=1012,
ER_CANT_GET_STAT=1013,
ER_CANT_GET_WD=1014,
ER_CANT_LOCK=1015,
ER_CANT_OPEN_FILE=1016,
ER_FILE_NOT_FOUND=1017,
ER_CANT_READ_DIR=1018,
ER_CANT_SET_WD=1019,
ER_CHECKREAD=1020,
ER_DISK_FULL=1021,
ER_DUP_KEY=1022,
ER_ERROR_ON_CLOSE=1023,
ER_ERROR_ON_READ=1024,
ER_ERROR_ON_RENAME=1025,
ER_ERROR_ON_WRITE=1026,
ER_FILE_USED=1027,
ER_FILSORT_ABORT=1028,
ER_FORM_NOT_FOUND=1029,
ER_GET_ERRNO=1030,
ER_ILLEGAL_HA=1031,
ER_KEY_NOT_FOUND=1032,
ER_NOT_FORM_FILE=1033,
ER_NOT_KEYFILE=1034,
ER_OLD_KEYFILE=1035,
ER_OPEN_AS_READONLY=1036,
ER_OUTOFMEMORY=1037,
ER_OUT_OF_SORTMEMORY=1038,
ER_UNEXPECTED_EOF=1039,
ER_CON_COUNT_ERROR=1040,
ER_OUT_OF_RESOURCES=1041,
ER_BAD_HOST_ERROR=1042,
ER_HANDSHAKE_ERROR=1043,
ER_DBACCESS_DENIED_ERROR=1044,
ER_ACCESS_DENIED_ERROR=1045,
ER_NO_DB_ERROR=1046,
ER_UNKNOWN_COM_ERROR=1047,
ER_BAD_NULL_ERROR=1048,
ER_BAD_DB_ERROR=1049,
ER_TABLE_EXISTS_ERROR=1050,
ER_BAD_TABLE_ERROR=1051,
ER_NON_UNIQ_ERROR=1052,
ER_SERVER_SHUTDOWN=1053,
ER_BAD_FIELD_ERROR=1054,
ER_WRONG_FIELD_WITH_GROUP=1055,
ER_WRONG_GROUP_FIELD=1056,
ER_WRONG_SUM_SELECT=1057,
ER_WRONG_VALUE_COUNT=1058,
ER_TOO_LONG_IDENT=1059,
ER_DUP_FIELDNAME=1060,
ER_DUP_KEYNAME=1061,
ER_DUP_ENTRY=1062,
ER_WRONG_FIELD_SPEC=1063,
ER_PARSE_ERROR=1064,
ER_EMPTY_QUERY=1065,
ER_NONUNIQ_TABLE=1066,
ER_INVALID_DEFAULT=1067,
ER_MULTIPLE_PRI_KEY=1068,
ER_TOO_MANY_KEYS=1069,
ER_TOO_MANY_KEY_PARTS=1070,
ER_TOO_LONG_KEY=1071,
ER_KEY_COLUMN_DOES_NOT_EXITS=1072,
ER_BLOB_USED_AS_KEY=1073,
ER_TOO_BIG_FIELDLENGTH=1074,
ER_WRONG_AUTO_KEY=1075,
ER_READY=1076,
ER_NORMAL_SHUTDOWN=1077,
ER_GOT_SIGNAL=1078,
ER_SHUTDOWN_COMPLETE=1079,
ER_FORCING_CLOSE=1080,
ER_IPSOCK_ERROR=1081,
ER_NO_SUCH_INDEX=1082,
ER_WRONG_FIELD_TERMINATORS=1083,
ER_BLOBS_AND_NO_TERMINATED=1084,
ER_TEXTFILE_NOT_READABLE=1085,
ER_FILE_EXISTS_ERROR=1086,
ER_LOAD_INFO=1087,
ER_ALTER_INFO=1088,
ER_WRONG_SUB_KEY=1089,
ER_CANT_REMOVE_ALL_FIELDS=1090,
ER_CANT_DROP_FIELD_OR_KEY=1091,
ER_INSERT_INFO=1092,
ER_INSERT_TABLE_USED=1093,
ER_NO_SUCH_THREAD=1094,
ER_KILL_DENIED_ERROR=1095,
ER_NO_TABLES_USED=1096,
ER_TOO_BIG_SET=1097,
ER_NO_UNIQUE_LOGFILE=1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE=1099,
ER_TABLE_NOT_LOCKED=1100,
ER_BLOB_CANT_HAVE_DEFAULT=1101,
ER_WRONG_DB_NAME=1102,
ER_WRONG_TABLE_NAME=1103,
ER_TOO_BIG_SELECT=1104,
ER_UNKNOWN_ERROR=1105,
ER_UNKNOWN_PROCEDURE=1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE=1108,
ER_UNKNOWN_TABLE=1109,
ER_FIELD_SPECIFIED_TWICE=1110,
ER_INVALID_GROUP_FUNC_USE=1111,
ER_UNSUPPORTED_EXTENSION=1112,
ER_TABLE_MUST_HAVE_COLUMNS=1113,
ER_RECORD_FILE_FULL=1114,
ER_UNKNOWN_CHARACTER_SET=1115,
ER_TOO_MANY_TABLES=1116,
ER_TOO_MANY_FIELDS=1117,
ER_TOO_BIG_ROWSIZE=1118,
ER_STACK_OVERRUN=1119,
ER_WRONG_OUTER_JOIN=1120,
ER_NULL_COLUMN_IN_INDEX=1121,
ER_CANT_FIND_UDF=1122,
ER_CANT_INITIALIZE_UDF=1123,
ER_UDF_NO_PATHS=1124,
ER_UDF_EXISTS=1125,
ER_CANT_OPEN_LIBRARY=1126,
ER_CANT_FIND_DL_ENTRY=1127,
ER_FUNCTION_NOT_DEFINED=1128,
ER_HOST_IS_BLOCKED=1129,
ER_HOST_NOT_PRIVILEGED=1130,
ER_PASSWORD_ANONYMOUS_USER=1131,
ER_PASSWORD_NOT_ALLOWED=1132,
ER_PASSWORD_NO_MATCH=1133,
ER_UPDATE_INFO=1134,
ER_CANT_CREATE_THREAD=1135,
ER_WRONG_VALUE_COUNT_ON_ROW=1136,
ER_CANT_REOPEN_TABLE=1137,
ER_INVALID_USE_OF_NULL=1138,
ER_REGEXP_ERROR=1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140,
ER_NONEXISTING_GRANT=1141,
ER_TABLEACCESS_DENIED_ERROR=1142,
ER_COLUMNACCESS_DENIED_ERROR=1143,
ER_ILLEGAL_GRANT_FOR_TABLE=1144,
ER_GRANT_WRONG_HOST_OR_USER=1145,
ER_NO_SUCH_TABLE=1146,
ER_NONEXISTING_TABLE_GRANT=1147,
ER_NOT_ALLOWED_COMMAND=1148,
ER_SYNTAX_ERROR=1149,
ER_DELAYED_CANT_CHANGE_LOCK=1150,
ER_TOO_MANY_DELAYED_THREADS=1151,
ER_ABORTING_CONNECTION=1152,
ER_NET_PACKET_TOO_LARGE=1153,
ER_NET_READ_ERROR_FROM_PIPE=1154,
ER_NET_FCNTL_ERROR=1155,
ER_NET_PACKETS_OUT_OF_ORDER=1156,
ER_NET_UNCOMPRESS_ERROR=1157,
ER_NET_READ_ERROR=1158,
ER_NET_READ_INTERRUPTED=1159,
ER_NET_ERROR_ON_WRITE=1160,
ER_NET_WRITE_INTERRUPTED=1161,
ER_TOO_LONG_STRING=1162,
ER_TABLE_CANT_HANDLE_BLOB=1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164,
ER_DELAYED_INSERT_TABLE_LOCKED=1165,
ER_WRONG_COLUMN_NAME=1166,
ER_WRONG_KEY_COLUMN=1167,
ER_WRONG_MRG_TABLE=1168,
ER_DUP_UNIQUE=1169,
ER_BLOB_KEY_WITHOUT_LENGTH=1170,
ER_PRIMARY_CANT_HAVE_NULL=1171,
ER_TOO_MANY_ROWS=1172,
ER_REQUIRES_PRIMARY_KEY=1173,
ER_NO_RAID_COMPILED=1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175,
ER_KEY_DOES_NOT_EXITS=1176,
ER_CHECK_NO_SUCH_TABLE=1177,
ER_CHECK_NOT_IMPLEMENTED=1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179,
ER_ERROR_DURING_COMMIT=1180,
ER_ERROR_DURING_ROLLBACK=1181,
ER_ERROR_DURING_FLUSH_LOGS=1182,
ER_ERROR_DURING_CHECKPOINT=1183,
ER_NEW_ABORTING_CONNECTION=1184,
ER_DUMP_NOT_IMPLEMENTED= 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED=1186,
ER_INDEX_REBUILD= 1187,
ER_MASTER=1188,
ER_MASTER_NET_READ=1189,
ER_MASTER_NET_WRITE=1190,
ER_FT_MATCHING_KEY_NOT_FOUND=1191,
ER_LOCK_OR_ACTIVE_TRANSACTION=1192,
ER_UNKNOWN_SYSTEM_VARIABLE=1193,
ER_CRASHED_ON_USAGE=1194,
ER_CRASHED_ON_REPAIR=1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK=1196,
ER_TRANS_CACHE_FULL=1197,
ER_SLAVE_MUST_STOP=1198,
ER_SLAVE_NOT_RUNNING=1199,
ER_BAD_SLAVE=1200,
ER_MASTER_INFO=1201,
ER_SLAVE_THREAD=1202,
ER_TOO_MANY_USER_CONNECTIONS=1203,
ER_SET_CONSTANTS_ONLY=1204,
ER_LOCK_WAIT_TIMEOUT=1205,
ER_LOCK_TABLE_FULL=1206,
ER_READ_ONLY_TRANSACTION=1207,
ER_DROP_DB_WITH_READ_LOCK=1208,
ER_CREATE_DB_WITH_READ_LOCK=1209,
ER_WRONG_ARGUMENTS=1210,
ER_NO_PERMISSION_TO_CREATE_USER=1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR=1212,
ER_LOCK_DEADLOCK=1213,
ER_TABLE_CANT_HANDLE_FULLTEXT=1214,
ER_CANNOT_ADD_FOREIGN=1215,
ER_NO_REFERENCED_ROW=1216,
ER_ROW_IS_REFERENCED=1217,
ER_CONNECT_TO_MASTER=1218,
ER_QUERY_ON_MASTER=1219,
ER_ERROR_WHEN_EXECUTING_COMMAND=1220,
ER_WRONG_USAGE=1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222,
ER_CANT_UPDATE_WITH_READLOCK=1223,
ER_MIXING_NOT_ALLOWED=1224,
ER_DUP_ARGUMENT=1225,
ER_USER_LIMIT_REACHED=1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR=1227,
ER_LOCAL_VARIABLE=1228,
ER_GLOBAL_VARIABLE=1229,
ER_NO_DEFAULT=1230,
ER_WRONG_VALUE_FOR_VAR=1231,
ER_WRONG_TYPE_FOR_VAR=1232,
ER_VAR_CANT_BE_READ=1233,
ER_CANT_USE_OPTION_HERE=1234,
ER_NOT_SUPPORTED_YET=1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG=1236,
ER_SLAVE_IGNORED_TABLE=1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR=1238,
CR_UNKNOWN_ERROR=1900,
CR_SOCKET_CREATE_ERROR=1901,
CR_CONNECTION_ERROR=1902,
CR_CONN_HOST_ERROR=1903,
CR_IPSOCK_ERROR =1904,
CR_UNKNOWN_HOST =1905,
CR_SERVER_GONE_ERROR=1906,
CR_VERSION_ERROR=1907,
CR_OUT_OF_MEMORY=1908,
CR_WRONG_HOST_INFO=1909,
CR_LOCALHOST_CONNECTION=1910,
CR_TCP_CONNECTION=1911,
CR_SERVER_HANDSHAKE_ERR=1912,
CR_SERVER_LOST=1913,
CR_COMMANDS_OUT_OF_SYNC=1914,
CR_NAMEDPIPE_CONNECTION=1915,
CR_NAMEDPIPEWAIT_ERROR=1916,
CR_NAMEDPIPEOPEN_ERROR=1917,
CR_NAMEDPIPESETSTATE_ERROR=1918,
CR_CANT_READ_CHARSET=1919,
CR_NET_PACKET_TOO_LARGE=1920,
CR_EMBEDDED_CONNECTION=1921,
CR_PROBE_SLAVE_STATUS=1922,
CR_PROBE_SLAVE_HOSTS=1923,
CR_PROBE_SLAVE_CONNECT=1924,
CR_PROBE_MASTER_CONNECT=1925,
CR_SSL_CONNECTION_ERROR=1926,
CR_MALFORMED_PACKET=1927,
CR_WRONG_LICENSE=1928,
};
};
#endif

84
src/search_local/index_storage/api/c_api_cc/example/updatetest.cpp Normal file
View File

@ -0,0 +1,84 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
unsigned int age;
std::string name;
std::string city;
std::string descr;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
uid = atoi(argv[1]);
name = std::string(argv[2]);
city = std::string(argv[3]);
descr = std::string(argv[4]);
age = atoi(argv[5]);
DTC::UpdateRequest UpdateReq(&stServer);
retCode = UpdateReq.SetKey(uid);
if(retCode != 0)
{
printf("update-req set key error: %d", retCode);
fflush(stdout);
return(-1);
}
retCode = UpdateReq.Set("name", name.c_str());
retCode = UpdateReq.Set("city", city.c_str());
retCode = UpdateReq.Set("descr", descr.c_str());
retCode = UpdateReq.Set("age", age);
if(retCode != 0)
{
printf("update-req set field error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
DTC::Result stResult;
retCode = UpdateReq.Execute(stResult);
printf("retCode:%d\n", retCode);
if(retCode == 0)
{
DTC::GetRequest getReq(&stServer);
getReq.SetKey(uid);
if(retCode == 0)
retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(retCode == 0)
retCode = getReq.Need("name");
if(retCode == 0)
retCode = getReq.Need("city");
if(retCode == 0)
retCode = getReq.Need("descr");
if(retCode == 0)
retCode = getReq.Need("age");
if(retCode != 0)
{
printf("get-req set key or need error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
retCode = getReq.Execute(stResult);
retCode = stResult.FetchRow();//开始获取数据
printf("uid:%d\n", stResult.IntValue("uid"));
printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据
printf("city: %s\n", stResult.StringValue("city"));
printf("descr: %s\n", stResult.BinaryValue("descr"));
printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据
}
return 0;
}

32
src/search_local/index_storage/api/c_api_cc/examples/cache.conf Normal file
View File

@ -0,0 +1,32 @@
LogLevel = debug
BindAddr = 127.0.0.1:9898/tcp
IdleTimeout = 30
HelperTimeout = 30
CacheShmKey = 892123
CacheMemorySize = 100M
CacheAverageDataSize = 50
MaxFlushSpeed = 1
MinDirtyNode = 40
MaxDirtyNode = 60
MinDirtyTime = 3600
MaxDirtyTime = 43200
DisableDataSource = 0
DelayUpdate = 0
#opening file descritor overflow warning.
OpenningFDAttrID = 41784
StartStatReporter = 1
EnableCoreDump = 1
DisableWatchDog = 0
#ServerRecovery = crash
#[LogApi]
#MessageId =
#CallerId =
#TargetId =
#InterfaceId =

39
src/search_local/index_storage/api/c_api_cc/examples/delete/Makefile Normal file
View File

@ -0,0 +1,39 @@
LIB = ../../libdtc.so
OUTPUT = delete
SOURCES = deletetest.cpp
OBJECTS= deletetest.o
CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs
CFLAGS = $(CPPFLAGS)
all:$(OUTPUT)
.SUFFIXES: .o .cpp
.cpp.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^)
.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^
$(OUTPUT): $(OBJECTS)
$(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB)
clean:
rm -f *.o *.so *.~ *.bak
rm -f $(OUTPUT1)
rm -f $(OUTPUT2)
install:
cp -f $(OUTPUT) $(INSTALL_PATH)
dep:
mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend
ifneq ($(wildcard .depend),)
include .depend
endif

34
src/search_local/index_storage/api/c_api_cc/examples/delete/deletetest.cpp Normal file
View File

@ -0,0 +1,34 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
DTC::DeleteRequest deleteReq(&stServer);
uid = atoi(argv[1]);
deleteReq.SetKey(uid);
DTC::Result stResult;
retCode = deleteReq.Execute(stResult);
printf("retCode:%d\n", retCode);
if(retCode == 0)
{
printf("delete success!\n");
}
return 0;
}

839
src/search_local/index_storage/api/c_api_cc/examples/delete/ttcapi.h Normal file
View File

@ -0,0 +1,839 @@
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DTC {
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log (const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
void write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)));
class Result;
class Server {
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void CloneTabDef(const Server& source);
int SetAddress(const char *host, const char *port=0);
int SetTableName(const char *);
const char * GetAddress(void) const;
const char * GetTableName(void) const;
int IntKey(void);
int BinaryKey(void);
int StringKey(void);
int AddKey(const char* name, int type);
int FieldType(const char* name);
const char *ErrorMessage(void) const;
void SetTimeout(int);
void SetMTimeout(int);
int Connect(void);
void Close(void);
int Ping(void);
void AutoPing(void);
void SetFD(int); // UNSUPPORTED API
void SetAutoUpdateTab(bool autoUpdate);
void SetAutoReconnect(int autoReconnect);
int DecodePacket(Result &, const char *, int);
int CheckPacketSize(const char *, int);
};
class Request {
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int AttachServer(Server *srv);
void SetAdminCode(int code);
void SetHotBackupID(long long);
void SetMasterHBTimestamp(long long);
void SetSlaveHBTimestamp(long long);
#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int FieldType(const char*);
void NoCache(void);
void NoNextServer(void);
void Limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//bits op
int SetMultiBits(const char *, int, int, unsigned int);
int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);}
int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);}
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void UnsetKey(void);
int SetKey(long long);
int SetKey(const char *);
int SetKey(const char *, int);
#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int AddKeyValue(const char* name, long long v);
int AddKeyValue(const char* name, const char *str);
int AddKeyValue(const char* name, const char *ptr, int len);
#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *Execute(void);
Result *Execute(long long);
Result *Execute(const char *);
Result *Execute(const char *, int);
#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int Execute(Result&);
int Execute(Result&, long long);
int Execute(Result&, const char *);
int Execute(Result&, const char *, int);
#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int EncodePacket(char *&, int&, long long&);
int EncodePacket(char *&, int&, long long&, long long);
int EncodePacket(char *&, int&, long long&, const char *);
int EncodePacket(char *&, int&, long long&, const char *, int);
#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int SetCacheID(long long);
#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);}
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
};
class GetRequest : public Request {
public:
GetRequest(Server *srv): Request(srv, RequestGet) {}
GetRequest(): Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request {
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request {
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request {
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request {
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest:public Request {
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request {
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest: public Request {
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest: public Request {
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class Result {
private:
void *addr;
long check;
Result(const Result &);
char *ServerInfo() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void SetError(int errcode, const char *from, const char *detail); // from will not dupped
int ResultCode(void) const;
const char *ErrorMessage(void) const;
const char *ErrorFrom(void) const;
long long HotBackupID() const;
long long MasterHBTimestamp() const;
long long SlaveHBTimestamp() const;
long long BinlogID() const;
long long BinlogOffset() const;
int NumRows(void) const;
int TotalRows(void) const;
int AffectedRows(void) const;
int NumFields(void) const;
const char* FieldName(int n) const;
int FieldPresent(const char* name) const;
int FieldType(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long ServerTimestamp(void) const;
long long InsertID(void) const;
long long IntKey(void) const;
const char *BinaryKey(void) const;
const char *BinaryKey(int *) const;
const char *BinaryKey(int &) const;
const char *StringKey(void) const;
const char *StringKey(int *) const;
const char *StringKey(int &) const;
long long IntValue(const char *) const;
double FloatValue(const char *) const;
const char *StringValue(const char *) const;
const char *StringValue(const char *, int*) const;
const char *StringValue(const char *, int&) const;
const char *BinaryValue(const char *) const;
const char *BinaryValue(const char *, int*) const;
const char *BinaryValue(const char *, int&) const;
long long IntValue(int) const;
double FloatValue(int) const;
const char *StringValue(int) const;
const char *StringValue(int, int*) const;
const char *StringValue(int, int&) const;
const char *BinaryValue(int) const;
const char *BinaryValue(int, int*) const;
const char *BinaryValue(int, int&) const;
int FetchRow(void);
int Rewind(void);
};
class ServerPool {
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int GetEpollFD(int size);
int AddServer(Server *srv, int mReq=1, int mConn=0);
int AddRequest(Request *, long long);
int AddRequest(Request *, long long, long long);
int AddRequest(Request *, long long, const char *);
int AddRequest(Request *, long long, const char *, int);
int AddRequest(Request *, void *);
int AddRequest(Request *, void *, long long);
int AddRequest(Request *, void *, const char *);
int AddRequest(Request *, void *, const char *, int);
int Execute(int msec);
int ExecuteAll(int msec);
int CancelRequest(int);
int CancelAllRequest(int type);
int AbortRequest(int);
int AbortAllRequest(int type);
Result *GetResult(void);
Result *GetResult(int);
int GetResult(Result&);
int GetResult(Result&, int);
int ServerCount(void) const;
int RequestCount(int type) const;
int RequestState(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT|SEND|RECV|DONE;
enum {
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
};
enum {
ER_HASHCHK=1000,
ER_NISAMCHK=1001,
ER_NO=1002,
ER_YES=1003,
ER_CANT_CREATE_FILE=1004,
ER_CANT_CREATE_TABLE=1005,
ER_CANT_CREATE_DB=1006,
ER_DB_CREATE_EXISTS=1007,
ER_DB_DROP_EXISTS=1008,
ER_DB_DROP_DELETE=1009,
ER_DB_DROP_RMDIR=1010,
ER_CANT_DELETE_FILE=1011,
ER_CANT_FIND_SYSTEM_REC=1012,
ER_CANT_GET_STAT=1013,
ER_CANT_GET_WD=1014,
ER_CANT_LOCK=1015,
ER_CANT_OPEN_FILE=1016,
ER_FILE_NOT_FOUND=1017,
ER_CANT_READ_DIR=1018,
ER_CANT_SET_WD=1019,
ER_CHECKREAD=1020,
ER_DISK_FULL=1021,
ER_DUP_KEY=1022,
ER_ERROR_ON_CLOSE=1023,
ER_ERROR_ON_READ=1024,
ER_ERROR_ON_RENAME=1025,
ER_ERROR_ON_WRITE=1026,
ER_FILE_USED=1027,
ER_FILSORT_ABORT=1028,
ER_FORM_NOT_FOUND=1029,
ER_GET_ERRNO=1030,
ER_ILLEGAL_HA=1031,
ER_KEY_NOT_FOUND=1032,
ER_NOT_FORM_FILE=1033,
ER_NOT_KEYFILE=1034,
ER_OLD_KEYFILE=1035,
ER_OPEN_AS_READONLY=1036,
ER_OUTOFMEMORY=1037,
ER_OUT_OF_SORTMEMORY=1038,
ER_UNEXPECTED_EOF=1039,
ER_CON_COUNT_ERROR=1040,
ER_OUT_OF_RESOURCES=1041,
ER_BAD_HOST_ERROR=1042,
ER_HANDSHAKE_ERROR=1043,
ER_DBACCESS_DENIED_ERROR=1044,
ER_ACCESS_DENIED_ERROR=1045,
ER_NO_DB_ERROR=1046,
ER_UNKNOWN_COM_ERROR=1047,
ER_BAD_NULL_ERROR=1048,
ER_BAD_DB_ERROR=1049,
ER_TABLE_EXISTS_ERROR=1050,
ER_BAD_TABLE_ERROR=1051,
ER_NON_UNIQ_ERROR=1052,
ER_SERVER_SHUTDOWN=1053,
ER_BAD_FIELD_ERROR=1054,
ER_WRONG_FIELD_WITH_GROUP=1055,
ER_WRONG_GROUP_FIELD=1056,
ER_WRONG_SUM_SELECT=1057,
ER_WRONG_VALUE_COUNT=1058,
ER_TOO_LONG_IDENT=1059,
ER_DUP_FIELDNAME=1060,
ER_DUP_KEYNAME=1061,
ER_DUP_ENTRY=1062,
ER_WRONG_FIELD_SPEC=1063,
ER_PARSE_ERROR=1064,
ER_EMPTY_QUERY=1065,
ER_NONUNIQ_TABLE=1066,
ER_INVALID_DEFAULT=1067,
ER_MULTIPLE_PRI_KEY=1068,
ER_TOO_MANY_KEYS=1069,
ER_TOO_MANY_KEY_PARTS=1070,
ER_TOO_LONG_KEY=1071,
ER_KEY_COLUMN_DOES_NOT_EXITS=1072,
ER_BLOB_USED_AS_KEY=1073,
ER_TOO_BIG_FIELDLENGTH=1074,
ER_WRONG_AUTO_KEY=1075,
ER_READY=1076,
ER_NORMAL_SHUTDOWN=1077,
ER_GOT_SIGNAL=1078,
ER_SHUTDOWN_COMPLETE=1079,
ER_FORCING_CLOSE=1080,
ER_IPSOCK_ERROR=1081,
ER_NO_SUCH_INDEX=1082,
ER_WRONG_FIELD_TERMINATORS=1083,
ER_BLOBS_AND_NO_TERMINATED=1084,
ER_TEXTFILE_NOT_READABLE=1085,
ER_FILE_EXISTS_ERROR=1086,
ER_LOAD_INFO=1087,
ER_ALTER_INFO=1088,
ER_WRONG_SUB_KEY=1089,
ER_CANT_REMOVE_ALL_FIELDS=1090,
ER_CANT_DROP_FIELD_OR_KEY=1091,
ER_INSERT_INFO=1092,
ER_INSERT_TABLE_USED=1093,
ER_NO_SUCH_THREAD=1094,
ER_KILL_DENIED_ERROR=1095,
ER_NO_TABLES_USED=1096,
ER_TOO_BIG_SET=1097,
ER_NO_UNIQUE_LOGFILE=1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE=1099,
ER_TABLE_NOT_LOCKED=1100,
ER_BLOB_CANT_HAVE_DEFAULT=1101,
ER_WRONG_DB_NAME=1102,
ER_WRONG_TABLE_NAME=1103,
ER_TOO_BIG_SELECT=1104,
ER_UNKNOWN_ERROR=1105,
ER_UNKNOWN_PROCEDURE=1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE=1108,
ER_UNKNOWN_TABLE=1109,
ER_FIELD_SPECIFIED_TWICE=1110,
ER_INVALID_GROUP_FUNC_USE=1111,
ER_UNSUPPORTED_EXTENSION=1112,
ER_TABLE_MUST_HAVE_COLUMNS=1113,
ER_RECORD_FILE_FULL=1114,
ER_UNKNOWN_CHARACTER_SET=1115,
ER_TOO_MANY_TABLES=1116,
ER_TOO_MANY_FIELDS=1117,
ER_TOO_BIG_ROWSIZE=1118,
ER_STACK_OVERRUN=1119,
ER_WRONG_OUTER_JOIN=1120,
ER_NULL_COLUMN_IN_INDEX=1121,
ER_CANT_FIND_UDF=1122,
ER_CANT_INITIALIZE_UDF=1123,
ER_UDF_NO_PATHS=1124,
ER_UDF_EXISTS=1125,
ER_CANT_OPEN_LIBRARY=1126,
ER_CANT_FIND_DL_ENTRY=1127,
ER_FUNCTION_NOT_DEFINED=1128,
ER_HOST_IS_BLOCKED=1129,
ER_HOST_NOT_PRIVILEGED=1130,
ER_PASSWORD_ANONYMOUS_USER=1131,
ER_PASSWORD_NOT_ALLOWED=1132,
ER_PASSWORD_NO_MATCH=1133,
ER_UPDATE_INFO=1134,
ER_CANT_CREATE_THREAD=1135,
ER_WRONG_VALUE_COUNT_ON_ROW=1136,
ER_CANT_REOPEN_TABLE=1137,
ER_INVALID_USE_OF_NULL=1138,
ER_REGEXP_ERROR=1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140,
ER_NONEXISTING_GRANT=1141,
ER_TABLEACCESS_DENIED_ERROR=1142,
ER_COLUMNACCESS_DENIED_ERROR=1143,
ER_ILLEGAL_GRANT_FOR_TABLE=1144,
ER_GRANT_WRONG_HOST_OR_USER=1145,
ER_NO_SUCH_TABLE=1146,
ER_NONEXISTING_TABLE_GRANT=1147,
ER_NOT_ALLOWED_COMMAND=1148,
ER_SYNTAX_ERROR=1149,
ER_DELAYED_CANT_CHANGE_LOCK=1150,
ER_TOO_MANY_DELAYED_THREADS=1151,
ER_ABORTING_CONNECTION=1152,
ER_NET_PACKET_TOO_LARGE=1153,
ER_NET_READ_ERROR_FROM_PIPE=1154,
ER_NET_FCNTL_ERROR=1155,
ER_NET_PACKETS_OUT_OF_ORDER=1156,
ER_NET_UNCOMPRESS_ERROR=1157,
ER_NET_READ_ERROR=1158,
ER_NET_READ_INTERRUPTED=1159,
ER_NET_ERROR_ON_WRITE=1160,
ER_NET_WRITE_INTERRUPTED=1161,
ER_TOO_LONG_STRING=1162,
ER_TABLE_CANT_HANDLE_BLOB=1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164,
ER_DELAYED_INSERT_TABLE_LOCKED=1165,
ER_WRONG_COLUMN_NAME=1166,
ER_WRONG_KEY_COLUMN=1167,
ER_WRONG_MRG_TABLE=1168,
ER_DUP_UNIQUE=1169,
ER_BLOB_KEY_WITHOUT_LENGTH=1170,
ER_PRIMARY_CANT_HAVE_NULL=1171,
ER_TOO_MANY_ROWS=1172,
ER_REQUIRES_PRIMARY_KEY=1173,
ER_NO_RAID_COMPILED=1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175,
ER_KEY_DOES_NOT_EXITS=1176,
ER_CHECK_NO_SUCH_TABLE=1177,
ER_CHECK_NOT_IMPLEMENTED=1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179,
ER_ERROR_DURING_COMMIT=1180,
ER_ERROR_DURING_ROLLBACK=1181,
ER_ERROR_DURING_FLUSH_LOGS=1182,
ER_ERROR_DURING_CHECKPOINT=1183,
ER_NEW_ABORTING_CONNECTION=1184,
ER_DUMP_NOT_IMPLEMENTED= 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED=1186,
ER_INDEX_REBUILD= 1187,
ER_MASTER=1188,
ER_MASTER_NET_READ=1189,
ER_MASTER_NET_WRITE=1190,
ER_FT_MATCHING_KEY_NOT_FOUND=1191,
ER_LOCK_OR_ACTIVE_TRANSACTION=1192,
ER_UNKNOWN_SYSTEM_VARIABLE=1193,
ER_CRASHED_ON_USAGE=1194,
ER_CRASHED_ON_REPAIR=1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK=1196,
ER_TRANS_CACHE_FULL=1197,
ER_SLAVE_MUST_STOP=1198,
ER_SLAVE_NOT_RUNNING=1199,
ER_BAD_SLAVE=1200,
ER_MASTER_INFO=1201,
ER_SLAVE_THREAD=1202,
ER_TOO_MANY_USER_CONNECTIONS=1203,
ER_SET_CONSTANTS_ONLY=1204,
ER_LOCK_WAIT_TIMEOUT=1205,
ER_LOCK_TABLE_FULL=1206,
ER_READ_ONLY_TRANSACTION=1207,
ER_DROP_DB_WITH_READ_LOCK=1208,
ER_CREATE_DB_WITH_READ_LOCK=1209,
ER_WRONG_ARGUMENTS=1210,
ER_NO_PERMISSION_TO_CREATE_USER=1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR=1212,
ER_LOCK_DEADLOCK=1213,
ER_TABLE_CANT_HANDLE_FULLTEXT=1214,
ER_CANNOT_ADD_FOREIGN=1215,
ER_NO_REFERENCED_ROW=1216,
ER_ROW_IS_REFERENCED=1217,
ER_CONNECT_TO_MASTER=1218,
ER_QUERY_ON_MASTER=1219,
ER_ERROR_WHEN_EXECUTING_COMMAND=1220,
ER_WRONG_USAGE=1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222,
ER_CANT_UPDATE_WITH_READLOCK=1223,
ER_MIXING_NOT_ALLOWED=1224,
ER_DUP_ARGUMENT=1225,
ER_USER_LIMIT_REACHED=1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR=1227,
ER_LOCAL_VARIABLE=1228,
ER_GLOBAL_VARIABLE=1229,
ER_NO_DEFAULT=1230,
ER_WRONG_VALUE_FOR_VAR=1231,
ER_WRONG_TYPE_FOR_VAR=1232,
ER_VAR_CANT_BE_READ=1233,
ER_CANT_USE_OPTION_HERE=1234,
ER_NOT_SUPPORTED_YET=1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG=1236,
ER_SLAVE_IGNORED_TABLE=1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR=1238,
CR_UNKNOWN_ERROR=1900,
CR_SOCKET_CREATE_ERROR=1901,
CR_CONNECTION_ERROR=1902,
CR_CONN_HOST_ERROR=1903,
CR_IPSOCK_ERROR =1904,
CR_UNKNOWN_HOST =1905,
CR_SERVER_GONE_ERROR=1906,
CR_VERSION_ERROR=1907,
CR_OUT_OF_MEMORY=1908,
CR_WRONG_HOST_INFO=1909,
CR_LOCALHOST_CONNECTION=1910,
CR_TCP_CONNECTION=1911,
CR_SERVER_HANDSHAKE_ERR=1912,
CR_SERVER_LOST=1913,
CR_COMMANDS_OUT_OF_SYNC=1914,
CR_NAMEDPIPE_CONNECTION=1915,
CR_NAMEDPIPEWAIT_ERROR=1916,
CR_NAMEDPIPEOPEN_ERROR=1917,
CR_NAMEDPIPESETSTATE_ERROR=1918,
CR_CANT_READ_CHARSET=1919,
CR_NET_PACKET_TOO_LARGE=1920,
CR_EMBEDDED_CONNECTION=1921,
CR_PROBE_SLAVE_STATUS=1922,
CR_PROBE_SLAVE_HOSTS=1923,
CR_PROBE_SLAVE_CONNECT=1924,
CR_PROBE_MASTER_CONNECT=1925,
CR_SSL_CONNECTION_ERROR=1926,
CR_MALFORMED_PACKET=1927,
CR_WRONG_LICENSE=1928,
};
};
#endif

38
src/search_local/index_storage/api/c_api_cc/examples/insert/Makefile Normal file
View File

@ -0,0 +1,38 @@
LIB = ../../libdtc.so
OUTPUT = insert
SOURCES = inserttest.cpp
OBJECTS= inserttest.o
CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs
CFLAGS = $(CPPFLAGS)
all:$(OUTPUT)
.SUFFIXES: .o .cpp
.cpp.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^)
.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^
$(OUTPUT): $(OBJECTS)
$(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB)
clean:
rm -f *.o *.so *.~ *.bak
rm -f $(OUTPUT)
install:
cp -f $(OUTPUT) $(INSTALL_PATH)
dep:
mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend
ifneq ($(wildcard .depend),)
include .depend
endif

164
src/search_local/index_storage/api/c_api_cc/examples/insert/inserttest.cpp Normal file
View File

@ -0,0 +1,164 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
unsigned int age;
unsigned int lv = 0;
std::string name;
std::string city;
std::string descr;
unsigned int salary;
unsigned int uid1;
unsigned int age1;
std::string name1;
std::string city1;
std::string descr1;
unsigned int salary1;
lv = atoi(argv[1]);
DTC::init_log("dtc_client");
DTC::set_log_level(lv);
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10901");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000090184d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
DTC::InsertRequest insertReq(&stServer);
//retCode = insertReq.SetKey(key);
uid = atoi(argv[2]);
name = std::string(argv[3]);
city = std::string(argv[4]);
descr = std::string(argv[5]);
age = atoi(argv[6]);
salary = atoi(argv[7]);
insertReq.SetKey(uid);
//insertReq.Set("key", 100003);
insertReq.Set("uid", uid);
insertReq.Set("name", name.c_str());
insertReq.Set("city", city.c_str());
insertReq.Set("descr", descr.c_str());
insertReq.Set("age", age);
insertReq.Set("salary", salary);
DTC::Result stResult;
retCode = insertReq.Execute(stResult);
printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult.ErrorMessage(), stResult.ErrorFrom());
if(retCode == 0)
{
DTC::GetRequest getReq(&stServer);
getReq.SetKey(uid);
if(retCode == 0)
retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(retCode == 0)
retCode = getReq.Need("name");
if(retCode == 0)
retCode = getReq.Need("city");
if(retCode == 0)
retCode = getReq.Need("descr");
if(retCode == 0)
retCode = getReq.Need("age");
if(retCode == 0)
retCode = getReq.Need("salary");
if(retCode != 0)
{
printf("get-req set key or need error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
stResult.Reset();
retCode = getReq.Execute(stResult);
printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult.ErrorMessage(), stResult.ErrorFrom());
retCode = stResult.FetchRow();//开始获取数据
printf("uid:%lu\n", stResult.IntValue("uid"));
printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据
printf("city: %s\n", stResult.StringValue("city"));
printf("descr: %s\n", stResult.BinaryValue("descr"));
printf("age: %lu\n", stResult.IntValue("age"));//输出int类型的数据
printf("salary: %lu\n", stResult.IntValue("salary"));//输出int类型的数据
}
printf("-------------first request end -----------------\n");
printf("-------------second request begin---------------\n");
DTC::Server stServer1; // 只要server不析构，后台会保持长连接
stServer1.IntKey(); // 声明key类型
stServer1.SetTableName("tp1");//设置dtc的表名，与table.conf中tablename应该一样
stServer1.SetAddress("192.168.214.62", "10201");//设置的dtc的ip和端口
stServer1.SetTimeout(5); // 设置网络超时时间
stServer1.SetAccessKey("0000020184d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
DTC::InsertRequest insertReq1(&stServer1);
//retCode = insertReq.SetKey(key);
uid1 = atoi(argv[8]);
name1 = std::string(argv[9]);
city1 = std::string(argv[10]);
descr1 = std::string(argv[11]);
age1 = atoi(argv[12]);
salary1 = atoi(argv[13]);
insertReq1.SetKey(uid1);
//insertReq.Set("key", 100003);
insertReq1.Set("uid", uid1);
insertReq1.Set("name", name1.c_str());
insertReq1.Set("city", city1.c_str());
insertReq1.Set("descr", descr1.c_str());
insertReq1.Set("age", age1);
insertReq1.Set("salary", salary1);
DTC::Result stResult1;
retCode = insertReq1.Execute(stResult1);
printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult1.ErrorMessage(), stResult1.ErrorFrom());
if(retCode == 0)
{
DTC::GetRequest getReq1(&stServer1);
getReq1.SetKey(uid);
if(retCode == 0)
retCode = getReq1.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(retCode == 0)
retCode = getReq1.Need("name");
if(retCode == 0)
retCode = getReq1.Need("city");
if(retCode == 0)
retCode = getReq1.Need("descr");
if(retCode == 0)
retCode = getReq1.Need("age");
if(retCode == 0)
retCode = getReq1.Need("salary");
if(retCode != 0)
{
printf("get-req set key or need error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
stResult1.Reset();
retCode = getReq1.Execute(stResult1);
printf("retCode:%d, errmsg:%s, errfrom:%s\n", retCode, stResult1.ErrorMessage(), stResult1.ErrorFrom());
retCode = stResult1.FetchRow();//开始获取数据
printf("uid:%lu\n", stResult1.IntValue("uid"));
printf("name: %s\n", stResult1.StringValue("name"));//输出binary类型的数据
printf("city: %s\n", stResult1.StringValue("city"));
printf("descr: %s\n", stResult1.BinaryValue("descr"));
printf("age: %lu\n", stResult1.IntValue("age"));//输出int类型的数据
printf("salary: %lu\n", stResult1.IntValue("salary"));//输出int类型的数据
}
return 0;
}

890
src/search_local/index_storage/api/c_api_cc/examples/insert/ttcapi.h Normal file
View File

@ -0,0 +1,890 @@
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DTC {
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
//mem monior Request;2014/06/6;by seanzheng
const int RequestMonitor = 15;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int NodeHandleChange = 13;
const int Migrate = 14;
const int ReloadClusterNodeList = 15;
const int SetClusterNodeState = 16;
const int ChangeNodeAddress = 17;
const int GetClusterState = 18;
const int PurgeForHit = 19;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log (const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
void write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)));
class Result;
class Server {
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void CloneTabDef(const Server& source);
int SetAddress(const char *host, const char *port=0);
int SetTableName(const char *);
//for compress
void SetCompressLevel(int);
//get address and tablename set by user
const char * GetAddress(void) const;
const char * GetTableName(void) const;
//get address and tablename set by dtc frame,for plugin only;
const char * GetServerAddress(void) const;
const char * GetServerTableName(void) const;
int IntKey(void);
int BinaryKey(void);
int StringKey(void);
int AddKey(const char* name, int type);
int FieldType(const char* name);
const char *ErrorMessage(void) const;
void SetTimeout(int);
void SetMTimeout(int);
int Connect(void);
void Close(void);
int Ping(void);
void AutoPing(void);
void SetFD(int); // UNSUPPORTED API
void SetAutoUpdateTab(bool autoUpdate);
void SetAutoReconnect(int autoReconnect);
int DecodePacket(Result &, const char *, int);
int CheckPacketSize(const char *, int);
void SetAccessKey(const char *token);
};
class Request {
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int AttachServer(Server *srv);
void SetAdminCode(int code);
void SetHotBackupID(long long);
void SetMasterHBTimestamp(long long);
void SetSlaveHBTimestamp(long long);
#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int FieldType(const char*);
void NoCache(void);
void NoNextServer(void);
void Limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int OR(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//just for compress,only support binary field
int CompressSet(const char *, const char *, int);
//just compress and set. Don't need compressflag
int CompressSetForce(const char *, const char *, int);
//bits op
int SetMultiBits(const char *, int, int, unsigned int);
int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);}
int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);}
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(OR, unsigned long long);
_REDIR_(OR, long);
_REDIR_(OR, unsigned long);
_REDIR_(OR, int);
_REDIR_(OR, unsigned int);
_REDIR_(OR, short);
_REDIR_(OR, unsigned short);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void UnsetKey(void);
int SetKey(long long);
int SetKey(const char *);
int SetKey(const char *, int);
#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int AddKeyValue(const char* name, long long v);
int AddKeyValue(const char* name, const char *str);
int AddKeyValue(const char* name, const char *ptr, int len);
#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *Execute(void);
Result *Execute(long long);
Result *Execute(const char *);
Result *Execute(const char *, int);
#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int Execute(Result&);
int Execute(Result&, long long);
int Execute(Result&, const char *);
int Execute(Result&, const char *, int);
#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int EncodePacket(char *&, int&, long long&);
int EncodePacket(char *&, int&, long long&, long long);
int EncodePacket(char *&, int&, long long&, const char *);
int EncodePacket(char *&, int&, long long&, const char *, int);
#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int SetCacheID(long long);
#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);}
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
const char *ErrorMessage(void) const;
};
class GetRequest : public Request {
public:
GetRequest(Server *srv): Request(srv, RequestGet) {}
GetRequest(): Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request {
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request {
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request {
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request {
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest:public Request {
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request {
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest: public Request {
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest: public Request {
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class MonitorRequest: public Request {
public:
MonitorRequest(Server *srv) : Request(srv, RequestMonitor){}
};
class Result {
private:
void *addr;
long check;
Result(const Result &);
char *ServerInfo() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void SetError(int errcode, const char *from, const char *detail); // from will not dupped
int ResultCode(void) const;
const char *ErrorMessage(void) const;
const char *ErrorFrom(void) const;
long long HotBackupID() const;
long long MasterHBTimestamp() const;
long long SlaveHBTimestamp() const;
long long BinlogID() const;
long long BinlogOffset() const;
long long MemSize() const;
long long DataSize() const;
int NumRows(void) const;
int TotalRows(void) const;
int AffectedRows(void) const;
int NumFields(void) const;
const char* FieldName(int n) const;
int FieldPresent(const char* name) const;
int FieldType(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long ServerTimestamp(void) const;
long long InsertID(void) const;
long long IntKey(void) const;
const char *BinaryKey(void) const;
const char *BinaryKey(int *) const;
const char *BinaryKey(int &) const;
const char *StringKey(void) const;
const char *StringKey(int *) const;
const char *StringKey(int &) const;
long long IntValue(const char *) const;
double FloatValue(const char *) const;
const char *StringValue(const char *) const;
const char *StringValue(const char *, int*) const;
const char *StringValue(const char *, int&) const;
const char *BinaryValue(const char *) const;
const char *BinaryValue(const char *, int*) const;
const char *BinaryValue(const char *, int&) const;
int UnCompressBinaryValue(const char *name,char **buf,int *lenp);
//Uncompress Binary Value without check compressflag
int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp);
const char * UnCompressErrorMessage() const;
long long IntValue(int) const;
double FloatValue(int) const;
const char *StringValue(int) const;
const char *StringValue(int, int*) const;
const char *StringValue(int, int&) const;
const char *BinaryValue(int) const;
const char *BinaryValue(int, int*) const;
const char *BinaryValue(int, int&) const;
int FetchRow(void);
int Rewind(void);
};
class ServerPool {
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int GetEpollFD(int size);
int AddServer(Server *srv, int mReq=1, int mConn=0);
int AddRequest(Request *, long long);
int AddRequest(Request *, long long, long long);
int AddRequest(Request *, long long, const char *);
int AddRequest(Request *, long long, const char *, int);
int AddRequest(Request *, void *);
int AddRequest(Request *, void *, long long);
int AddRequest(Request *, void *, const char *);
int AddRequest(Request *, void *, const char *, int);
int Execute(int msec);
int ExecuteAll(int msec);
int CancelRequest(int);
int CancelAllRequest(int type);
int AbortRequest(int);
int AbortAllRequest(int type);
Result *GetResult(void);
Result *GetResult(int);
int GetResult(Result&);
int GetResult(Result&, int);
int ServerCount(void) const;
int RequestCount(int type) const;
int RequestState(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT|SEND|RECV|DONE;
enum {
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
};
enum {
ER_HASHCHK=1000,
ER_NISAMCHK=1001,
ER_NO=1002,
ER_YES=1003,
ER_CANT_CREATE_FILE=1004,
ER_CANT_CREATE_TABLE=1005,
ER_CANT_CREATE_DB=1006,
ER_DB_CREATE_EXISTS=1007,
ER_DB_DROP_EXISTS=1008,
ER_DB_DROP_DELETE=1009,
ER_DB_DROP_RMDIR=1010,
ER_CANT_DELETE_FILE=1011,
ER_CANT_FIND_SYSTEM_REC=1012,
ER_CANT_GET_STAT=1013,
ER_CANT_GET_WD=1014,
ER_CANT_LOCK=1015,
ER_CANT_OPEN_FILE=1016,
ER_FILE_NOT_FOUND=1017,
ER_CANT_READ_DIR=1018,
ER_CANT_SET_WD=1019,
ER_CHECKREAD=1020,
ER_DISK_FULL=1021,
ER_DUP_KEY=1022,
ER_ERROR_ON_CLOSE=1023,
ER_ERROR_ON_READ=1024,
ER_ERROR_ON_RENAME=1025,
ER_ERROR_ON_WRITE=1026,
ER_FILE_USED=1027,
ER_FILSORT_ABORT=1028,
ER_FORM_NOT_FOUND=1029,
ER_GET_ERRNO=1030,
ER_ILLEGAL_HA=1031,
ER_KEY_NOT_FOUND=1032,
ER_NOT_FORM_FILE=1033,
ER_NOT_KEYFILE=1034,
ER_OLD_KEYFILE=1035,
ER_OPEN_AS_READONLY=1036,
ER_OUTOFMEMORY=1037,
ER_OUT_OF_SORTMEMORY=1038,
ER_UNEXPECTED_EOF=1039,
ER_CON_COUNT_ERROR=1040,
ER_OUT_OF_RESOURCES=1041,
ER_BAD_HOST_ERROR=1042,
ER_HANDSHAKE_ERROR=1043,
ER_DBACCESS_DENIED_ERROR=1044,
ER_ACCESS_DENIED_ERROR=1045,
ER_NO_DB_ERROR=1046,
ER_UNKNOWN_COM_ERROR=1047,
ER_BAD_NULL_ERROR=1048,
ER_BAD_DB_ERROR=1049,
ER_TABLE_EXISTS_ERROR=1050,
ER_BAD_TABLE_ERROR=1051,
ER_NON_UNIQ_ERROR=1052,
ER_SERVER_SHUTDOWN=1053,
ER_BAD_FIELD_ERROR=1054,
ER_WRONG_FIELD_WITH_GROUP=1055,
ER_WRONG_GROUP_FIELD=1056,
ER_WRONG_SUM_SELECT=1057,
ER_WRONG_VALUE_COUNT=1058,
ER_TOO_LONG_IDENT=1059,
ER_DUP_FIELDNAME=1060,
ER_DUP_KEYNAME=1061,
ER_DUP_ENTRY=1062,
ER_WRONG_FIELD_SPEC=1063,
ER_PARSE_ERROR=1064,
ER_EMPTY_QUERY=1065,
ER_NONUNIQ_TABLE=1066,
ER_INVALID_DEFAULT=1067,
ER_MULTIPLE_PRI_KEY=1068,
ER_TOO_MANY_KEYS=1069,
ER_TOO_MANY_KEY_PARTS=1070,
ER_TOO_LONG_KEY=1071,
ER_KEY_COLUMN_DOES_NOT_EXITS=1072,
ER_BLOB_USED_AS_KEY=1073,
ER_TOO_BIG_FIELDLENGTH=1074,
ER_WRONG_AUTO_KEY=1075,
ER_READY=1076,
ER_NORMAL_SHUTDOWN=1077,
ER_GOT_SIGNAL=1078,
ER_SHUTDOWN_COMPLETE=1079,
ER_FORCING_CLOSE=1080,
ER_IPSOCK_ERROR=1081,
ER_NO_SUCH_INDEX=1082,
ER_WRONG_FIELD_TERMINATORS=1083,
ER_BLOBS_AND_NO_TERMINATED=1084,
ER_TEXTFILE_NOT_READABLE=1085,
ER_FILE_EXISTS_ERROR=1086,
ER_LOAD_INFO=1087,
ER_ALTER_INFO=1088,
ER_WRONG_SUB_KEY=1089,
ER_CANT_REMOVE_ALL_FIELDS=1090,
ER_CANT_DROP_FIELD_OR_KEY=1091,
ER_INSERT_INFO=1092,
ER_INSERT_TABLE_USED=1093,
ER_NO_SUCH_THREAD=1094,
ER_KILL_DENIED_ERROR=1095,
ER_NO_TABLES_USED=1096,
ER_TOO_BIG_SET=1097,
ER_NO_UNIQUE_LOGFILE=1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE=1099,
ER_TABLE_NOT_LOCKED=1100,
ER_BLOB_CANT_HAVE_DEFAULT=1101,
ER_WRONG_DB_NAME=1102,
ER_WRONG_TABLE_NAME=1103,
ER_TOO_BIG_SELECT=1104,
ER_UNKNOWN_ERROR=1105,
ER_UNKNOWN_PROCEDURE=1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE=1108,
ER_UNKNOWN_TABLE=1109,
ER_FIELD_SPECIFIED_TWICE=1110,
ER_INVALID_GROUP_FUNC_USE=1111,
ER_UNSUPPORTED_EXTENSION=1112,
ER_TABLE_MUST_HAVE_COLUMNS=1113,
ER_RECORD_FILE_FULL=1114,
ER_UNKNOWN_CHARACTER_SET=1115,
ER_TOO_MANY_TABLES=1116,
ER_TOO_MANY_FIELDS=1117,
ER_TOO_BIG_ROWSIZE=1118,
ER_STACK_OVERRUN=1119,
ER_WRONG_OUTER_JOIN=1120,
ER_NULL_COLUMN_IN_INDEX=1121,
ER_CANT_FIND_UDF=1122,
ER_CANT_INITIALIZE_UDF=1123,
ER_UDF_NO_PATHS=1124,
ER_UDF_EXISTS=1125,
ER_CANT_OPEN_LIBRARY=1126,
ER_CANT_FIND_DL_ENTRY=1127,
ER_FUNCTION_NOT_DEFINED=1128,
ER_HOST_IS_BLOCKED=1129,
ER_HOST_NOT_PRIVILEGED=1130,
ER_PASSWORD_ANONYMOUS_USER=1131,
ER_PASSWORD_NOT_ALLOWED=1132,
ER_PASSWORD_NO_MATCH=1133,
ER_UPDATE_INFO=1134,
ER_CANT_CREATE_THREAD=1135,
ER_WRONG_VALUE_COUNT_ON_ROW=1136,
ER_CANT_REOPEN_TABLE=1137,
ER_INVALID_USE_OF_NULL=1138,
ER_REGEXP_ERROR=1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140,
ER_NONEXISTING_GRANT=1141,
ER_TABLEACCESS_DENIED_ERROR=1142,
ER_COLUMNACCESS_DENIED_ERROR=1143,
ER_ILLEGAL_GRANT_FOR_TABLE=1144,
ER_GRANT_WRONG_HOST_OR_USER=1145,
ER_NO_SUCH_TABLE=1146,
ER_NONEXISTING_TABLE_GRANT=1147,
ER_NOT_ALLOWED_COMMAND=1148,
ER_SYNTAX_ERROR=1149,
ER_DELAYED_CANT_CHANGE_LOCK=1150,
ER_TOO_MANY_DELAYED_THREADS=1151,
ER_ABORTING_CONNECTION=1152,
ER_NET_PACKET_TOO_LARGE=1153,
ER_NET_READ_ERROR_FROM_PIPE=1154,
ER_NET_FCNTL_ERROR=1155,
ER_NET_PACKETS_OUT_OF_ORDER=1156,
ER_NET_UNCOMPRESS_ERROR=1157,
ER_NET_READ_ERROR=1158,
ER_NET_READ_INTERRUPTED=1159,
ER_NET_ERROR_ON_WRITE=1160,
ER_NET_WRITE_INTERRUPTED=1161,
ER_TOO_LONG_STRING=1162,
ER_TABLE_CANT_HANDLE_BLOB=1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164,
ER_DELAYED_INSERT_TABLE_LOCKED=1165,
ER_WRONG_COLUMN_NAME=1166,
ER_WRONG_KEY_COLUMN=1167,
ER_WRONG_MRG_TABLE=1168,
ER_DUP_UNIQUE=1169,
ER_BLOB_KEY_WITHOUT_LENGTH=1170,
ER_PRIMARY_CANT_HAVE_NULL=1171,
ER_TOO_MANY_ROWS=1172,
ER_REQUIRES_PRIMARY_KEY=1173,
ER_NO_RAID_COMPILED=1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175,
ER_KEY_DOES_NOT_EXITS=1176,
ER_CHECK_NO_SUCH_TABLE=1177,
ER_CHECK_NOT_IMPLEMENTED=1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179,
ER_ERROR_DURING_COMMIT=1180,
ER_ERROR_DURING_ROLLBACK=1181,
ER_ERROR_DURING_FLUSH_LOGS=1182,
ER_ERROR_DURING_CHECKPOINT=1183,
ER_NEW_ABORTING_CONNECTION=1184,
ER_DUMP_NOT_IMPLEMENTED= 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED=1186,
ER_INDEX_REBUILD= 1187,
ER_MASTER=1188,
ER_MASTER_NET_READ=1189,
ER_MASTER_NET_WRITE=1190,
ER_FT_MATCHING_KEY_NOT_FOUND=1191,
ER_LOCK_OR_ACTIVE_TRANSACTION=1192,
ER_UNKNOWN_SYSTEM_VARIABLE=1193,
ER_CRASHED_ON_USAGE=1194,
ER_CRASHED_ON_REPAIR=1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK=1196,
ER_TRANS_CACHE_FULL=1197,
ER_SLAVE_MUST_STOP=1198,
ER_SLAVE_NOT_RUNNING=1199,
ER_BAD_SLAVE=1200,
ER_MASTER_INFO=1201,
ER_SLAVE_THREAD=1202,
ER_TOO_MANY_USER_CONNECTIONS=1203,
ER_SET_CONSTANTS_ONLY=1204,
ER_LOCK_WAIT_TIMEOUT=1205,
ER_LOCK_TABLE_FULL=1206,
ER_READ_ONLY_TRANSACTION=1207,
ER_DROP_DB_WITH_READ_LOCK=1208,
ER_CREATE_DB_WITH_READ_LOCK=1209,
ER_WRONG_ARGUMENTS=1210,
ER_NO_PERMISSION_TO_CREATE_USER=1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR=1212,
ER_LOCK_DEADLOCK=1213,
ER_TABLE_CANT_HANDLE_FULLTEXT=1214,
ER_CANNOT_ADD_FOREIGN=1215,
ER_NO_REFERENCED_ROW=1216,
ER_ROW_IS_REFERENCED=1217,
ER_CONNECT_TO_MASTER=1218,
ER_QUERY_ON_MASTER=1219,
ER_ERROR_WHEN_EXECUTING_COMMAND=1220,
ER_WRONG_USAGE=1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222,
ER_CANT_UPDATE_WITH_READLOCK=1223,
ER_MIXING_NOT_ALLOWED=1224,
ER_DUP_ARGUMENT=1225,
ER_USER_LIMIT_REACHED=1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR=1227,
ER_LOCAL_VARIABLE=1228,
ER_GLOBAL_VARIABLE=1229,
ER_NO_DEFAULT=1230,
ER_WRONG_VALUE_FOR_VAR=1231,
ER_WRONG_TYPE_FOR_VAR=1232,
ER_VAR_CANT_BE_READ=1233,
ER_CANT_USE_OPTION_HERE=1234,
ER_NOT_SUPPORTED_YET=1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG=1236,
ER_SLAVE_IGNORED_TABLE=1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR=1238,
CR_UNKNOWN_ERROR=1900,
CR_SOCKET_CREATE_ERROR=1901,
CR_CONNECTION_ERROR=1902,
CR_CONN_HOST_ERROR=1903,
CR_IPSOCK_ERROR =1904,
CR_UNKNOWN_HOST =1905,
CR_SERVER_GONE_ERROR=1906,
CR_VERSION_ERROR=1907,
CR_OUT_OF_MEMORY=1908,
CR_WRONG_HOST_INFO=1909,
CR_LOCALHOST_CONNECTION=1910,
CR_TCP_CONNECTION=1911,
CR_SERVER_HANDSHAKE_ERR=1912,
CR_SERVER_LOST=1913,
CR_COMMANDS_OUT_OF_SYNC=1914,
CR_NAMEDPIPE_CONNECTION=1915,
CR_NAMEDPIPEWAIT_ERROR=1916,
CR_NAMEDPIPEOPEN_ERROR=1917,
CR_NAMEDPIPESETSTATE_ERROR=1918,
CR_CANT_READ_CHARSET=1919,
CR_NET_PACKET_TOO_LARGE=1920,
CR_EMBEDDED_CONNECTION=1921,
CR_PROBE_SLAVE_STATUS=1922,
CR_PROBE_SLAVE_HOSTS=1923,
CR_PROBE_SLAVE_CONNECT=1924,
CR_PROBE_MASTER_CONNECT=1925,
CR_SSL_CONNECTION_ERROR=1926,
CR_MALFORMED_PACKET=1927,
CR_WRONG_LICENSE=1928,
};
};
#endif

47
src/search_local/index_storage/api/c_api_cc/examples/select/Makefile Normal file
View File

@ -0,0 +1,47 @@
LIB = ../../libdtc.so
OUTPUT1 = selecttest
SOURCES = selecttest.cpp
OUTPUT2 = TestServerPool
SOURCES = TestServerPool.cpp
OBJECTS1= selecttest.o
CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs
CFLAGS = $(CPPFLAGS)
OBJECTS2=TestServerPool.o
CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs
CFLAGS = $(CPPFLAGS)
all:$(OUTPUT1) $(OUTPUT2)
.SUFFIXES: .o .cpp
.cpp.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^)
.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^
$(OUTPUT1): $(OBJECTS1)
$(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB)
$(OUTPUT2): $(OBJECTS2)
$(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB)
clean:
rm -f *.o *.so *.~ *.bak
rm -f $(OUTPUT1)
rm -f $(OUTPUT2)
install:
cp -f $(OUTPUT) $(INSTALL_PATH)
dep:
mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend
ifneq ($(wildcard .depend),)
include .depend
endif

118
src/search_local/index_storage/api/c_api_cc/examples/select/TestServerPool.cpp Normal file
View File

@ -0,0 +1,118 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
using namespace DTC;
int main(int argc,char* argv[])
{
int ret = 0;
int uid = 0;
int age = 0;
int salary = 0;
std::string name;
std::string city;
std::string descr;
uid = atoi(argv[1]);
name = std::string(argv[2]);
city = std::string(argv[3]);
descr = std::string(argv[4]);
age = atoi(argv[5]);
salary = atoi(argv[6]);
ServerPool svp(5, 5);
Server sv1, sv2, sv3, sv4, sv5;
/* server-request-1 */
sv1.IntKey();
sv1.SetTableName("tp1");
sv1.SetAddress("192.168.214.62", "10201");
sv1.SetAccessKey("0000020184d9cfc2f395ce883a41d7ffc1bbcf4e");
sv1.SetTimeout(5);
InsertRequest req1(&sv1);
req1.SetKey(uid);
//req1.Set("uid", 100001);
req1.Set("name", name.c_str());
req1.Set("city", city.c_str());
req1.Set("descr", descr.c_str());
req1.Set("age", age);
req1.Set("salary", salary);
/* server-request-2 */
sv2.IntKey();
sv2.SetTableName("tp2");
sv2.SetAddress("192.168.214.62", "10202");
sv2.SetAccessKey("0000020284d9cfc2f395ce883a41d7ffc1bbcf4e");
sv2.SetTimeout(5);
InsertRequest req2(&sv2);
req2.SetKey(uid);
//req2.Set("uid", 100001);
req2.Set("name", name.c_str());
req2.Set("city", city.c_str());
req2.Set("descr", descr.c_str());
req2.Set("age", age);
req2.Set("salary", salary);
/* server-request-3 */
sv3.IntKey();
sv3.SetTableName("tp3");
sv3.SetAddress("192.168.214.62", "10203");
sv3.SetAccessKey("0000020384d9cfc2f395ce883a41d7ffc1bbcf4e");
sv3.SetTimeout(5);
InsertRequest req3(&sv3);
req3.SetKey(uid);
//req2.Set("uid", 100001);
req3.Set("name", name.c_str());
req3.Set("city", city.c_str());
req3.Set("descr", descr.c_str());
req3.Set("age", age);
req3.Set("salary", salary);
/* server-request-4 */
sv4.IntKey();
sv4.SetTableName("tp4");
sv4.SetAddress("192.168.214.62", "10204");
sv4.SetAccessKey("0000020484d9cfc2f395ce883a41d7ffc1bbcf4e");
sv4.SetTimeout(5);
InsertRequest req4(&sv4);
req4.SetKey(uid);
//req4.Set("uid", 100001);
req4.Set("name", name.c_str());
req4.Set("city", city.c_str());
req4.Set("descr", descr.c_str());
req4.Set("age", age);
req4.Set("salary", salary);
/* server-request-5 */
sv5.IntKey();
sv5.SetTableName("tp5");
sv5.SetAddress("192.168.214.62", "10205");
sv5.SetAccessKey("0000020584d9cfc2f395ce883a41d7ffc1bbcf4e");
sv5.SetTimeout(5);
InsertRequest req5(&sv5);
req5.SetKey(uid);
//req5.Set("uid", 100001);
req5.Set("name", name.c_str());
req5.Set("city", city.c_str());
req5.Set("descr", descr.c_str());
req5.Set("age", age);
req5.Set("salary", salary);
svp.AddRequest(&req1, 1);
svp.AddRequest(&req2, 2);
svp.AddRequest(&req3, 3);
svp.AddRequest(&req4, 4);
svp.AddRequest(&req5, 5);
ret = svp.Execute(120000);
printf("ret:%d\n", ret);
return 0;
}

91
src/search_local/index_storage/api/c_api_cc/examples/select/selecttest.cpp Normal file
View File

@ -0,0 +1,91 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int iRet;
unsigned int uid;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10109");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
//下面三行代码是Get数据
unsigned int age;
std::string name;
std::string city;
std::string descr;
DTC::GetRequest stGetReq(&stServer);
uid = atoi(argv[1]);
iRet = stGetReq.SetKey(uid);
if(iRet == 0)
iRet = stGetReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(iRet == 0)
iRet = stGetReq.Need("name");
if(iRet == 0)
iRet = stGetReq.Need("city");
if(iRet == 0)
iRet = stGetReq.Need("descr");
if(iRet == 0)
iRet = stGetReq.Need("age");
if(iRet == 0)
iRet = stGetReq.GT("age", 24);
//iRet = stGetReq.OR("age", 27);
if(iRet != 0)
{
printf("get-req set key or need error: %d", iRet);
fflush(stdout);
return(-1);
}
// execute & get result
DTC::Result stResult;
iRet = stGetReq.Execute(stResult);
if(iRet != 0)
//如果出错,则输出错误码、错误阶段，错误信息，stResult.ErrorFrom(), stResult.ErrorMessage() 这两个错误信息很重要，一定要打印出来，方便定位问题
{
printf ("uin[%u] dtc execute get error: %d, error_from:%s, msg:%s\n",
uid,//出错的key是多少
iRet,//错误码为多少
stResult.ErrorFrom(),//返回错误阶段
stResult.ErrorMessage()//返回错误信息
);
fflush(stdout);
return(-2);
}
if(stResult.NumRows() <= 0)
{
// 数据不存在
printf("uin[%u] data not exist.\n", uid);
return(0);
}
printf("nubrow:%d\n", stResult.NumRows());
for(int i=0;i<=stResult.NumRows();++i)
{
iRet = stResult.FetchRow();//开始获取数据
if(iRet < 0)
{
printf ("uid[%lu] dtc fetch row error: %d\n", uid, iRet);
fflush(stdout);
return(-3);
}
//如果一切正确，则可以输出数据了
printf("uid: %d\n", stResult.IntValue("uid"));//输出int类型的数据
printf("name: %s\n", stResult.StringValue("name"));//输出string类型的数据
printf("city: %s\n", stResult.StringValue("city"));//输出string类型的数据
printf("descr: %s\n", stResult.BinaryValue("descr"));//输出binary类型的数据
printf("age:%d\n",stResult.IntValue("age"));
}
return(0);
}

890
src/search_local/index_storage/api/c_api_cc/examples/select/ttcapi.h Normal file
View File

@ -0,0 +1,890 @@
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DTC {
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
//mem monior Request;2014/06/6;by seanzheng
const int RequestMonitor = 15;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int NodeHandleChange = 13;
const int Migrate = 14;
const int ReloadClusterNodeList = 15;
const int SetClusterNodeState = 16;
const int ChangeNodeAddress = 17;
const int GetClusterState = 18;
const int PurgeForHit = 19;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log (const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
void write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)));
class Result;
class Server {
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void CloneTabDef(const Server& source);
int SetAddress(const char *host, const char *port=0);
int SetTableName(const char *);
//for compress
void SetCompressLevel(int);
//get address and tablename set by user
const char * GetAddress(void) const;
const char * GetTableName(void) const;
//get address and tablename set by dtc frame,for plugin only;
const char * GetServerAddress(void) const;
const char * GetServerTableName(void) const;
int IntKey(void);
int BinaryKey(void);
int StringKey(void);
int AddKey(const char* name, int type);
int FieldType(const char* name);
const char *ErrorMessage(void) const;
void SetTimeout(int);
void SetMTimeout(int);
int Connect(void);
void Close(void);
int Ping(void);
void AutoPing(void);
void SetFD(int); // UNSUPPORTED API
void SetAutoUpdateTab(bool autoUpdate);
void SetAutoReconnect(int autoReconnect);
int DecodePacket(Result &, const char *, int);
int CheckPacketSize(const char *, int);
void SetAccessKey(const char *token);
};
class Request {
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int AttachServer(Server *srv);
void SetAdminCode(int code);
void SetHotBackupID(long long);
void SetMasterHBTimestamp(long long);
void SetSlaveHBTimestamp(long long);
#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int FieldType(const char*);
void NoCache(void);
void NoNextServer(void);
void Limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int OR(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//just for compress,only support binary field
int CompressSet(const char *, const char *, int);
//just compress and set. Don't need compressflag
int CompressSetForce(const char *, const char *, int);
//bits op
int SetMultiBits(const char *, int, int, unsigned int);
int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);}
int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);}
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(OR, unsigned long long);
_REDIR_(OR, long);
_REDIR_(OR, unsigned long);
_REDIR_(OR, int);
_REDIR_(OR, unsigned int);
_REDIR_(OR, short);
_REDIR_(OR, unsigned short);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void UnsetKey(void);
int SetKey(long long);
int SetKey(const char *);
int SetKey(const char *, int);
#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int AddKeyValue(const char* name, long long v);
int AddKeyValue(const char* name, const char *str);
int AddKeyValue(const char* name, const char *ptr, int len);
#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *Execute(void);
Result *Execute(long long);
Result *Execute(const char *);
Result *Execute(const char *, int);
#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int Execute(Result&);
int Execute(Result&, long long);
int Execute(Result&, const char *);
int Execute(Result&, const char *, int);
#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int EncodePacket(char *&, int&, long long&);
int EncodePacket(char *&, int&, long long&, long long);
int EncodePacket(char *&, int&, long long&, const char *);
int EncodePacket(char *&, int&, long long&, const char *, int);
#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int SetCacheID(long long);
#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);}
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
const char *ErrorMessage(void) const;
};
class GetRequest : public Request {
public:
GetRequest(Server *srv): Request(srv, RequestGet) {}
GetRequest(): Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request {
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request {
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request {
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request {
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest:public Request {
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request {
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest: public Request {
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest: public Request {
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class MonitorRequest: public Request {
public:
MonitorRequest(Server *srv) : Request(srv, RequestMonitor){}
};
class Result {
private:
void *addr;
long check;
Result(const Result &);
char *ServerInfo() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void SetError(int errcode, const char *from, const char *detail); // from will not dupped
int ResultCode(void) const;
const char *ErrorMessage(void) const;
const char *ErrorFrom(void) const;
long long HotBackupID() const;
long long MasterHBTimestamp() const;
long long SlaveHBTimestamp() const;
long long BinlogID() const;
long long BinlogOffset() const;
long long MemSize() const;
long long DataSize() const;
int NumRows(void) const;
int TotalRows(void) const;
int AffectedRows(void) const;
int NumFields(void) const;
const char* FieldName(int n) const;
int FieldPresent(const char* name) const;
int FieldType(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long ServerTimestamp(void) const;
long long InsertID(void) const;
long long IntKey(void) const;
const char *BinaryKey(void) const;
const char *BinaryKey(int *) const;
const char *BinaryKey(int &) const;
const char *StringKey(void) const;
const char *StringKey(int *) const;
const char *StringKey(int &) const;
long long IntValue(const char *) const;
double FloatValue(const char *) const;
const char *StringValue(const char *) const;
const char *StringValue(const char *, int*) const;
const char *StringValue(const char *, int&) const;
const char *BinaryValue(const char *) const;
const char *BinaryValue(const char *, int*) const;
const char *BinaryValue(const char *, int&) const;
int UnCompressBinaryValue(const char *name,char **buf,int *lenp);
//Uncompress Binary Value without check compressflag
int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp);
const char * UnCompressErrorMessage() const;
long long IntValue(int) const;
double FloatValue(int) const;
const char *StringValue(int) const;
const char *StringValue(int, int*) const;
const char *StringValue(int, int&) const;
const char *BinaryValue(int) const;
const char *BinaryValue(int, int*) const;
const char *BinaryValue(int, int&) const;
int FetchRow(void);
int Rewind(void);
};
class ServerPool {
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int GetEpollFD(int size);
int AddServer(Server *srv, int mReq=1, int mConn=0);
int AddRequest(Request *, long long);
int AddRequest(Request *, long long, long long);
int AddRequest(Request *, long long, const char *);
int AddRequest(Request *, long long, const char *, int);
int AddRequest(Request *, void *);
int AddRequest(Request *, void *, long long);
int AddRequest(Request *, void *, const char *);
int AddRequest(Request *, void *, const char *, int);
int Execute(int msec);
int ExecuteAll(int msec);
int CancelRequest(int);
int CancelAllRequest(int type);
int AbortRequest(int);
int AbortAllRequest(int type);
Result *GetResult(void);
Result *GetResult(int);
int GetResult(Result&);
int GetResult(Result&, int);
int ServerCount(void) const;
int RequestCount(int type) const;
int RequestState(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT|SEND|RECV|DONE;
enum {
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
};
enum {
ER_HASHCHK=1000,
ER_NISAMCHK=1001,
ER_NO=1002,
ER_YES=1003,
ER_CANT_CREATE_FILE=1004,
ER_CANT_CREATE_TABLE=1005,
ER_CANT_CREATE_DB=1006,
ER_DB_CREATE_EXISTS=1007,
ER_DB_DROP_EXISTS=1008,
ER_DB_DROP_DELETE=1009,
ER_DB_DROP_RMDIR=1010,
ER_CANT_DELETE_FILE=1011,
ER_CANT_FIND_SYSTEM_REC=1012,
ER_CANT_GET_STAT=1013,
ER_CANT_GET_WD=1014,
ER_CANT_LOCK=1015,
ER_CANT_OPEN_FILE=1016,
ER_FILE_NOT_FOUND=1017,
ER_CANT_READ_DIR=1018,
ER_CANT_SET_WD=1019,
ER_CHECKREAD=1020,
ER_DISK_FULL=1021,
ER_DUP_KEY=1022,
ER_ERROR_ON_CLOSE=1023,
ER_ERROR_ON_READ=1024,
ER_ERROR_ON_RENAME=1025,
ER_ERROR_ON_WRITE=1026,
ER_FILE_USED=1027,
ER_FILSORT_ABORT=1028,
ER_FORM_NOT_FOUND=1029,
ER_GET_ERRNO=1030,
ER_ILLEGAL_HA=1031,
ER_KEY_NOT_FOUND=1032,
ER_NOT_FORM_FILE=1033,
ER_NOT_KEYFILE=1034,
ER_OLD_KEYFILE=1035,
ER_OPEN_AS_READONLY=1036,
ER_OUTOFMEMORY=1037,
ER_OUT_OF_SORTMEMORY=1038,
ER_UNEXPECTED_EOF=1039,
ER_CON_COUNT_ERROR=1040,
ER_OUT_OF_RESOURCES=1041,
ER_BAD_HOST_ERROR=1042,
ER_HANDSHAKE_ERROR=1043,
ER_DBACCESS_DENIED_ERROR=1044,
ER_ACCESS_DENIED_ERROR=1045,
ER_NO_DB_ERROR=1046,
ER_UNKNOWN_COM_ERROR=1047,
ER_BAD_NULL_ERROR=1048,
ER_BAD_DB_ERROR=1049,
ER_TABLE_EXISTS_ERROR=1050,
ER_BAD_TABLE_ERROR=1051,
ER_NON_UNIQ_ERROR=1052,
ER_SERVER_SHUTDOWN=1053,
ER_BAD_FIELD_ERROR=1054,
ER_WRONG_FIELD_WITH_GROUP=1055,
ER_WRONG_GROUP_FIELD=1056,
ER_WRONG_SUM_SELECT=1057,
ER_WRONG_VALUE_COUNT=1058,
ER_TOO_LONG_IDENT=1059,
ER_DUP_FIELDNAME=1060,
ER_DUP_KEYNAME=1061,
ER_DUP_ENTRY=1062,
ER_WRONG_FIELD_SPEC=1063,
ER_PARSE_ERROR=1064,
ER_EMPTY_QUERY=1065,
ER_NONUNIQ_TABLE=1066,
ER_INVALID_DEFAULT=1067,
ER_MULTIPLE_PRI_KEY=1068,
ER_TOO_MANY_KEYS=1069,
ER_TOO_MANY_KEY_PARTS=1070,
ER_TOO_LONG_KEY=1071,
ER_KEY_COLUMN_DOES_NOT_EXITS=1072,
ER_BLOB_USED_AS_KEY=1073,
ER_TOO_BIG_FIELDLENGTH=1074,
ER_WRONG_AUTO_KEY=1075,
ER_READY=1076,
ER_NORMAL_SHUTDOWN=1077,
ER_GOT_SIGNAL=1078,
ER_SHUTDOWN_COMPLETE=1079,
ER_FORCING_CLOSE=1080,
ER_IPSOCK_ERROR=1081,
ER_NO_SUCH_INDEX=1082,
ER_WRONG_FIELD_TERMINATORS=1083,
ER_BLOBS_AND_NO_TERMINATED=1084,
ER_TEXTFILE_NOT_READABLE=1085,
ER_FILE_EXISTS_ERROR=1086,
ER_LOAD_INFO=1087,
ER_ALTER_INFO=1088,
ER_WRONG_SUB_KEY=1089,
ER_CANT_REMOVE_ALL_FIELDS=1090,
ER_CANT_DROP_FIELD_OR_KEY=1091,
ER_INSERT_INFO=1092,
ER_INSERT_TABLE_USED=1093,
ER_NO_SUCH_THREAD=1094,
ER_KILL_DENIED_ERROR=1095,
ER_NO_TABLES_USED=1096,
ER_TOO_BIG_SET=1097,
ER_NO_UNIQUE_LOGFILE=1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE=1099,
ER_TABLE_NOT_LOCKED=1100,
ER_BLOB_CANT_HAVE_DEFAULT=1101,
ER_WRONG_DB_NAME=1102,
ER_WRONG_TABLE_NAME=1103,
ER_TOO_BIG_SELECT=1104,
ER_UNKNOWN_ERROR=1105,
ER_UNKNOWN_PROCEDURE=1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE=1108,
ER_UNKNOWN_TABLE=1109,
ER_FIELD_SPECIFIED_TWICE=1110,
ER_INVALID_GROUP_FUNC_USE=1111,
ER_UNSUPPORTED_EXTENSION=1112,
ER_TABLE_MUST_HAVE_COLUMNS=1113,
ER_RECORD_FILE_FULL=1114,
ER_UNKNOWN_CHARACTER_SET=1115,
ER_TOO_MANY_TABLES=1116,
ER_TOO_MANY_FIELDS=1117,
ER_TOO_BIG_ROWSIZE=1118,
ER_STACK_OVERRUN=1119,
ER_WRONG_OUTER_JOIN=1120,
ER_NULL_COLUMN_IN_INDEX=1121,
ER_CANT_FIND_UDF=1122,
ER_CANT_INITIALIZE_UDF=1123,
ER_UDF_NO_PATHS=1124,
ER_UDF_EXISTS=1125,
ER_CANT_OPEN_LIBRARY=1126,
ER_CANT_FIND_DL_ENTRY=1127,
ER_FUNCTION_NOT_DEFINED=1128,
ER_HOST_IS_BLOCKED=1129,
ER_HOST_NOT_PRIVILEGED=1130,
ER_PASSWORD_ANONYMOUS_USER=1131,
ER_PASSWORD_NOT_ALLOWED=1132,
ER_PASSWORD_NO_MATCH=1133,
ER_UPDATE_INFO=1134,
ER_CANT_CREATE_THREAD=1135,
ER_WRONG_VALUE_COUNT_ON_ROW=1136,
ER_CANT_REOPEN_TABLE=1137,
ER_INVALID_USE_OF_NULL=1138,
ER_REGEXP_ERROR=1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140,
ER_NONEXISTING_GRANT=1141,
ER_TABLEACCESS_DENIED_ERROR=1142,
ER_COLUMNACCESS_DENIED_ERROR=1143,
ER_ILLEGAL_GRANT_FOR_TABLE=1144,
ER_GRANT_WRONG_HOST_OR_USER=1145,
ER_NO_SUCH_TABLE=1146,
ER_NONEXISTING_TABLE_GRANT=1147,
ER_NOT_ALLOWED_COMMAND=1148,
ER_SYNTAX_ERROR=1149,
ER_DELAYED_CANT_CHANGE_LOCK=1150,
ER_TOO_MANY_DELAYED_THREADS=1151,
ER_ABORTING_CONNECTION=1152,
ER_NET_PACKET_TOO_LARGE=1153,
ER_NET_READ_ERROR_FROM_PIPE=1154,
ER_NET_FCNTL_ERROR=1155,
ER_NET_PACKETS_OUT_OF_ORDER=1156,
ER_NET_UNCOMPRESS_ERROR=1157,
ER_NET_READ_ERROR=1158,
ER_NET_READ_INTERRUPTED=1159,
ER_NET_ERROR_ON_WRITE=1160,
ER_NET_WRITE_INTERRUPTED=1161,
ER_TOO_LONG_STRING=1162,
ER_TABLE_CANT_HANDLE_BLOB=1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164,
ER_DELAYED_INSERT_TABLE_LOCKED=1165,
ER_WRONG_COLUMN_NAME=1166,
ER_WRONG_KEY_COLUMN=1167,
ER_WRONG_MRG_TABLE=1168,
ER_DUP_UNIQUE=1169,
ER_BLOB_KEY_WITHOUT_LENGTH=1170,
ER_PRIMARY_CANT_HAVE_NULL=1171,
ER_TOO_MANY_ROWS=1172,
ER_REQUIRES_PRIMARY_KEY=1173,
ER_NO_RAID_COMPILED=1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175,
ER_KEY_DOES_NOT_EXITS=1176,
ER_CHECK_NO_SUCH_TABLE=1177,
ER_CHECK_NOT_IMPLEMENTED=1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179,
ER_ERROR_DURING_COMMIT=1180,
ER_ERROR_DURING_ROLLBACK=1181,
ER_ERROR_DURING_FLUSH_LOGS=1182,
ER_ERROR_DURING_CHECKPOINT=1183,
ER_NEW_ABORTING_CONNECTION=1184,
ER_DUMP_NOT_IMPLEMENTED= 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED=1186,
ER_INDEX_REBUILD= 1187,
ER_MASTER=1188,
ER_MASTER_NET_READ=1189,
ER_MASTER_NET_WRITE=1190,
ER_FT_MATCHING_KEY_NOT_FOUND=1191,
ER_LOCK_OR_ACTIVE_TRANSACTION=1192,
ER_UNKNOWN_SYSTEM_VARIABLE=1193,
ER_CRASHED_ON_USAGE=1194,
ER_CRASHED_ON_REPAIR=1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK=1196,
ER_TRANS_CACHE_FULL=1197,
ER_SLAVE_MUST_STOP=1198,
ER_SLAVE_NOT_RUNNING=1199,
ER_BAD_SLAVE=1200,
ER_MASTER_INFO=1201,
ER_SLAVE_THREAD=1202,
ER_TOO_MANY_USER_CONNECTIONS=1203,
ER_SET_CONSTANTS_ONLY=1204,
ER_LOCK_WAIT_TIMEOUT=1205,
ER_LOCK_TABLE_FULL=1206,
ER_READ_ONLY_TRANSACTION=1207,
ER_DROP_DB_WITH_READ_LOCK=1208,
ER_CREATE_DB_WITH_READ_LOCK=1209,
ER_WRONG_ARGUMENTS=1210,
ER_NO_PERMISSION_TO_CREATE_USER=1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR=1212,
ER_LOCK_DEADLOCK=1213,
ER_TABLE_CANT_HANDLE_FULLTEXT=1214,
ER_CANNOT_ADD_FOREIGN=1215,
ER_NO_REFERENCED_ROW=1216,
ER_ROW_IS_REFERENCED=1217,
ER_CONNECT_TO_MASTER=1218,
ER_QUERY_ON_MASTER=1219,
ER_ERROR_WHEN_EXECUTING_COMMAND=1220,
ER_WRONG_USAGE=1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222,
ER_CANT_UPDATE_WITH_READLOCK=1223,
ER_MIXING_NOT_ALLOWED=1224,
ER_DUP_ARGUMENT=1225,
ER_USER_LIMIT_REACHED=1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR=1227,
ER_LOCAL_VARIABLE=1228,
ER_GLOBAL_VARIABLE=1229,
ER_NO_DEFAULT=1230,
ER_WRONG_VALUE_FOR_VAR=1231,
ER_WRONG_TYPE_FOR_VAR=1232,
ER_VAR_CANT_BE_READ=1233,
ER_CANT_USE_OPTION_HERE=1234,
ER_NOT_SUPPORTED_YET=1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG=1236,
ER_SLAVE_IGNORED_TABLE=1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR=1238,
CR_UNKNOWN_ERROR=1900,
CR_SOCKET_CREATE_ERROR=1901,
CR_CONNECTION_ERROR=1902,
CR_CONN_HOST_ERROR=1903,
CR_IPSOCK_ERROR =1904,
CR_UNKNOWN_HOST =1905,
CR_SERVER_GONE_ERROR=1906,
CR_VERSION_ERROR=1907,
CR_OUT_OF_MEMORY=1908,
CR_WRONG_HOST_INFO=1909,
CR_LOCALHOST_CONNECTION=1910,
CR_TCP_CONNECTION=1911,
CR_SERVER_HANDSHAKE_ERR=1912,
CR_SERVER_LOST=1913,
CR_COMMANDS_OUT_OF_SYNC=1914,
CR_NAMEDPIPE_CONNECTION=1915,
CR_NAMEDPIPEWAIT_ERROR=1916,
CR_NAMEDPIPEOPEN_ERROR=1917,
CR_NAMEDPIPESETSTATE_ERROR=1918,
CR_CANT_READ_CHARSET=1919,
CR_NET_PACKET_TOO_LARGE=1920,
CR_EMBEDDED_CONNECTION=1921,
CR_PROBE_SLAVE_STATUS=1922,
CR_PROBE_SLAVE_HOSTS=1923,
CR_PROBE_SLAVE_CONNECT=1924,
CR_PROBE_MASTER_CONNECT=1925,
CR_SSL_CONNECTION_ERROR=1926,
CR_MALFORMED_PACKET=1927,
CR_WRONG_LICENSE=1928,
};
};
#endif

46
src/search_local/index_storage/api/c_api_cc/examples/table.conf Normal file
View File

@ -0,0 +1,46 @@
[DB_DEFINE]
DbName = test
DbNum = (1,1)
dbMax = 1
MachineNum = 1
Deploy = 0
[Machine1]
Procs = 10
WriteProcs = 10
CommitProcs = 10
#DbIdx = [0]
DbAddr = 10.152.23.179:9005
DbUser = root
DbPass = rootcdb
[TABLE_DEFINE]
TableName = test
FieldCount = 4
KeyFieldCount = 1
#TableNum = (1,100)
#None=0, Signed=1, Unsigned=2, Float=3, String=4, Binary=5
[FIELD1]
FieldName = name
FieldType = 4
FieldSize = 255
UniqField = 1
[FIELD2]
FieldName = age
FieldType = 1
FieldSize = 4
Value = 0
[FIELD3]
FieldName = sex
FieldType = 1
FieldSize = 4
Value = 0
[FIELD4]
FieldName = city
FieldType = 1
FieldSize = 4
Value = 0

38
src/search_local/index_storage/api/c_api_cc/examples/update/Makefile Normal file
View File

@ -0,0 +1,38 @@
LIB = ../../libdtc.so
OUTPUT = update
SOURCES = updatetest.cpp
OBJECTS= updatetest.o
CPPFLAGS = -g -O -Wall -fPIC -DMACROFOR64BIT -Wl,--no-undefined -Xlinker -zmuldefs
CFLAGS = $(CPPFLAGS)
all:$(OUTPUT)
.SUFFIXES: .o .cpp
.cpp.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -c $(filter %.cpp, $^)
.o:
$(CXX) $(CPPFLAGS) $(INCLUDE) -o $@ $^
$(OUTPUT): $(OBJECTS)
$(CXX) $(CFLAGS) -fPIC -o $@ $^ $(LIB)
clean:
rm -f *.o *.so *.~ *.bak
rm -f $(OUTPUT)
install:
cp -f $(OUTPUT) $(INSTALL_PATH)
dep:
mkdep -p$(VPATH) $(INCLUDE) $(SOURCES) > .depend
ifneq ($(wildcard .depend),)
include .depend
endif

890
src/search_local/index_storage/api/c_api_cc/examples/update/ttcapi.h Normal file
View File

@ -0,0 +1,890 @@
#ifndef __DTC_API_H__
#define __DTC_API_H__
#include <stdlib.h>
namespace DTC {
class Server;
class ServerPool;
class Request;
class Result;
const int RequestSvrAdmin = 3;
const int RequestGet = 4;
const int RequestPurge = 5;
const int RequestInsert = 6;
const int RequestUpdate = 7;
const int RequestDelete = 8;
const int RequestReplace = 12;
const int RequestFlush = 13;
const int RequestInvalidate = 14;
//mem monior Request;2014/06/6;by seanzheng
const int RequestMonitor = 15;
// sub-code of admin cmd
const int RegisterHB = 1;
const int LogoutHB = 2;
const int GetKeyList = 3;
const int GetUpdateKey = 4;
const int GetRawData = 5;
const int ReplaceRawData = 6;
const int AdjustLRU = 7;
const int VerifyHBT = 8;
const int GetHBTime = 9;
const int SET_READONLY = 10;
const int SET_READWRITE = 11;
const int QueryBinlogID = 12;
const int NodeHandleChange = 13;
const int Migrate = 14;
const int ReloadClusterNodeList = 15;
const int SetClusterNodeState = 16;
const int ChangeNodeAddress = 17;
const int GetClusterState = 18;
const int PurgeForHit = 19;
const int KeyTypeNone = 0; // undefined
const int KeyTypeInt = 1; // Signed Integer
const int KeyTypeString = 4; // String, case insensitive, null ended
const int FieldTypeNone = 0; // undefined
const int FieldTypeSigned = 1; // Signed Integer
const int FieldTypeUnsigned = 2; // Unsigned Integer
const int FieldTypeFloat = 3; // float
const int FieldTypeString = 4; // String, case insensitive, null ended
const int FieldTypeBinary = 5; // binary
void init_log (const char *app, const char *dir = NULL);
void set_log_level(int n);
int set_key_value_max(unsigned int count); // 设置批量操作一次最多多少个key(默认最多32个)
void write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)));
class Result;
class Server {
private:
void *addr;
long check;
public:
friend class Request;
friend class Result;
friend class ServerPool;
Server(void);
~Server(void);
Server(const Server &);
void CloneTabDef(const Server& source);
int SetAddress(const char *host, const char *port=0);
int SetTableName(const char *);
//for compress
void SetCompressLevel(int);
//get address and tablename set by user
const char * GetAddress(void) const;
const char * GetTableName(void) const;
//get address and tablename set by dtc frame,for plugin only;
const char * GetServerAddress(void) const;
const char * GetServerTableName(void) const;
int IntKey(void);
int BinaryKey(void);
int StringKey(void);
int AddKey(const char* name, int type);
int FieldType(const char* name);
const char *ErrorMessage(void) const;
void SetTimeout(int);
void SetMTimeout(int);
int Connect(void);
void Close(void);
int Ping(void);
void AutoPing(void);
void SetFD(int); // UNSUPPORTED API
void SetAutoUpdateTab(bool autoUpdate);
void SetAutoReconnect(int autoReconnect);
int DecodePacket(Result &, const char *, int);
int CheckPacketSize(const char *, int);
void SetAccessKey(const char *token);
};
class Request {
private:
void *addr;
long check;
Request(const Request &);
public:
friend class Server;
friend class Result;
friend class ServerPool;
Request(Server *srv, int op);
Request(void);
~Request(void);
void Reset(void);
void Reset(int);
int AttachServer(Server *srv);
void SetAdminCode(int code);
void SetHotBackupID(long long);
void SetMasterHBTimestamp(long long);
void SetSlaveHBTimestamp(long long);
#define _REDIR_(op, t) int op(const char *n, t a) { return op(n, (long long)a); }
#define _REDIRF_(op, t) int op(const char *n, t a) { return op(n, (double)a); }
int Need(const char *);
int Need(const char *, int);
int FieldType(const char*);
void NoCache(void);
void NoNextServer(void);
void Limit(unsigned int, unsigned int);
int EQ(const char *, long long);
int NE(const char *, long long);
int LT(const char *, long long);
int LE(const char *, long long);
int GT(const char *, long long);
int GE(const char *, long long);
int EQ(const char *, const char *);
int NE(const char *, const char *);
int EQ(const char *, const char *, int);
int NE(const char *, const char *, int);
_REDIR_(EQ, unsigned long long);
_REDIR_(EQ, long);
_REDIR_(EQ, unsigned long);
_REDIR_(EQ, int);
_REDIR_(EQ, unsigned int);
_REDIR_(EQ, short);
_REDIR_(EQ, unsigned short);
_REDIR_(EQ, char);
_REDIR_(EQ, unsigned char);
_REDIR_(NE, unsigned long long);
_REDIR_(NE, long);
_REDIR_(NE, unsigned long);
_REDIR_(NE, int);
_REDIR_(NE, unsigned int);
_REDIR_(NE, short);
_REDIR_(NE, unsigned short);
_REDIR_(NE, char);
_REDIR_(NE, unsigned char);
_REDIR_(GT, unsigned long long);
_REDIR_(GT, long);
_REDIR_(GT, unsigned long);
_REDIR_(GT, int);
_REDIR_(GT, unsigned int);
_REDIR_(GT, short);
_REDIR_(GT, unsigned short);
_REDIR_(GT, char);
_REDIR_(GT, unsigned char);
_REDIR_(GE, unsigned long long);
_REDIR_(GE, long);
_REDIR_(GE, unsigned long);
_REDIR_(GE, int);
_REDIR_(GE, unsigned int);
_REDIR_(GE, short);
_REDIR_(GE, unsigned short);
_REDIR_(GE, char);
_REDIR_(GE, unsigned char);
_REDIR_(LT, unsigned long long);
_REDIR_(LT, long);
_REDIR_(LT, unsigned long);
_REDIR_(LT, int);
_REDIR_(LT, unsigned int);
_REDIR_(LT, short);
_REDIR_(LT, unsigned short);
_REDIR_(LT, char);
_REDIR_(LT, unsigned char);
_REDIR_(LE, unsigned long long);
_REDIR_(LE, long);
_REDIR_(LE, unsigned long);
_REDIR_(LE, int);
_REDIR_(LE, unsigned int);
_REDIR_(LE, short);
_REDIR_(LE, unsigned short);
_REDIR_(LE, char);
_REDIR_(LE, unsigned char);
int Set(const char *, long long);
int OR(const char *, long long);
int Add(const char *, long long);
int Sub(const char *, long long);
int Set(const char *, double);
int Add(const char *, double);
int Sub(const char *, double);
int Set(const char *, const char *);
int Set(const char *, const char *, int);
//just for compress,only support binary field
int CompressSet(const char *, const char *, int);
//just compress and set. Don't need compressflag
int CompressSetForce(const char *, const char *, int);
//bits op
int SetMultiBits(const char *, int, int, unsigned int);
int SetBit (const char *f, int o) { return SetMultiBits(f, o, 1, 1);}
int ClearBit(const char *f, int o) { return SetMultiBits(f, o, 1, 0);}
_REDIR_(Set, unsigned long long);
_REDIR_(Set, long);
_REDIR_(Set, unsigned long);
_REDIR_(Set, int);
_REDIR_(Set, unsigned int);
_REDIR_(Set, short);
_REDIR_(Set, unsigned short);
_REDIR_(Set, char);
_REDIR_(Set, unsigned char);
_REDIRF_(Set, float);
_REDIRF_(Set, long double);
_REDIR_(OR, unsigned long long);
_REDIR_(OR, long);
_REDIR_(OR, unsigned long);
_REDIR_(OR, int);
_REDIR_(OR, unsigned int);
_REDIR_(OR, short);
_REDIR_(OR, unsigned short);
_REDIR_(Add, unsigned long long);
_REDIR_(Add, long);
_REDIR_(Add, unsigned long);
_REDIR_(Add, int);
_REDIR_(Add, unsigned int);
_REDIR_(Add, short);
_REDIR_(Add, unsigned short);
_REDIR_(Add, char);
_REDIR_(Add, unsigned char);
_REDIRF_(Add, float);
_REDIRF_(Add, long double);
_REDIR_(Sub, unsigned long long);
_REDIR_(Sub, long);
_REDIR_(Sub, unsigned long);
_REDIR_(Sub, int);
_REDIR_(Sub, unsigned int);
_REDIR_(Sub, short);
_REDIR_(Sub, unsigned short);
_REDIR_(Sub, char);
_REDIR_(Sub, unsigned char);
_REDIRF_(Sub, float);
_REDIRF_(Sub, long double);
#undef _REDIR_
void UnsetKey(void);
int SetKey(long long);
int SetKey(const char *);
int SetKey(const char *, int);
#define _REDIR_(t) int SetKey(t a) { return SetKey((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int AddKeyValue(const char* name, long long v);
int AddKeyValue(const char* name, const char *str);
int AddKeyValue(const char* name, const char *ptr, int len);
#define _REDIR_(t) int AddKeyValue(const char* name, t a) { return AddKeyValue(name, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
Result *Execute(void);
Result *Execute(long long);
Result *Execute(const char *);
Result *Execute(const char *, int);
#define _REDIR_(t) Result *Execute(t a) { return Execute((long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int Execute(Result&);
int Execute(Result&, long long);
int Execute(Result&, const char *);
int Execute(Result&, const char *, int);
#define _REDIR_(t) int Execute(Result &r, t a) { return Execute(r, (long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int EncodePacket(char *&, int&, long long&);
int EncodePacket(char *&, int&, long long&, long long);
int EncodePacket(char *&, int&, long long&, const char *);
int EncodePacket(char *&, int&, long long&, const char *, int);
#define _REDIR_(t) int EncodePacket(char *&p, int &l, long long &m, t a) { return EncodePacket(p,l,m,(long long)a); }
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
int SetCacheID(long long);
#define _REDIR_(t) int SetCacheID(t a) {return SetCacheID((long long)a);}
_REDIR_(unsigned long long);
_REDIR_(long);
_REDIR_(unsigned long);
_REDIR_(int);
_REDIR_(unsigned int);
_REDIR_(short);
_REDIR_(unsigned short);
_REDIR_(char);
_REDIR_(unsigned char);
#undef _REDIR_
const char *ErrorMessage(void) const;
};
class GetRequest : public Request {
public:
GetRequest(Server *srv): Request(srv, RequestGet) {}
GetRequest(): Request((Server *)0, RequestGet) {}
};
class InsertRequest : public Request {
public:
InsertRequest(Server *srv) : Request(srv, RequestInsert) {}
InsertRequest() : Request((Server *)0, RequestInsert) {}
};
class DeleteRequest : public Request {
public:
DeleteRequest(Server *srv) : Request(srv, RequestDelete) {}
DeleteRequest() : Request((Server *)0, RequestDelete) {}
};
class UpdateRequest : public Request {
public:
UpdateRequest(Server *srv) : Request(srv, RequestUpdate) {}
UpdateRequest() : Request((Server *)0, RequestUpdate) {}
};
class PurgeRequest : public Request {
public:
PurgeRequest(Server *srv) : Request(srv, RequestPurge) {}
PurgeRequest() : Request((Server *)0, RequestPurge) {}
};
class ReplaceRequest:public Request {
public:
ReplaceRequest(Server *srv) : Request(srv, RequestReplace) {}
ReplaceRequest() : Request((Server *)0, RequestReplace) {}
};
class FlushRequest : public Request {
public:
FlushRequest(Server *srv) : Request(srv, RequestFlush) {}
FlushRequest(void) : Request((Server *)0, RequestFlush) {}
};
class InvalidateRequest: public Request {
public:
InvalidateRequest(Server *srv) : Request(srv, RequestInvalidate) {}
};
class SvrAdminRequest: public Request {
public:
SvrAdminRequest(Server *srv) : Request(srv, RequestSvrAdmin) {}
};
class MonitorRequest: public Request {
public:
MonitorRequest(Server *srv) : Request(srv, RequestMonitor){}
};
class Result {
private:
void *addr;
long check;
Result(const Result &);
char *ServerInfo() const;
public:
friend class Server;
friend class Request;
friend class ServerPool;
Result(void);
~Result(void);
void Reset(void);
void SetError(int errcode, const char *from, const char *detail); // from will not dupped
int ResultCode(void) const;
const char *ErrorMessage(void) const;
const char *ErrorFrom(void) const;
long long HotBackupID() const;
long long MasterHBTimestamp() const;
long long SlaveHBTimestamp() const;
long long BinlogID() const;
long long BinlogOffset() const;
long long MemSize() const;
long long DataSize() const;
int NumRows(void) const;
int TotalRows(void) const;
int AffectedRows(void) const;
int NumFields(void) const;
const char* FieldName(int n) const;
int FieldPresent(const char* name) const;
int FieldType(int n) const;
long long Tag(void) const;
void *TagPtr(void) const;
long long Magic(void) const;
long long ServerTimestamp(void) const;
long long InsertID(void) const;
long long IntKey(void) const;
const char *BinaryKey(void) const;
const char *BinaryKey(int *) const;
const char *BinaryKey(int &) const;
const char *StringKey(void) const;
const char *StringKey(int *) const;
const char *StringKey(int &) const;
long long IntValue(const char *) const;
double FloatValue(const char *) const;
const char *StringValue(const char *) const;
const char *StringValue(const char *, int*) const;
const char *StringValue(const char *, int&) const;
const char *BinaryValue(const char *) const;
const char *BinaryValue(const char *, int*) const;
const char *BinaryValue(const char *, int&) const;
int UnCompressBinaryValue(const char *name,char **buf,int *lenp);
//Uncompress Binary Value without check compressflag
int UnCompressBinaryValueForce(const char *name,char **buf,int *lenp);
const char * UnCompressErrorMessage() const;
long long IntValue(int) const;
double FloatValue(int) const;
const char *StringValue(int) const;
const char *StringValue(int, int*) const;
const char *StringValue(int, int&) const;
const char *BinaryValue(int) const;
const char *BinaryValue(int, int*) const;
const char *BinaryValue(int, int&) const;
int FetchRow(void);
int Rewind(void);
};
class ServerPool {
private:
void *addr;
long check;
ServerPool(ServerPool &);
public:
friend class Server;
friend class Request;
friend class Result;
ServerPool(int maxServers, int maxRequests);
~ServerPool(void);
int GetEpollFD(int size);
int AddServer(Server *srv, int mReq=1, int mConn=0);
int AddRequest(Request *, long long);
int AddRequest(Request *, long long, long long);
int AddRequest(Request *, long long, const char *);
int AddRequest(Request *, long long, const char *, int);
int AddRequest(Request *, void *);
int AddRequest(Request *, void *, long long);
int AddRequest(Request *, void *, const char *);
int AddRequest(Request *, void *, const char *, int);
int Execute(int msec);
int ExecuteAll(int msec);
int CancelRequest(int);
int CancelAllRequest(int type);
int AbortRequest(int);
int AbortAllRequest(int type);
Result *GetResult(void);
Result *GetResult(int);
int GetResult(Result&);
int GetResult(Result&, int);
int ServerCount(void) const;
int RequestCount(int type) const;
int RequestState(int reqId) const;
};
const int WAIT = 1;
const int SEND = 2;
const int RECV = 4;
const int DONE = 8;
const int ALL_STATE = WAIT|SEND|RECV|DONE;
enum {
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalud field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
};
enum {
ER_HASHCHK=1000,
ER_NISAMCHK=1001,
ER_NO=1002,
ER_YES=1003,
ER_CANT_CREATE_FILE=1004,
ER_CANT_CREATE_TABLE=1005,
ER_CANT_CREATE_DB=1006,
ER_DB_CREATE_EXISTS=1007,
ER_DB_DROP_EXISTS=1008,
ER_DB_DROP_DELETE=1009,
ER_DB_DROP_RMDIR=1010,
ER_CANT_DELETE_FILE=1011,
ER_CANT_FIND_SYSTEM_REC=1012,
ER_CANT_GET_STAT=1013,
ER_CANT_GET_WD=1014,
ER_CANT_LOCK=1015,
ER_CANT_OPEN_FILE=1016,
ER_FILE_NOT_FOUND=1017,
ER_CANT_READ_DIR=1018,
ER_CANT_SET_WD=1019,
ER_CHECKREAD=1020,
ER_DISK_FULL=1021,
ER_DUP_KEY=1022,
ER_ERROR_ON_CLOSE=1023,
ER_ERROR_ON_READ=1024,
ER_ERROR_ON_RENAME=1025,
ER_ERROR_ON_WRITE=1026,
ER_FILE_USED=1027,
ER_FILSORT_ABORT=1028,
ER_FORM_NOT_FOUND=1029,
ER_GET_ERRNO=1030,
ER_ILLEGAL_HA=1031,
ER_KEY_NOT_FOUND=1032,
ER_NOT_FORM_FILE=1033,
ER_NOT_KEYFILE=1034,
ER_OLD_KEYFILE=1035,
ER_OPEN_AS_READONLY=1036,
ER_OUTOFMEMORY=1037,
ER_OUT_OF_SORTMEMORY=1038,
ER_UNEXPECTED_EOF=1039,
ER_CON_COUNT_ERROR=1040,
ER_OUT_OF_RESOURCES=1041,
ER_BAD_HOST_ERROR=1042,
ER_HANDSHAKE_ERROR=1043,
ER_DBACCESS_DENIED_ERROR=1044,
ER_ACCESS_DENIED_ERROR=1045,
ER_NO_DB_ERROR=1046,
ER_UNKNOWN_COM_ERROR=1047,
ER_BAD_NULL_ERROR=1048,
ER_BAD_DB_ERROR=1049,
ER_TABLE_EXISTS_ERROR=1050,
ER_BAD_TABLE_ERROR=1051,
ER_NON_UNIQ_ERROR=1052,
ER_SERVER_SHUTDOWN=1053,
ER_BAD_FIELD_ERROR=1054,
ER_WRONG_FIELD_WITH_GROUP=1055,
ER_WRONG_GROUP_FIELD=1056,
ER_WRONG_SUM_SELECT=1057,
ER_WRONG_VALUE_COUNT=1058,
ER_TOO_LONG_IDENT=1059,
ER_DUP_FIELDNAME=1060,
ER_DUP_KEYNAME=1061,
ER_DUP_ENTRY=1062,
ER_WRONG_FIELD_SPEC=1063,
ER_PARSE_ERROR=1064,
ER_EMPTY_QUERY=1065,
ER_NONUNIQ_TABLE=1066,
ER_INVALID_DEFAULT=1067,
ER_MULTIPLE_PRI_KEY=1068,
ER_TOO_MANY_KEYS=1069,
ER_TOO_MANY_KEY_PARTS=1070,
ER_TOO_LONG_KEY=1071,
ER_KEY_COLUMN_DOES_NOT_EXITS=1072,
ER_BLOB_USED_AS_KEY=1073,
ER_TOO_BIG_FIELDLENGTH=1074,
ER_WRONG_AUTO_KEY=1075,
ER_READY=1076,
ER_NORMAL_SHUTDOWN=1077,
ER_GOT_SIGNAL=1078,
ER_SHUTDOWN_COMPLETE=1079,
ER_FORCING_CLOSE=1080,
ER_IPSOCK_ERROR=1081,
ER_NO_SUCH_INDEX=1082,
ER_WRONG_FIELD_TERMINATORS=1083,
ER_BLOBS_AND_NO_TERMINATED=1084,
ER_TEXTFILE_NOT_READABLE=1085,
ER_FILE_EXISTS_ERROR=1086,
ER_LOAD_INFO=1087,
ER_ALTER_INFO=1088,
ER_WRONG_SUB_KEY=1089,
ER_CANT_REMOVE_ALL_FIELDS=1090,
ER_CANT_DROP_FIELD_OR_KEY=1091,
ER_INSERT_INFO=1092,
ER_INSERT_TABLE_USED=1093,
ER_NO_SUCH_THREAD=1094,
ER_KILL_DENIED_ERROR=1095,
ER_NO_TABLES_USED=1096,
ER_TOO_BIG_SET=1097,
ER_NO_UNIQUE_LOGFILE=1098,
ER_TABLE_NOT_LOCKED_FOR_WRITE=1099,
ER_TABLE_NOT_LOCKED=1100,
ER_BLOB_CANT_HAVE_DEFAULT=1101,
ER_WRONG_DB_NAME=1102,
ER_WRONG_TABLE_NAME=1103,
ER_TOO_BIG_SELECT=1104,
ER_UNKNOWN_ERROR=1105,
ER_UNKNOWN_PROCEDURE=1106,
ER_WRONG_PARAMCOUNT_TO_PROCEDURE=1107,
ER_WRONG_PARAMETERS_TO_PROCEDURE=1108,
ER_UNKNOWN_TABLE=1109,
ER_FIELD_SPECIFIED_TWICE=1110,
ER_INVALID_GROUP_FUNC_USE=1111,
ER_UNSUPPORTED_EXTENSION=1112,
ER_TABLE_MUST_HAVE_COLUMNS=1113,
ER_RECORD_FILE_FULL=1114,
ER_UNKNOWN_CHARACTER_SET=1115,
ER_TOO_MANY_TABLES=1116,
ER_TOO_MANY_FIELDS=1117,
ER_TOO_BIG_ROWSIZE=1118,
ER_STACK_OVERRUN=1119,
ER_WRONG_OUTER_JOIN=1120,
ER_NULL_COLUMN_IN_INDEX=1121,
ER_CANT_FIND_UDF=1122,
ER_CANT_INITIALIZE_UDF=1123,
ER_UDF_NO_PATHS=1124,
ER_UDF_EXISTS=1125,
ER_CANT_OPEN_LIBRARY=1126,
ER_CANT_FIND_DL_ENTRY=1127,
ER_FUNCTION_NOT_DEFINED=1128,
ER_HOST_IS_BLOCKED=1129,
ER_HOST_NOT_PRIVILEGED=1130,
ER_PASSWORD_ANONYMOUS_USER=1131,
ER_PASSWORD_NOT_ALLOWED=1132,
ER_PASSWORD_NO_MATCH=1133,
ER_UPDATE_INFO=1134,
ER_CANT_CREATE_THREAD=1135,
ER_WRONG_VALUE_COUNT_ON_ROW=1136,
ER_CANT_REOPEN_TABLE=1137,
ER_INVALID_USE_OF_NULL=1138,
ER_REGEXP_ERROR=1139,
ER_MIX_OF_GROUP_FUNC_AND_FIELDS=1140,
ER_NONEXISTING_GRANT=1141,
ER_TABLEACCESS_DENIED_ERROR=1142,
ER_COLUMNACCESS_DENIED_ERROR=1143,
ER_ILLEGAL_GRANT_FOR_TABLE=1144,
ER_GRANT_WRONG_HOST_OR_USER=1145,
ER_NO_SUCH_TABLE=1146,
ER_NONEXISTING_TABLE_GRANT=1147,
ER_NOT_ALLOWED_COMMAND=1148,
ER_SYNTAX_ERROR=1149,
ER_DELAYED_CANT_CHANGE_LOCK=1150,
ER_TOO_MANY_DELAYED_THREADS=1151,
ER_ABORTING_CONNECTION=1152,
ER_NET_PACKET_TOO_LARGE=1153,
ER_NET_READ_ERROR_FROM_PIPE=1154,
ER_NET_FCNTL_ERROR=1155,
ER_NET_PACKETS_OUT_OF_ORDER=1156,
ER_NET_UNCOMPRESS_ERROR=1157,
ER_NET_READ_ERROR=1158,
ER_NET_READ_INTERRUPTED=1159,
ER_NET_ERROR_ON_WRITE=1160,
ER_NET_WRITE_INTERRUPTED=1161,
ER_TOO_LONG_STRING=1162,
ER_TABLE_CANT_HANDLE_BLOB=1163,
ER_TABLE_CANT_HANDLE_AUTO_INCREMENT=1164,
ER_DELAYED_INSERT_TABLE_LOCKED=1165,
ER_WRONG_COLUMN_NAME=1166,
ER_WRONG_KEY_COLUMN=1167,
ER_WRONG_MRG_TABLE=1168,
ER_DUP_UNIQUE=1169,
ER_BLOB_KEY_WITHOUT_LENGTH=1170,
ER_PRIMARY_CANT_HAVE_NULL=1171,
ER_TOO_MANY_ROWS=1172,
ER_REQUIRES_PRIMARY_KEY=1173,
ER_NO_RAID_COMPILED=1174,
ER_UPDATE_WITHOUT_KEY_IN_SAFE_MODE=1175,
ER_KEY_DOES_NOT_EXITS=1176,
ER_CHECK_NO_SUCH_TABLE=1177,
ER_CHECK_NOT_IMPLEMENTED=1178,
ER_CANT_DO_THIS_DURING_AN_TRANSACTION=1179,
ER_ERROR_DURING_COMMIT=1180,
ER_ERROR_DURING_ROLLBACK=1181,
ER_ERROR_DURING_FLUSH_LOGS=1182,
ER_ERROR_DURING_CHECKPOINT=1183,
ER_NEW_ABORTING_CONNECTION=1184,
ER_DUMP_NOT_IMPLEMENTED= 1185,
ER_FLUSH_MASTER_BINLOG_CLOSED=1186,
ER_INDEX_REBUILD= 1187,
ER_MASTER=1188,
ER_MASTER_NET_READ=1189,
ER_MASTER_NET_WRITE=1190,
ER_FT_MATCHING_KEY_NOT_FOUND=1191,
ER_LOCK_OR_ACTIVE_TRANSACTION=1192,
ER_UNKNOWN_SYSTEM_VARIABLE=1193,
ER_CRASHED_ON_USAGE=1194,
ER_CRASHED_ON_REPAIR=1195,
ER_WARNING_NOT_COMPLETE_ROLLBACK=1196,
ER_TRANS_CACHE_FULL=1197,
ER_SLAVE_MUST_STOP=1198,
ER_SLAVE_NOT_RUNNING=1199,
ER_BAD_SLAVE=1200,
ER_MASTER_INFO=1201,
ER_SLAVE_THREAD=1202,
ER_TOO_MANY_USER_CONNECTIONS=1203,
ER_SET_CONSTANTS_ONLY=1204,
ER_LOCK_WAIT_TIMEOUT=1205,
ER_LOCK_TABLE_FULL=1206,
ER_READ_ONLY_TRANSACTION=1207,
ER_DROP_DB_WITH_READ_LOCK=1208,
ER_CREATE_DB_WITH_READ_LOCK=1209,
ER_WRONG_ARGUMENTS=1210,
ER_NO_PERMISSION_TO_CREATE_USER=1211,
ER_UNION_TABLES_IN_DIFFERENT_DIR=1212,
ER_LOCK_DEADLOCK=1213,
ER_TABLE_CANT_HANDLE_FULLTEXT=1214,
ER_CANNOT_ADD_FOREIGN=1215,
ER_NO_REFERENCED_ROW=1216,
ER_ROW_IS_REFERENCED=1217,
ER_CONNECT_TO_MASTER=1218,
ER_QUERY_ON_MASTER=1219,
ER_ERROR_WHEN_EXECUTING_COMMAND=1220,
ER_WRONG_USAGE=1221,
ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT=1222,
ER_CANT_UPDATE_WITH_READLOCK=1223,
ER_MIXING_NOT_ALLOWED=1224,
ER_DUP_ARGUMENT=1225,
ER_USER_LIMIT_REACHED=1226,
ER_SPECIFIC_ACCESS_DENIED_ERROR=1227,
ER_LOCAL_VARIABLE=1228,
ER_GLOBAL_VARIABLE=1229,
ER_NO_DEFAULT=1230,
ER_WRONG_VALUE_FOR_VAR=1231,
ER_WRONG_TYPE_FOR_VAR=1232,
ER_VAR_CANT_BE_READ=1233,
ER_CANT_USE_OPTION_HERE=1234,
ER_NOT_SUPPORTED_YET=1235,
ER_MASTER_FATAL_ERROR_READING_BINLOG=1236,
ER_SLAVE_IGNORED_TABLE=1237,
ER_INCORRECT_GLOBAL_LOCAL_VAR=1238,
CR_UNKNOWN_ERROR=1900,
CR_SOCKET_CREATE_ERROR=1901,
CR_CONNECTION_ERROR=1902,
CR_CONN_HOST_ERROR=1903,
CR_IPSOCK_ERROR =1904,
CR_UNKNOWN_HOST =1905,
CR_SERVER_GONE_ERROR=1906,
CR_VERSION_ERROR=1907,
CR_OUT_OF_MEMORY=1908,
CR_WRONG_HOST_INFO=1909,
CR_LOCALHOST_CONNECTION=1910,
CR_TCP_CONNECTION=1911,
CR_SERVER_HANDSHAKE_ERR=1912,
CR_SERVER_LOST=1913,
CR_COMMANDS_OUT_OF_SYNC=1914,
CR_NAMEDPIPE_CONNECTION=1915,
CR_NAMEDPIPEWAIT_ERROR=1916,
CR_NAMEDPIPEOPEN_ERROR=1917,
CR_NAMEDPIPESETSTATE_ERROR=1918,
CR_CANT_READ_CHARSET=1919,
CR_NET_PACKET_TOO_LARGE=1920,
CR_EMBEDDED_CONNECTION=1921,
CR_PROBE_SLAVE_STATUS=1922,
CR_PROBE_SLAVE_HOSTS=1923,
CR_PROBE_SLAVE_CONNECT=1924,
CR_PROBE_MASTER_CONNECT=1925,
CR_SSL_CONNECTION_ERROR=1926,
CR_MALFORMED_PACKET=1927,
CR_WRONG_LICENSE=1928,
};
};
#endif

84
src/search_local/index_storage/api/c_api_cc/examples/update/updatetest.cpp Normal file
View File

@ -0,0 +1,84 @@
#include<stdio.h>
#include<stdlib.h>
#include<string>
#include<stdint.h>
#include<unistd.h>
#include "dtcapi.h"
int main(int argc,char* argv[])
{
int retCode = 0;
unsigned int uid;
unsigned int age;
std::string name;
std::string city;
std::string descr;
DTC::Server stServer; // 只要server不析构，后台会保持长连接
stServer.IntKey(); // 声明key类型
stServer.SetTableName("t_dtc_example");//设置dtc的表名，与table.conf中tablename应该一样
stServer.SetAddress("192.168.214.62", "10009");//设置的dtc的ip和端口
stServer.SetTimeout(5); // 设置网络超时时间
stServer.SetAccessKey("0000010284d9cfc2f395ce883a41d7ffc1bbcf4e"); // 设置访问码 AccessToken，在申请dtc实例的时候网站端会生成
uid = atoi(argv[1]);
name = std::string(argv[2]);
city = std::string(argv[3]);
descr = std::string(argv[4]);
age = atoi(argv[5]);
DTC::UpdateRequest UpdateReq(&stServer);
retCode = UpdateReq.SetKey(uid);
if(retCode != 0)
{
printf("update-req set key error: %d", retCode);
fflush(stdout);
return(-1);
}
retCode = UpdateReq.Set("name", name.c_str());
retCode = UpdateReq.Set("city", city.c_str());
retCode = UpdateReq.Set("descr", descr.c_str());
retCode = UpdateReq.Set("age", age);
if(retCode != 0)
{
printf("update-req set field error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
DTC::Result stResult;
retCode = UpdateReq.Execute(stResult);
printf("retCode:%d\n", retCode);
if(retCode == 0)
{
DTC::GetRequest getReq(&stServer);
getReq.SetKey(uid);
if(retCode == 0)
retCode = getReq.Need("uid");//设置需要select的字段，注意第一个key字段不能在这里出现
if(retCode == 0)
retCode = getReq.Need("name");
if(retCode == 0)
retCode = getReq.Need("city");
if(retCode == 0)
retCode = getReq.Need("descr");
if(retCode == 0)
retCode = getReq.Need("age");
if(retCode != 0)
{
printf("get-req set key or need error: %d", retCode);
fflush(stdout);
return(-1);
}
// execute & get result
retCode = getReq.Execute(stResult);
retCode = stResult.FetchRow();//开始获取数据
printf("uid:%d\n", stResult.IntValue("uid"));
printf("name: %s\n", stResult.StringValue("name"));//输出binary类型的数据
printf("city: %s\n", stResult.StringValue("city"));
printf("descr: %s\n", stResult.BinaryValue("descr"));
printf("age: %d\n", stResult.IntValue("age"));//输出int类型的数据
}
return 0;
}

141
src/search_local/index_storage/api/c_api_cc/keylist.cc Normal file
View File

@ -0,0 +1,141 @@
#include <errno.h>
#include "memcheck.h"
#include "protocol.h"
#include "keylist.h"
#include "cache_error.h"
int NCKeyValueList::KeyValueMax = 32;
NCKeyInfo::~NCKeyInfo()
{
for(int i = 0; i < 8; ++i)
{
if(keyName[i])
{
free(keyName[i]);
keyName[i] = NULL;
}
}
}
int NCKeyInfo::KeyIndex(const char *n) const
{
namemap_t::const_iterator i = keyMap.find(n);
return i == keyMap.end() ? -1 : i->second;
}
int NCKeyInfo::AddKey(const char* name, int type)
{
switch(type){
case DField::Signed:
case DField::String:
break;
default:
return -EC_BAD_KEY_TYPE;
}
if(KeyIndex(name) >= 0) {
//return -EC_DUPLICATE_FIELD;
// ignore duplicate key field name
// because NCKeyInfo may be initialized by CheckInternalService()
// add again must be allowed for code compatibility
return 0;
}
int cnt = KeyFields();
if(cnt >= (int)(sizeof(keyType)/sizeof(keyType[0]))) return -EC_BAD_MULTIKEY;
char* localName = (char*)malloc(strlen(name)+1);
strcpy(localName, name);
keyName[cnt] = localName;
keyMap[localName] = cnt;
keyType[cnt] = type;
keyCount++;
return 0;
}
int NCKeyInfo::Equal(const NCKeyInfo &other) const {
int n = KeyFields();
// key field count mis-match
if(other.KeyFields() != n)
return 0;
// key type mis-match
if(memcmp(keyType, other.keyType, n)!=0)
return 0;
for(int i=0; i<n; i++) {
// key name mis-match
if(strcasecmp(keyName[i], other.keyName[i]) != 0)
return 0;
}
return 1;
}
void NCKeyValueList::Unset(void) {
if(kcount) {
const int kn = KeyFields();
for(int i=0; i<kcount; i++){
for(int j=0; j<kn; j++)
if(KeyType(j) == DField::String || KeyType(j) == DField::Binary)
FREE(Value(i, j).bin.ptr);
}
FREE_CLEAR(val);
memset(fcount, 0, sizeof(fcount));
kcount = 0;
}
}
int NCKeyValueList::AddValue(const char* name, const DTCValue &v, int type)
{
const int kn = KeyFields();
int col = keyinfo->KeyIndex(name);
if(col < 0 || col >= kn)
return -EC_BAD_KEY_NAME;
switch(KeyType(col)){
case DField::Signed:
case DField::Unsigned:
if(type != DField::Signed && type != DField::Unsigned)
return -EC_BAD_VALUE_TYPE;
break;
case DField::String:
case DField::Binary:
if(type != DField::String && type != DField::Binary)
return -EC_BAD_VALUE_TYPE;
if(v.bin.len > 255)
return -EC_KEY_OVERFLOW;
break;
default:
return -EC_BAD_KEY_TYPE;
}
int row = fcount[col];
if(row >= KeyValueMax)
return -EC_TOO_MANY_KEY_VALUE; // key值太多
if(row >= kcount){
if(REALLOC(val, (kcount+1)*kn*sizeof(DTCValue)) == NULL)
throw std::bad_alloc();
memset(&Value(row, 0), 0, kn*sizeof(DTCValue));
kcount++;
}
DTCValue &slot = Value(row, col);
switch(KeyType(col)){
case DField::Signed:
case DField::Unsigned:
slot = v;
break;
case DField::String:
case DField::Binary:
slot.bin.len = v.bin.len;
slot.bin.ptr = (char *)MALLOC(v.bin.len+1);
if(slot.bin.ptr==NULL)
throw std::bad_alloc();
memcpy(slot.bin.ptr, v.bin.ptr, v.bin.len);
slot.bin.ptr[v.bin.len] = '\0';
break;
}
fcount[col]++;
return 0;
}

90
src/search_local/index_storage/api/c_api_cc/keylist.h Normal file
View File

@ -0,0 +1,90 @@
#ifndef __CHC_KEYLIST_H
#define __CHC_KEYLIST_H
#include <string.h>
#include <map>
#include <value.h>
class NCKeyInfo {
private:
struct nocase
{
bool operator()(const char * const & a, const char * const & b) const
{ return strcasecmp(a, b) < 0; }
};
typedef std::map<const char *, int, nocase> namemap_t;
private:
namemap_t keyMap;
// const char *keyName[8];
char *keyName[8];
uint8_t keyType[8];
int keyCount;
public:
// zero is KeyField::None
void Clear(void) {
keyCount = 0;
memset(keyType, 0, sizeof(keyType));
memset(keyName, 0, sizeof(keyName));
keyMap.clear();
}
NCKeyInfo() {
keyCount = 0;
memset(keyType, 0, sizeof(keyType));
memset(keyName, 0, sizeof(keyName));
}
NCKeyInfo(const NCKeyInfo&that) {
keyCount = that.keyCount;
memcpy(keyType, that.keyType, sizeof(keyType));
memcpy(keyName, that.keyName, sizeof(keyName));
for(int i=0; i<keyCount; i++)
keyMap[keyName[i]] = i;
}
//modified by neolv to fix char* name
~NCKeyInfo();
int AddKey(const char *name, int type);
int Equal(const NCKeyInfo &other) const;
int KeyIndex(const char *n) const;
const char *KeyName(int n) const { return keyName[n]; }
int KeyType(int id) const { return keyType[id]; }
int KeyFields(void) const { return keyCount; }
};
class NCKeyValueList {
public:
static int KeyValueMax;
public:
NCKeyInfo *keyinfo;
DTCValue *val;
int fcount[8];
int kcount;
public:
NCKeyValueList(void) : keyinfo(NULL), val(NULL), kcount(0) {
memset(fcount, 0, sizeof(fcount));
}
~NCKeyValueList() {
FREE_CLEAR(val);
}
int KeyFields(void) const { return keyinfo->KeyFields(); }
int KeyType(int id) const { return keyinfo->KeyType(id); }
int KeyCount(void) const { return kcount; }
const char * KeyName(int id) const { return keyinfo->KeyName(id); }
void Unset(void);
int AddValue(const char *, const DTCValue &, int);
DTCValue &Value(int row, int col) { return val[row*keyinfo->KeyFields()+col]; }
const DTCValue &Value(int row, int col) const { return val[row*keyinfo->KeyFields()+col]; }
DTCValue & operator()(int row, int col) { return Value(row, col); }
const DTCValue & operator()(int row, int col) const { return Value(row, col); }
int IsFlat(void) const {
for(int i=1; i<keyinfo->KeyFields(); i++)
if(fcount[0] != fcount[i])
return 0;
return 1;
}
};
#endif

11
src/search_local/index_storage/api/c_api_cc/log_client.cc Normal file
View File

@ -0,0 +1,11 @@
#include "dtcapi.h"
#include <log.cc>
__EXPORT
void DTC::write_log (int level,
const char*file, const char *func, int lineno,
const char *fmt, ...)
__attribute__((format(printf,5,6)))
__attribute__((__alias__("_write_log_")))
;

3
src/search_local/index_storage/api/c_api_cc/readme.txt Normal file
View File

@ -0,0 +1,3 @@
注意事项：
1.auto_dtcc++.sh 该脚本是需要root权限去执行的，它的作用是将dtc sdk 的动态库
放置在 /usr/lib 目录下，并且执行命令 ldconfig 使配置生效

33
src/search_local/index_storage/api/c_api_cc/somain.c Normal file
View File

@ -0,0 +1,33 @@
#undef _GNU_SOURCE
#define _GNU_SOURCE
#include <stdlib.h>
#include <unistd.h>
#include <version.h>
#include <compiler.h>
const char __invoke_dynamic_linker__[]
__attribute__ ((section (".interp")))
__HIDDEN
=
#if __x86_64__
"/lib64/ld-linux-x86-64.so.2"
#else
"/lib/ld-linux.so.2"
#endif
;
__HIDDEN
void _so_start(char *arg1,...) {
#define BANNER "DTC client API v" DTC_VERSION_DETAIL "\n" \
" - TCP connection supported\n" \
" - UDP connection supported\n" \
" - UNIX stream connection supported\n" \
" - embeded threading connection supported\n" \
" - protocol packet encode/decode interface supported\n" \
" - async execute (except embeded threading) supported \n"
int unused;
unused = write(1, BANNER, sizeof(BANNER)-1);
_exit(0);
}

240
src/search_local/index_storage/api/c_api_cc/udppool.cc Normal file
View File

@ -0,0 +1,240 @@
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <netinet/in.h>
#include <net/if.h>
#include <pthread.h>
#include <arpa/inet.h>
#include <string.h>
#include <unistd.h>
#include <alloca.h>
#include "udppool.h"
#include "log.h"
static unsigned int bindip = 0xFFFFFFFF;
unsigned int GetBindIp(void)
{
const char *name = getenv("DTCAPI_UDP_INTERFACE");
if (name == NULL || name[0] == 0 || strcmp(name, "*") == 0)
{
return 0;
}
int fd = socket(AF_INET, SOCK_DGRAM, 0);
struct ifconf ifc;
struct ifreq *ifr = NULL;
int n = 0;
int i;
if (fd < 0)
return 0;
ifc.ifc_len = 0;
ifc.ifc_req = NULL;
if (ioctl(fd, SIOCGIFCONF, &ifc) == 0)
{
ifr = (struct ifreq *) alloca(ifc.ifc_len > 128 ? ifc.ifc_len : 128);
ifc.ifc_req = ifr;
if (ioctl(fd, SIOCGIFCONF, &ifc) == 0)
n = ifc.ifc_len / sizeof(struct ifreq);
}
close(fd);
for (i = 0; i < n; i++)
{
if (strncmp(ifr[i].ifr_name, name, sizeof(ifr[i].ifr_name)) != 0)
continue;
if (ifr[i].ifr_addr.sa_family == AF_INET)
return ((struct sockaddr_in *)&ifr[i].ifr_addr)->sin_addr.s_addr;
}
return 0;
}
static int CreatePortIpv4(void)
{
int fd = socket(AF_INET, SOCK_DGRAM, 0);
if (bindip == 0xFFFFFFFF)
bindip = GetBindIp();
if (bindip != 0)
{
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = bindip;
if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0)
{
close(fd);
return -1;
}
}
return fd;
}
static int CreatePortIpv6(void)
{
int fd = socket(AF_INET6, SOCK_DGRAM, 0);
return fd;
}
static int CreatePortUnix(void)
{
int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
snprintf(addr.sun_path, sizeof(addr.sun_path),
"@dtcapi-global-%d-%d-%d", getpid(), fd, (int)time(NULL));
socklen_t len = SUN_LEN(&addr);
addr.sun_path[0] = 0;
if (bind(fd, (const struct sockaddr *)&addr, len) < 0)
{
close(fd);
return -1;
}
return fd;
}
struct NCUdpPortList {
public:
int stopped;
int family;
int (*newport)(void);
pthread_mutex_t lock;
NCUdpPort *list;
public:
NCUdpPort *Get(void);
void Put(NCUdpPort *);
public:
~NCUdpPortList(void);
};
// this destructor only called when unloading libdtc.so
NCUdpPortList::~NCUdpPortList(void)
{
if (pthread_mutex_lock(&lock) == 0)
{
stopped = 1;
while (list != NULL)
{
NCUdpPort *port = list;
list = port->next;
delete port;
}
pthread_mutex_unlock(&lock);
}
}
NCUdpPort *NCUdpPortList::Get(void)
{
NCUdpPort *port = NULL;
if (pthread_mutex_lock(&lock) == 0)
{
if (list != NULL)
{
port = list;
list = port->next;
}
pthread_mutex_unlock(&lock);
}
else
{
log_error("api mutex_lock error,may have fd leak");
}
if (port != NULL)
{
if (getpid() == port->pid)
{
port->sn++;
}
else
{
delete port;
port = NULL;
}
}
if (port == NULL)
{
int fd = newport();
if (fd > 0)
{
port = new NCUdpPort;
port->fd = fd;
unsigned int seed = fd + (long)port + (long) & port + (long)pthread_self() + (long)port;
port->sn = rand_r(&seed);
port->pid = getpid();
port->timeout = -1;
port->family = family;
}
}
return port;
}
void NCUdpPortList::Put(NCUdpPort *port)
{
if (this != NULL && pthread_mutex_lock(&lock) == 0)
{
if(stopped) {
// always delete port after unloading process
port->Eat();
} else {
port->next = list;
list = port;
}
pthread_mutex_unlock(&lock);
}
else
{
port->Eat();
}
}
static NCUdpPortList ipv4List = { 0, AF_INET, CreatePortIpv4, PTHREAD_MUTEX_INITIALIZER, NULL };
static NCUdpPortList ipv6List = { 0, AF_INET6, CreatePortIpv6, PTHREAD_MUTEX_INITIALIZER, NULL };
static NCUdpPortList unixList = { 0, AF_UNIX, CreatePortUnix, PTHREAD_MUTEX_INITIALIZER, NULL };
struct NCUdpPortList *GetPortList(int family) {
switch(family) {
case AF_INET:
return &ipv4List;
case AF_INET6:
return &ipv6List;
case AF_UNIX:
return &unixList;
}
return NULL;
}
NCUdpPort *NCUdpPort::Get(int family)
{
NCUdpPortList *portList = GetPortList(family);
if(portList == NULL) {
return NULL;
}
return portList->Get();
}
void NCUdpPort::Put()
{
NCUdpPortList *portList = GetPortList(family);
portList->Put(this);
}

62
src/search_local/index_storage/api/c_api_cc/udppool.h Normal file
View File

@ -0,0 +1,62 @@
#include <stdlib.h>
class NCPort
{
public:
uint64_t sn;
int fd;
int timeout;//the timeout this socket is.
public:
NCPort()
{
sn = 0;
fd = -1;
timeout = -1;
}
NCPort(const NCPort &that)
{
sn = that.sn;
fd = that.fd;
timeout = that.timeout;
}
~NCPort()
{
if (fd >= 0)
{
close(fd);
}
}
};
class NCUdpPortList;
class NCUdpPort: public NCPort
{
public:
int family;
pid_t pid;
private:
NCUdpPort *next;
private:
NCUdpPort()
{
pid = -1;
};
~NCUdpPort()
{
};
public:
friend class NCUdpPortList;
static NCUdpPort *Get(int family); // get from cache
void Put(); // put back cache
void Eat()
{
delete this;
} // eat and delete
};

101
src/search_local/index_storage/common/Condition.h Normal file
View File

@ -0,0 +1,101 @@
/*
* =====================================================================================
*
* Filename: condition.h
*
* Description: conditional thread operation.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __DTC_CONDITION_H__
#define __DTC_CONDITION_H__
#include "non_copyable.h"
class condition : private noncopyable
{
public:
condition(void)
{
pthread_mutex_init(&m_lock, NULL);
pthread_cond_init(&m_cond, NULL);
}
~condition(void)
{
pthread_mutex_destroy(&m_lock);
pthread_cond_destroy(&m_cond);
}
void notify_one(void)
{
pthread_cond_signal(&m_cond);
}
void notify_all(void)
{
pthread_cond_broadcast(&m_cond);
}
void wait(void)
{
pthread_cond_wait(&m_cond, &m_lock);
}
void lock(void)
{
pthread_mutex_lock(&m_lock);
}
void unlock(void)
{
pthread_mutex_unlock(&m_lock);
}
private:
pthread_cond_t m_cond;
pthread_mutex_t m_lock;
};
class copedEnterCritical
{
public:
copedEnterCritical(condition &c) : m_cond(c)
{
m_cond.lock();
}
~copedEnterCritical(void)
{
m_cond.unlock();
}
private:
condition &m_cond;
};
class copedLeaveCritical
{
public:
copedLeaveCritical(condition &c) : m_cond(c)
{
m_cond.unlock();
}
~copedLeaveCritical(void)
{
m_cond.lock();
}
private:
condition &m_cond;
};
#endif //__DTC_CONDITION_H__

42
src/search_local/index_storage/common/Makefile Normal file
View File

@ -0,0 +1,42 @@
include ../Make.conf
target = libcommon.a libcommon_nopic.a
CFLAGS+=-I../api/c_api -I. -I../watchdog -I../stat -I../../../3rdlib/attr_api $(ZINC)
LIBS = $(Z_LIB)
VPATH = ../watchdog
#SVN_VERSION = $(shell test -d .svn && (svn info ../ | grep "Last Changed Rev" | cut -d " " -f 4))
GIT_VERSION=$(shell git log | head -n 4 | grep "commit" | cut -d " " -f 2 | cut -c 1-7)
ifeq "$(GIT_VERSION)a" "a"
GIT_REVISION = "(unknown)"
endif
$(shell sed -i '11s/"[^"]*"/"$(GIT_VERSION)"/' version.h)
filelist := md5 value decode encode field section table_def \
packet_base packet_server packet_result \
task_const task_base task_pkey task_server task_copy task_request task_api \
poller timer_list config version system_lock log log_stat log_alert \
shmem mem_check dbconfig dbconfig_tdef admin_tdef \
daemon daemon_wait proc_title thread poll_thread poll_thread_group thread_cpu_stat \
buffer localip \
key_guard bitsop net_addr net_bind blacklist blacklist_unit \
multi_request task_multiplexer \
helper_client helper_collect helper_group \
listener listener_pool decoder_base \
client_sync client_async client_unit client_dgram \
plugin_request plugin_mgr plugin_dgram plugin_sync \
plugin_worker plugin_unit plugin_listener_pool \
plugin_global plugin_decoder plugin_timer dtc_global\
compress client_resource_pool \
new_hash dynamic_helper_collection consistent_hash_selector chash\
file_backed_key_set markup_stl parse_cluster_config key_route \
proxy_client proxy_client_unit proxy_listener proxy_listen_pool proxy_process proxy_receiver proxy_sender \
proxy_multi_request plugin_proxy_mgr plugin_proxy_listener_pool watchdog_fork hotback_task table_def_manager
libcommon_objs := $(filelist:%=%_pic.o)
%_pic.o: CFLAGS += -pthread -fPIC
libcommon_nopic_objs := $(filelist:%=%.o)
%.o: CFLAGS += -pthread
include ../Make.rules

65
src/search_local/index_storage/common/admin_tdef.cc Normal file
View File

@ -0,0 +1,65 @@
/*
* =====================================================================================
*
* Filename: admin_tdef.cc
*
* Description: table definition.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "log.h"
#include "admin_tdef.h"
#include "table_def.h"
#include "table_def_manager.h"
#include "protocol.h"
#include "dbconfig.h"
static struct FieldConfig HBTabField[] = {
{"type", DField::Unsigned, 4, DTCValue::Make(0), 0},
{"flag", DField::Unsigned, 1, DTCValue::Make(0), 0},
{"key", DField::Binary, 255, DTCValue::Make(0), 0},
{"value", DField::Binary, MAXPACKETSIZE, DTCValue::Make(0), 0},
};
DTCTableDefinition *build_hot_backup_table(void)
{
DTCTableDefinition *tdef = new DTCTableDefinition(4);
tdef->set_table_name("@HOT_BACKUP");
struct FieldConfig *field = HBTabField;
int field_cnt = sizeof(HBTabField) / sizeof(HBTabField[0]);
// build hotback table key info base on the actual user definite table
field[0].type = TableDefinitionManager::Instance()->get_cur_table_def()->key_type();
field[0].size = TableDefinitionManager::Instance()->get_cur_table_def()->field_size(0);
tdef->set_admin_table();
for (int i = 0; i < field_cnt; i++)
{
if (tdef->add_field(i, field[i].name, field[i].type, field[i].size) != 0)
{
log_error("add_field failed, name=%s, size=%d, type=%d",
field[i].name, field[i].size, field[i].type);
delete tdef;
return NULL;
}
tdef->set_default_value(i, &field[i].dval);
}
if (tdef->set_key_fields(1) < 0)
{
log_error("Table key size %d too large, must <= 255",
tdef->key_format() > 0 ? tdef->key_format() : -tdef->key_format());
delete tdef;
return NULL;
}
tdef->build_info_cache();
return tdef;
}

72
src/search_local/index_storage/common/admin_tdef.h Normal file
View File

@ -0,0 +1,72 @@
/*
* =====================================================================================
*
* Filename: admin_tdef.h
*
* Description: table definition.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ADMIN_TDEF_H__
#define __ADMIN_TDEF_H__
class DTCTableDefinition;
class DTCHotBackup
{
public:
//type
enum
{
SYNC_LRU = 1,
SYNC_INSERT = 2,
SYNC_UPDATE = 4,
SYNC_PURGE = 8,
SYNC_DELETE = 16,
SYNC_CLEAR = 32,
SYNC_COLEXPAND = 64,
SYNC_COLEXPAND_CMD = 128
};
//flag
enum
{
NON_VALUE = 1,
HAS_VALUE = 2,
EMPTY_NODE = 4,
KEY_NOEXIST = 8,
};
};
class DTCMigrate
{
public:
//type
enum
{
FROM_CLIENT = 1,
FROM_SERVER = 2
};
//flag
enum
{
NON_VALUE = 1,
HAS_VALUE = 2,
EMPTY_NODE = 4,
KEY_NOEXIST = 8,
};
};
extern DTCTableDefinition *build_hot_backup_table(void);
#define _DTC_HB_COL_EXPAND_ "_dtc_hb_col_expand_"
#define _DTC_HB_COL_EXPAND_DONE_ "_dtc_hb_col_expand_done_"
#endif

37
src/search_local/index_storage/common/atomic.h Normal file
View File

@ -0,0 +1,37 @@
/*
* =====================================================================================
*
* Filename: atomic.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ATOMIC__T
#define __ATOMIC__T
#include <stdint.h>
#if __GNUC__ < 4
#include "atomic_asm.h"
#else
#include "atomic_gcc.h"
#endif
#if __WORDSIZE == 64 || __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)
#define HAS_ATOMIC8 1
#include "atomic_gcc8.h"
#else
#define HAS_ATOMIC8 1
#include "atomic_asm8.h"
#endif
#endif

97
src/search_local/index_storage/common/atomic.hpp Normal file
View File

@ -0,0 +1,97 @@
#include "atomic.h"
struct AtomicU32
{
private:
typedef uint32_t V;
atomic_t count;
public:
~AtomicU32(void) {}
AtomicU32(V v = 0) { set(v); }
inline V get(void) const { return atomic_read((atomic_t *)&count); }
inline V set(V v)
{
atomic_set(&count, v);
return v;
}
inline V add(V v) { return atomic_add_return(v, &count); }
inline V sub(V v) { return atomic_sub_return(v, &count); }
inline V clear(void) { return atomic_clear(&count); }
inline V inc(void) { return add(1); }
inline V dec(void) { return sub(1); }
inline operator V(void) const { return get(); }
inline V operator=(V v) { return set(v); }
inline V operator+=(V v) { return add(v); }
inline V operator-=(V v) { return sub(v); }
inline V operator++(void) { return inc(); }
inline V operator--(void) { return dec(); }
inline V operator++(int) { return inc() - 1; }
inline V operator--(int) { return dec() + 1; }
};
struct AtomicS32
{
private:
typedef int32_t V;
atomic_t count;
public:
~AtomicS32(void) {}
AtomicS32(V v = 0) { set(v); }
inline V get(void) const { return atomic_read((atomic_t *)&count); }
inline V set(V v)
{
atomic_set(&count, v);
return v;
}
inline V add(V v) { return atomic_add_return(v, &count); }
inline V sub(V v) { return atomic_sub_return(v, &count); }
inline V clear(void) { return atomic_clear(&count); }
inline V inc(void) { return add(1); }
inline V dec(void) { return sub(1); }
inline operator V(void) const { return get(); }
inline V operator=(V v) { return set(v); }
inline V operator+=(V v) { return add(v); }
inline V operator-=(V v) { return sub(v); }
inline V operator++(void) { return inc(); }
inline V operator--(void) { return dec(); }
inline V operator++(int) { return inc() - 1; }
inline V operator--(int) { return dec() + 1; }
};
#if HAS_ATOMIC8
struct AtomicS64
{
private:
typedef int64_t V;
atomic8_t count;
public:
~AtomicS64(void) {}
AtomicS64(V v = 0) { set(v); }
inline V get(void) const { return atomic8_read((atomic8_t *)&count); }
inline V set(V v)
{
atomic8_set(&count, v);
return v;
}
inline V add(V v) { return atomic8_add_return(v, &count); }
inline V sub(V v) { return atomic8_sub_return(v, &count); }
inline V clear(void) { return atomic8_clear(&count); }
inline V inc(void) { return add(1); }
inline V dec(void) { return sub(1); }
inline operator V(void) const { return get(); }
inline V operator=(V v) { return set(v); }
inline V operator+=(V v) { return add(v); }
inline V operator-=(V v) { return sub(v); }
inline V operator++(void) { return inc(); }
inline V operator--(void) { return dec(); }
inline V operator++(int) { return inc() - 1; }
inline V operator--(int) { return dec() + 1; }
};
#endif

247
src/search_local/index_storage/common/atomic_asm.h Normal file
View File

@ -0,0 +1,247 @@
/*
* =====================================================================================
*
* Filename: atomic_asm.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ARCH_I386_ATOMIC__
#define __ARCH_I386_ATOMIC__
#include <sys/cdefs.h>
__BEGIN_DECLS
/* xaddl: require Modern 486+ processor */
/*
* Make sure gcc doesn't try to be clever and move things around
* on us. We need to use _exactly_ the address the user gave us,
* not some alias that contains the same information.
*/
typedef struct
{
volatile int counter;
} atomic_t;
#define ATOMIC_INIT(i) \
{ \
(i) \
}
/**
* atomic_read - read atomic variable
* @v: pointer of type atomic_t
*
* Atomically reads the value of @v.
*/
#define atomic_read(v) ((v)->counter)
/**
* atomic_set - set atomic variable
* @v: pointer of type atomic_t
* @i: required value
*
* Atomically sets the value of @v to @i.
*/
#define atomic_set(v, i) (((v)->counter) = (i))
static __inline__ int atomic_clear(atomic_t *v)
{
int i;
__asm__ __volatile__(
" xorl %0, %0\n"
"lock; xchgl %0, %1;"
: "=r"(i)
: "m"(v->counter));
return i;
}
/**
* atomic_add - add integer to atomic variable
* @i: integer value to add
* @v: pointer of type atomic_t
*
* Atomically adds @i to @v.
*/
static __inline__ int atomic_add(int i, atomic_t *v)
{
int __i = i;
__asm__ __volatile__(
"lock; xaddl %0, %1;"
: "=r"(i)
: "m"(v->counter), "0"(i));
return __i;
}
/**
* atomic_sub - subtract the atomic variable
* @i: integer value to subtract
* @v: pointer of type atomic_t
*
* Atomically subtracts @i from @v.
*/
static __inline__ int atomic_sub(int i, atomic_t *v)
{
return atomic_add(-i, v);
}
/**
* atomic_sub_and_test - subtract value from variable and test result
* @i: integer value to subtract
* @v: pointer of type atomic_t
*
* Atomically subtracts @i from @v and returns
* true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
{
unsigned char c;
__asm__ __volatile__(
"lock; subl %2,%0; sete %1"
: "=m"(v->counter), "=qm"(c)
: "ir"(i), "m"(v->counter)
: "memory");
return c;
}
/**
* atomic_inc - increment atomic variable
* @v: pointer of type atomic_t
*
* Atomically increments @v by 1.
*/
static __inline__ void atomic_inc(atomic_t *v)
{
__asm__ __volatile__(
"lock; incl %0"
: "=m"(v->counter)
: "m"(v->counter));
}
/**
* atomic_dec - decrement atomic variable
* @v: pointer of type atomic_t
*
* Atomically decrements @v by 1.
*/
static __inline__ void atomic_dec(atomic_t *v)
{
__asm__ __volatile__(
"lock; decl %0"
: "=m"(v->counter)
: "m"(v->counter));
}
/**
* atomic_dec_and_test - decrement and test
* @v: pointer of type atomic_t
*
* Atomically decrements @v by 1 and
* returns true if the result is 0, or false for all other
* cases.
*/
static __inline__ int atomic_dec_and_test(atomic_t *v)
{
unsigned char c;
__asm__ __volatile__(
"lock; decl %0; sete %1"
: "=m"(v->counter), "=qm"(c)
: "m"(v->counter)
: "memory");
return c != 0;
}
/**
* atomic_inc_and_test - increment and test
* @v: pointer of type atomic_t
*
* Atomically increments @v by 1
* and returns true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic_inc_and_test(atomic_t *v)
{
unsigned char c;
__asm__ __volatile__(
"lock; incl %0; sete %1"
: "=m"(v->counter), "=qm"(c)
: "m"(v->counter)
: "memory");
return c != 0;
}
/**
* atomic_add_negative - add and test if negative
* @v: pointer of type atomic_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns true
* if the result is negative, or false when
* result is greater than or equal to zero.
*/
static __inline__ int atomic_add_negative(int i, atomic_t *v)
{
unsigned char c;
__asm__ __volatile__(
"lock; addl %2,%0; sets %1"
: "=m"(v->counter), "=qm"(c)
: "ir"(i), "m"(v->counter)
: "memory");
return c;
}
/**
* atomic_add_return - add and return
* @v: pointer of type atomic_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns @i + @v
*/
static __inline__ int atomic_add_return(int i, atomic_t *v)
{
int __i = i;
__asm__ __volatile__(
"lock; xaddl %0, %1;"
: "=r"(i)
: "m"(v->counter), "0"(i));
return i + __i;
}
static __inline__ int atomic_sub_return(int i, atomic_t *v)
{
return atomic_add_return(-i, v);
}
#define atomic_inc_return(v) (atomic_add_return(1, v))
#define atomic_dec_return(v) (atomic_sub_return(1, v))
/* These are x86-specific, used by some header files */
#define atomic_clear_mask(mask, addr) \
__asm__ __volatile__("lock; andl %0,%1" \
: \
: "r"(~(mask)), "m"(*addr) \
: "memory")
#define atomic_set_mask(mask, addr) \
__asm__ __volatile__("lock; orl %0,%1" \
: \
: "r"(mask), "m"(*(addr)) \
: "memory")
__END_DECLS
#endif

297
src/search_local/index_storage/common/atomic_asm8.h Normal file
View File

@ -0,0 +1,297 @@
/*
* =====================================================================================
*
* Filename: atomic_asm8.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ARCH_I386_ATOMIC8__
#define __ARCH_I386_ATOMIC8__
#include <sys/cdefs.h>
__BEGIN_DECLS
/*
* cmpxchg8b: require Pentium or above
* compare edx:eax with target memory,
* if equal, write ecx:ebx to memory
* if non-equal, keep old value in edx:eax and write back
*/
/*
* Make sure gcc doesn't try to be clever and move things around
* on us. We need to use _exactly_ the address the user gave us,
* not some alias that contains the same information.
*/
typedef struct
{
volatile long long counter;
} atomic8_t;
#define ATOMIC8_INIT(i) \
{ \
(i) \
}
/**
* atomic8_read - read atomic variable
* @v: pointer of type atomic8_t
*
* Atomically reads the value of @v.
*/
static __inline__ long long atomic8_read(atomic8_t *v)
{
register long long r;
__asm__ __volatile__(
/* original value for ebx, ecx doesn't matter */
" movl %%ecx, %%edx\n"
" movl %%ebx, %%eax\n"
"lock; cmpxchg8b %1\n"
: "=&A"(r), "+o"(v->counter)
:
: "memory", "cc");
return r;
}
/**
* atomic8_set - set atomic variable
* @v: pointer of type atomic8_t
* @i: required value
*
* Atomically sets the value of @v to @i.
*/
static __inline__ long long atomic8_return_set(atomic8_t *v, long long i)
{
register long long r = v->counter;
#if __pic__ || __PIC__
__asm__ __volatile__(
" pushl %%ebx\n"
" movl %2, %%ebx\n"
"1: repz; nop\n" // aka pause
"lock; cmpxchg8b (%1)\n"
" jnz 1b\n"
" popl %%ebx\n"
: "+A"(r)
: "ir"(v), "ir"((int)i), "c"((int)((unsigned long long)i >> 32))
: "memory", "cc");
#else
__asm__ __volatile__(
"1: repz; nop\n" // aka pause
"lock; cmpxchg8b %1\n"
" jnz 1b\n"
: "+A"(r), "+o"(v->counter)
: "b"((int)i), "c"((int)((unsigned long long)i >> 32))
: "memory", "cc");
#endif
return r;
}
static __inline__ void atomic8_set(atomic8_t *v, long long i)
{
atomic8_return_set(v, i);
}
/**
* atomic8_set - set atomic variable
* @v: pointer of type atomic8_t
* @i: required value
*
* Atomically sets the value of @v to @i.
*/
static __inline__ long long atomic8_clear(atomic8_t *v)
{
return atomic8_return_set(v, 0);
}
/**
* atomic8_add - add integer to atomic variable
* @i: integer value to add
* @v: pointer of type atomic8_t
*
* Atomically adds @i to @v.
*/
static __inline__ long long atomic8_add(long long i, atomic8_t *v)
{
register long long r = v->counter;
#if __pic__ || __PIC__
__asm__ __volatile__(
" pushl %%ebx\n"
"1: movl %2, %%ebx\n"
" movl %3, %%ecx\n"
" addl %%eax, %%ebx\n"
" adcl %%edx, %%ecx\n"
"repz; nop\n"
"lock; cmpxchg8b (%1)\n"
" jnz 1b\n"
" popl %%ebx\n"
: "+A"(r)
: "ir"(v), "ir"((int)i), "ir"((int)((unsigned long long)i >> 32))
: "memory", "ecx", "cc");
#else
__asm__ __volatile__(
"1: movl %2, %%ebx\n"
" movl %3, %%ecx\n"
" addl %%eax, %%ebx\n"
" adcl %%edx, %%ecx\n"
"repz; nop\n"
"lock; cmpxchg8b %1\n"
" jnz 1b\n"
: "+A"(r), "+o"(v->counter)
: "g"((int)i), "g"((int)((unsigned long long)i >> 32))
: "memory", "ebx", "ecx", "cc");
#endif
return r;
}
/**
* atomic8_add_return - add and return
* @v: pointer of type atomic8_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns @i + @v
*/
static __inline__ long long atomic8_add_return(long long i, atomic8_t *v)
{
register long long r = v->counter;
#if __pic__ || __PIC__
__asm__ __volatile__(
" pushl %%ebx\n"
"1: movl %2, %%ebx\n"
" movl %3, %%ecx\n"
" addl %%eax, %%ebx\n"
" adcl %%edx, %%ecx\n"
"repz; nop\n"
"lock; cmpxchg8b (%1)\n"
" jnz 1b\n"
" movl %%ebx, %%eax\n"
" movl %%ecx, %%edx\n"
" popl %%ebx\n"
: "+A"(r)
: "ir"(v), "ir"((int)i), "ir"((int)((unsigned long long)i >> 32))
: "memory", "ecx", "cc");
#else
__asm__ __volatile__(
"1: movl %2, %%ebx\n"
" movl %3, %%ecx\n"
" addl %%eax, %%ebx\n"
" adcl %%edx, %%ecx\n"
"repz; nop\n"
"lock; cmpxchg8b %1\n"
" jnz 1b\n"
" movl %%ebx, %%eax\n"
" movl %%ecx, %%edx\n"
: "+A"(r), "+o"(v->counter)
: "g"((int)i), "g"((int)((unsigned long long)i >> 32))
: "memory", "ebx", "ecx", "cc");
#endif
return r;
}
/**
* atomic8_sub - subtract the atomic variable
* @i: integer value to subtract
* @v: pointer of type atomic8_t
*
* Atomically subtracts @i from @v.
*/
static __inline__ long long atomic8_sub(long long i, atomic8_t *v)
{
return atomic8_add(-i, v);
}
/**
* atomic8_sub_and_test - subtract value from variable and test result
* @i: integer value to subtract
* @v: pointer of type atomic8_t
*
* Atomically subtracts @i from @v and returns
* true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic8_sub_and_test(long long i, atomic8_t *v)
{
return atomic8_add_return(-i, v) == 0;
}
/**
* atomic8_inc - increment atomic variable
* @v: pointer of type atomic8_t
*
* Atomically increments @v by 1.
*/
static __inline__ long long atomic8_inc(atomic8_t *v)
{
return atomic8_add(1, v);
}
/**
* atomic8_dec - decrement atomic variable
* @v: pointer of type atomic8_t
*
* Atomically decrements @v by 1.
*/
static __inline__ long long atomic8_dec(atomic8_t *v)
{
return atomic8_add(-1, v);
}
/**
* atomic8_dec_and_test - decrement and test
* @v: pointer of type atomic8_t
*
* Atomically decrements @v by 1 and
* returns true if the result is 0, or false for all other
* cases.
*/
static __inline__ int atomic8_dec_and_test(atomic8_t *v)
{
return atomic8_add_return(-1, v) == 0;
}
/**
* atomic8_inc_and_test - increment and test
* @v: pointer of type atomic8_t
*
* Atomically increments @v by 1
* and returns true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic8_inc_and_test(atomic8_t *v)
{
return atomic8_add_return(1, v) == 0;
}
/**
* atomic8_add_negative - add and test if negative
* @v: pointer of type atomic8_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns true
* if the result is negative, or false when
* result is greater than or equal to zero.
*/
static __inline__ int atomic8_add_negative(long long i, atomic8_t *v)
{
return atomic8_add_return(i, v) < 0;
}
static __inline__ long long atomic8_sub_return(long long i, atomic8_t *v)
{
return atomic8_add_return(-i, v);
}
#define atomic8_inc_return(v) (atomic8_add_return(1, v))
#define atomic8_dec_return(v) (atomic8_sub_return(1, v))
__END_DECLS
#endif

179
src/search_local/index_storage/common/atomic_gcc.h Normal file
View File

@ -0,0 +1,179 @@
/*
* =====================================================================================
*
* Filename: atomic_gcc.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ARCH_I386_ATOMIC__
#define __ARCH_I386_ATOMIC__
#include <sys/cdefs.h>
__BEGIN_DECLS
/*
* Make sure gcc doesn't try to be clever and move things around
* on us. We need to use _exactly_ the address the user gave us,
* not some alias that contains the same information.
*/
typedef volatile int atomic_t;
#define ATOMIC_INIT(i) \
{ \
(i) \
}
/**
* atomic_read - read atomic variable
* @v: pointer of type atomic_t
*
* Atomically reads the value of @v.
*/
#define atomic_read(v) (*(v))
/**
* atomic_set - set atomic variable
* @v: pointer of type atomic_t
* @i: required value
*
* Atomically sets the value of @v to @i.
*/
#define atomic_set(v, i) (*(v) = (i))
static __inline__ int atomic_clear(atomic_t *v)
{
return __sync_fetch_and_and(v, 0);
}
/**
* atomic_add - add integer to atomic variable
* @i: integer value to add
* @v: pointer of type atomic_t
*
* Atomically adds @i to @v.
*/
static __inline__ int atomic_add(int i, atomic_t *v)
{
return __sync_fetch_and_add(v, i);
}
/**
* atomic_sub - subtract the atomic variable
* @i: integer value to subtract
* @v: pointer of type atomic_t
*
* Atomically subtracts @i from @v.
*/
static __inline__ int atomic_sub(int i, atomic_t *v)
{
return __sync_fetch_and_sub(v, i);
}
/**
* atomic_sub_and_test - subtract value from variable and test result
* @i: integer value to subtract
* @v: pointer of type atomic_t
*
* Atomically subtracts @i from @v and returns
* true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
{
return __sync_sub_and_fetch(v, i) == 0;
}
/**
* atomic_inc - increment atomic variable
* @v: pointer of type atomic_t
*
* Atomically increments @v by 1.
*/
static __inline__ int atomic_inc(atomic_t *v)
{
return __sync_fetch_and_add(v, 1);
}
/**
* atomic_dec - decrement atomic variable
* @v: pointer of type atomic_t
*
* Atomically decrements @v by 1.
*/
static __inline__ int atomic_dec(atomic_t *v)
{
return __sync_fetch_and_sub(v, 1);
}
/**
* atomic_dec_and_test - decrement and test
* @v: pointer of type atomic_t
*
* Atomically decrements @v by 1 and
* returns true if the result is 0, or false for all other
* cases.
*/
static __inline__ int atomic_dec_and_test(atomic_t *v)
{
return __sync_sub_and_fetch(v, 1) == 0;
}
/**
* atomic_inc_and_test - increment and test
* @v: pointer of type atomic_t
*
* Atomically increments @v by 1
* and returns true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic_inc_and_test(atomic_t *v)
{
return __sync_add_and_fetch(v, 1) == 0;
}
/**
* atomic_add_negative - add and test if negative
* @v: pointer of type atomic_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns true
* if the result is negative, or false when
* result is greater than or equal to zero.
*/
static __inline__ int atomic_add_negative(int i, atomic_t *v)
{
return __sync_add_and_fetch(v, i) < 0;
}
/**
* atomic_add_return - add and return
* @v: pointer of type atomic_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns @i + @v
*/
static __inline__ int atomic_add_return(int i, atomic_t *v)
{
return __sync_add_and_fetch(v, i);
}
static __inline__ int atomic_sub_return(int i, atomic_t *v)
{
return __sync_sub_and_fetch(v, i);
}
#define atomic_inc_return(v) (atomic_add_return(1, v))
#define atomic_dec_return(v) (atomic_sub_return(1, v))
__END_DECLS
#endif

186
src/search_local/index_storage/common/atomic_gcc8.h Normal file
View File

@ -0,0 +1,186 @@
/*
* =====================================================================================
*
* Filename: atomic_gcc8.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __ARCH_I386_ATOMIC8__
#define __ARCH_I386_ATOMIC8__
#include <sys/cdefs.h>
__BEGIN_DECLS
/*
* Make sure gcc doesn't try to be clever and move things around
* on us. We need to use _exactly_ the address the user gave us,
* not some alias that contains the same information.
*/
typedef volatile int64_t atomic8_t;
#define ATOMIC_INIT(i) \
{ \
(i) \
}
/**
* atomic8_read - read atomic8 variable
* @v: pointer of type atomic8_t
*
* Atomically reads the value of @v.
*/
#if __WORDSIZE == 64
#define atomic8_read(v) (*(v))
#else
#define atomic8_read(v) atomic8_add_return(0, v)
#endif
/**
* atomic8_set - set atomic8 variable
* @v: pointer of type atomic8_t
* @i: required value
*
* Atomically sets the value of @v to @i.
*/
#if __WORDSIZE == 64
#define atomic8_set(v, i) (*(v) = (i))
#else
#define atomic8_set(v, i) (*(v) = (i))
#endif
static __inline__ int64_t atomic8_clear(atomic8_t *v)
{
return __sync_fetch_and_and(v, 0);
}
/**
* atomic8_add - add integer to atomic8 variable
* @i: integer value to add
* @v: pointer of type atomic8_t
*
* Atomically adds @i to @v.
*/
static __inline__ int64_t atomic8_add(int64_t i, atomic8_t *v)
{
return __sync_fetch_and_add(v, i);
}
/**
* atomic8_sub - subtract the atomic8 variable
* @i: integer value to subtract
* @v: pointer of type atomic8_t
*
* Atomically subtracts @i from @v.
*/
static __inline__ int64_t atomic8_sub(int64_t i, atomic8_t *v)
{
return __sync_fetch_and_sub(v, i);
}
/**
* atomic8_sub_and_test - subtract value from variable and test result
* @i: integer value to subtract
* @v: pointer of type atomic8_t
*
* Atomically subtracts @i from @v and returns
* true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic8_sub_and_test(int64_t i, atomic8_t *v)
{
return __sync_sub_and_fetch(v, i) == 0;
}
/**
* atomic8_inc - increment atomic8 variable
* @v: pointer of type atomic8_t
*
* Atomically increments @v by 1.
*/
static __inline__ int64_t atomic8_inc(atomic8_t *v)
{
return __sync_fetch_and_add(v, 1);
}
/**
* atomic8_dec - decrement atomic8 variable
* @v: pointer of type atomic8_t
*
* Atomically decrements @v by 1.
*/
static __inline__ int64_t atomic8_dec(atomic8_t *v)
{
return __sync_fetch_and_sub(v, 1);
}
/**
* atomic8_dec_and_test - decrement and test
* @v: pointer of type atomic8_t
*
* Atomically decrements @v by 1 and
* returns true if the result is 0, or false for all other
* cases.
*/
static __inline__ int atomic8_dec_and_test(atomic8_t *v)
{
return __sync_sub_and_fetch(v, 1) == 0;
}
/**
* atomic8_inc_and_test - increment and test
* @v: pointer of type atomic8_t
*
* Atomically increments @v by 1
* and returns true if the result is zero, or false for all
* other cases.
*/
static __inline__ int atomic8_inc_and_test(atomic8_t *v)
{
return __sync_add_and_fetch(v, 1) == 0;
}
/**
* atomic8_add_negative - add and test if negative
* @v: pointer of type atomic8_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns true
* if the result is negative, or false when
* result is greater than or equal to zero.
*/
static __inline__ int atomic8_add_negative(int64_t i, atomic8_t *v)
{
return __sync_add_and_fetch(v, i) < 0;
}
/**
* atomic8_add_return - add and return
* @v: pointer of type atomic8_t
* @i: integer value to add
*
* Atomically adds @i to @v and returns @i + @v
*/
static __inline__ int64_t atomic8_add_return(int64_t i, atomic8_t *v)
{
return __sync_add_and_fetch(v, i);
}
static __inline__ int64_t atomic8_sub_return(int64_t i, atomic8_t *v)
{
return __sync_sub_and_fetch(v, i);
}
#define atomic8_inc_return(v) (atomic8_add_return(1, v))
#define atomic8_dec_return(v) (atomic8_sub_return(1, v))
__END_DECLS
#endif

47
src/search_local/index_storage/common/bitsop.c Normal file
View File

@ -0,0 +1,47 @@
/*
* =====================================================================================
*
* Filename: bitsop.cc
*
* Description: bits op functions
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author:
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "bitsop.h"
const unsigned char __bitcount[256] = {
/* 00 */ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
/* 10 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
/* 20 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
/* 30 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* 40 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
/* 50 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* 60 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* 70 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
/* 80 */ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
/* 90 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* a0 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* b0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
/* c0 */ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
/* d0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
/* e0 */ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
/* f0 */ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};
int CountBits(const char *buf, int sz)
{
int c;
for (c = 0; sz > 0; buf++, sz--)
{
c += __bitcount[*(unsigned char *)buf];
}
return c;
}

101
src/search_local/index_storage/common/bitsop.h Normal file
View File

@ -0,0 +1,101 @@
/*
* =====================================================================================
*
* Filename: bitsop.h
*
* Description: bit operation function. setting location for any platform.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __BITSOP_H__
#define __BITSOP_H__
#include <linux/types.h>
#include <sys/cdefs.h>
__BEGIN_DECLS
/*
bits操作函数
*/
/*
*
* select.h中提供的FD_*宏在32位机器上是按照byte来置位的（汇编实现），但在64位
* 机器上是按照8 bytes来置位的，所以当碰到mmap文件末尾时，有可能segment fault。
*/
#define CHAR_BITS 8
extern const unsigned char __bitcount[];
extern int CountBits(const char *buf, int sz);
//bits op interface
#define SET_B(bit, addr) __set_b(bit, addr)
#define CLR_B(bit, addr) __clr_b(bit, addr)
#define ISSET_B(bit, addr) __isset_b(bit, addr)
#define COPY_B(dest_bit, dest_addr, src_bit, src_addr, count) __bit_copy(dest_bit, dest_addr, src_bit, src_addr, count)
#define COUNT_B(buf, size) CountBits(buf, size)
static inline void __set_b(__u32 bit, const volatile void *addr)
{
volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS);
__u8 shift = bit % CHAR_BITS;
*begin |= ((__u8)0x1 << shift);
return;
}
static inline int __isset_b(__u32 bit, const volatile void *addr)
{
volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS);
__u8 shift = bit % CHAR_BITS;
return (*begin & ((__u8)0x1 << shift)) > 0 ? 1 : 0;
}
static inline void __clr_b(__u32 bit, const volatile void *addr)
{
volatile __u8 *begin = (volatile __u8 *)addr + (bit / CHAR_BITS);
__u8 shift = bit % CHAR_BITS;
*begin &= ~((__u8)0x1 << shift);
}
static inline __u8 __readbyte(const volatile void *addr)
{
return *(volatile __u8 *)addr;
}
static inline void __writebyte(__u8 val, volatile void *addr)
{
*(volatile __u8 *)addr = val;
}
static inline void __bit_copy(__u32 dest_bit, volatile void *dest,
__u32 src_bit, volatile void *src,
__u32 count)
{
__u32 i;
for (i = 0; i < count; i++)
{
if (__isset_b(src_bit, src))
__set_b(dest_bit, dest);
else
__clr_b(dest_bit, dest);
dest_bit++;
src_bit++;
}
return;
}
__END_DECLS
#endif

193
src/search_local/index_storage/common/blacklist.cc Normal file
View File

@ -0,0 +1,193 @@
/*
* =====================================================================================
*
* Filename: blacklist.cc
*
* Description: blacklist in order to block super large node into cache.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <time.h>
#include "log.h"
#include "blacklist.h"
#include "keycodec.h"
DTC_USING_NAMESPACE
BlackList::BlackList() : current_blslot_count(0),
max_blslot_count(0),
blslot_expired_time(0)
{
}
BlackList::~BlackList()
{
}
int BlackList::init_blacklist(const unsigned max, const unsigned keytype, const unsigned expired)
{
for (int i = 0; i < MAX_HASH_DEPTH; ++i)
{
INIT_HLIST_HEAD(hash_list + i);
}
INIT_LIST_HEAD(&time_list);
key_type = keytype;
current_blslot_count = 0;
max_blslot_count = max < 1000000 ? max : 1000000; /* max:1000000 slots */
blslot_expired_time = expired;
return 0;
}
int BlackList::add_blacklist(const char *ptr, const unsigned vsize)
{
if (!ptr)
return 0;
KeyCodec key_codec(key_type);
struct hlist_head *h = hash_list + key_codec.key_hash(ptr) % MAX_HASH_DEPTH;
struct hlist_node *pos = 0;
struct blslot *tpos = 0;
hlist_for_each_entry(tpos, pos, h, hash)
{
/* hit */
if (0 == key_codec.key_compare(ptr, tpos->key))
{
tpos->vsize = vsize;
tpos->expired = time(NULL) + blslot_expired_time;
/* adjust time list */
_list_move_tail(&tpos->time, &time_list);
return 0;
}
}
if (current_blslot_count >= max_blslot_count)
/* overflow */
return -1;
tpos = (struct blslot *)malloc(offsetof(struct blslot, key) + key_codec.total_length(ptr));
if (NULL == tpos)
{
log_notice("allocate blacklist slot failed");
return -1;
}
tpos->vsize = vsize;
tpos->expired = time(NULL) + blslot_expired_time;
memcpy(tpos->key, ptr, key_codec.total_length(ptr));
list_add_tail(&tpos->time, &time_list);
hlist_add_head(&tpos->hash, h);
stat_everything(tpos, 1);
return 0;
}
int BlackList::in_blacklist(const char *ptr)
{
if (!ptr)
return 0;
KeyCodec key_codec(key_type);
struct hlist_head *h = hash_list + key_codec.key_hash(ptr) % MAX_HASH_DEPTH;
struct hlist_node *pos = 0;
struct blslot *tpos = 0;
hlist_for_each_entry(tpos, pos, h, hash)
{
/* found */
if (0 == key_codec.key_compare(ptr, tpos->key))
return 1;
}
return 0;
}
int BlackList::try_expired_blacklist(void)
{
unsigned now = time(NULL);
struct blslot *pos = 0;
/* time->next is the oldest slot */
while (!list_empty(&time_list))
{
pos = list_entry(time_list.next, struct blslot, time);
if (pos->expired > now)
break;
list_del(&pos->time);
hlist_del(&pos->hash);
stat_everything(pos, 0);
free(pos);
}
return 0;
}
/* TODO: 统计top10 */
void BlackList::stat_everything(const struct blslot *slot, const int add)
{
/* add */
if (add)
++current_blslot_count;
/* delete */
else
--current_blslot_count;
return;
}
void BlackList::dump_all_blslot(void)
{
KeyCodec key_codec(key_type);
struct blslot *pos = 0;
list_for_each_entry(pos, &time_list, time)
{
switch (key_type)
{
case 1:
log_debug("key: %u size: %u expired: %u hash: %u",
*(uint8_t *)key_codec.key_pointer(pos->key),
pos->vsize,
pos->expired,
key_codec.key_hash(pos->key) % MAX_HASH_DEPTH);
break;
case 2:
log_debug("key: %u size: %u expired: %u hash: %u",
*(uint16_t *)key_codec.key_pointer(pos->key),
pos->vsize,
pos->expired,
key_codec.key_hash(pos->key) % MAX_HASH_DEPTH);
break;
case 4:
log_debug("key: %u size: %u expired: %u hash: %u",
*(uint32_t *)key_codec.key_pointer(pos->key),
pos->vsize,
pos->expired,
key_codec.key_hash(pos->key) % MAX_HASH_DEPTH);
break;
default:
log_debug("key: %10s size: %u expired: %u hash: %u",
key_codec.key_pointer(pos->key),
pos->vsize,
pos->expired,
key_codec.key_hash(pos->key) % MAX_HASH_DEPTH);
break;
}
}
return;
}

73
src/search_local/index_storage/common/blacklist.h Normal file
View File

@ -0,0 +1,73 @@
/*
* =====================================================================================
*
* Filename: blacklist.h
*
* Description: blacklist in order to block super large node into cache.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __DTC_BLACKLIST_H
#define __DTC_BLACKLIST_H
#include "list.h"
#include "hlist.h"
#include <stdint.h>
#include "namespace.h"
#define MAX_HASH_DEPTH 65535
DTC_BEGIN_NAMESPACE
struct blslot
{
struct hlist_node hash; /* link to hash_list */
struct list_head time; /* link to time_list */
unsigned int vsize;
unsigned int expired;
char key[0];
} __attribute__((__aligned__(1)));
class BlackList
{
public:
BlackList();
~BlackList();
int init_blacklist(const unsigned max, const unsigned type, const unsigned expired);
int add_blacklist(const char *packedkey, const unsigned vsize);
int in_blacklist(const char *packedkey);
/* dump all blslot in blacklist, debug only */
void dump_all_blslot(void);
protected:
/* try expire all expired slot */
int try_expired_blacklist(void);
unsigned current_blslot_count;
private:
/* double linked hash list with single pointer list head */
struct hlist_head hash_list[MAX_HASH_DEPTH];
/* time list */
struct list_head time_list;
unsigned max_blslot_count;
unsigned blslot_expired_time;
unsigned key_type;
void stat_everything(const struct blslot *, const int add);
};
DTC_END_NAMESPACE
#endif

43
src/search_local/index_storage/common/blacklist_unit.cc Normal file
View File

@ -0,0 +1,43 @@
/*
* =====================================================================================
*
* Filename: blacklist_unit.cc
*
* Description: balcklist attach excute unit, arun expired task and output stastic result at regular intervals.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <blacklist_unit.h>
DTC_USING_NAMESPACE
BlackListUnit::BlackListUnit(TimerList *t) : timer(t)
{
}
BlackListUnit::~BlackListUnit()
{
}
void BlackListUnit::timer_notify(void)
{
log_debug("sched blacklist-expired task");
/* expire all expired eslot */
try_expired_blacklist();
BlackList::dump_all_blslot();
/* set timer agagin */
attach_timer(timer);
return;
}

93
src/search_local/index_storage/common/blacklist_unit.h Normal file
View File

@ -0,0 +1,93 @@
/*
* =====================================================================================
*
* Filename: blacklist_unit.h
*
* Description: balcklist attach excute unit, arun expired task and output stastic result at regular intervals.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __DTC_BLACKLIST_UNIT_H
#define __DTC_BLACKLIST_UNIT_H
#include "namespace.h"
#include "timer_list.h"
#include "blacklist.h"
#include "log.h"
#include "stat_dtc.h"
DTC_BEGIN_NAMESPACE
class TimerObject;
class BlackListUnit : public BlackList, private TimerObject
{
public:
BlackListUnit(TimerList *timer);
virtual ~BlackListUnit(void);
int init_blacklist(const unsigned max, const unsigned type, const unsigned expired = 10 * 60 /*10 mins*/)
{
/* init statisitc item */
stat_blacksize = statmgr.get_sample(BLACKLIST_SIZE);
stat_blslot_count = statmgr.get_item_u32(BLACKLIST_CURRENT_SLOT);
return BlackList::init_blacklist(max, type, expired);
}
int add_blacklist(const char *key, const unsigned vsize)
{
int ret = BlackList::add_blacklist(key, vsize);
if (0 == ret)
{
/* statistic */
stat_blacksize.push(vsize);
stat_blslot_count = current_blslot_count;
}
return ret;
}
int try_expired_blacklist(void)
{
int ret = BlackList::try_expired_blacklist();
if (0 == ret)
{
/* statistic */
stat_blslot_count = current_blslot_count;
}
return ret;
}
void start_blacklist_expired_task(void)
{
log_info("start blacklist-expired task");
attach_timer(timer);
return;
}
public:
virtual void timer_notify(void);
private:
TimerList *timer;
/* for statistic */
StatSample stat_blacksize; /* black size distribution */
StatItemU32 stat_blslot_count;
};
DTC_END_NAMESPACE
#endif

86
src/search_local/index_storage/common/buffer.cc Normal file
View File

@ -0,0 +1,86 @@
/*
* =====================================================================================
*
* Filename: buffer.cc
*
* Description: buffer block relevant operation.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdarg.h>
#include <stdio.h>
#include "buffer.h"
void CreateBuff(int inlen, int &len, char **buff)
{
if (*buff == NULL)
{
*buff = (char *)CALLOC(inlen, sizeof(char));
if (*buff == NULL)
len = 0;
else
len = inlen;
}
else
{
if (len < inlen)
{
FREE_IF(*buff);
*buff = (char *)CALLOC(inlen, sizeof(char));
if (*buff == NULL)
len = 0;
else
len = inlen;
}
else
{
memset(*buff, 0x0, len);
}
}
}
int buffer::bprintf(const char *format, ...)
{
va_list ap;
long len;
va_start(ap, format);
len = bprintf(format, ap);
va_end(ap);
return len;
}
int buffer::vbprintf(const char *format, va_list ap)
{
long len;
va_list ap2;
#ifdef __va_copy
__va_copy(ap2, ap);
#else
va_copy(ap2, ap);
#endif
len = vsnprintf(cursor(), remain(), format, ap);
if (len < 0)
return len;
if ((unsigned long)len >= remain())
{
if (expand(len) < 0)
return -1;
len = vsnprintf(cursor(), remain(), format, ap2);
if (len < 0)
return len;
}
dataSize += len;
return len;
}

200
src/search_local/index_storage/common/buffer.h Normal file
View File

@ -0,0 +1,200 @@
/*
* =====================================================================================
*
* Filename: buffer.h
*
* Description: buffer block relevant operation.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __BUFFER_H
#define __BUFFER_H
#include "mem_check.h"
#include <string.h>
#include <stdarg.h>
extern void CreateBuff(int inlen, int &len, char **buff);
class buffer
{
private:
char *buf;
unsigned int bufSize;
unsigned int dataSize;
unsigned int needSize;
public:
buffer(void)
{
buf = NULL;
bufSize = dataSize = needSize = 0;
}
~buffer(void)
{
FREE_IF(buf);
bufSize = dataSize = needSize = 0;
}
inline int expand(int size)
{
needSize = dataSize + size + 1;
if (needSize > bufSize)
{
int sz = needSize + (needSize >> 3);
sz = (sz | 0xf) + 0x10; // align to 16 bytes
if (REALLOC(buf, sz) == NULL)
return -1;
bufSize = sz;
}
return 0;
}
inline void trunc(int n)
{
if (n >= (long long)dataSize)
return;
if (n >= 0)
{
buf[dataSize = n] = 0;
return;
}
n += dataSize;
if (n < 0)
return;
buf[dataSize = n] = 0;
}
inline int resize(int n)
{
if (n > (long long)dataSize && expand(n - dataSize) < 0)
return -1;
buf[dataSize = n] = 0;
return 0;
}
inline void clear(void) { trunc(0); }
inline void release(void)
{
trunc(0);
FREE_CLEAR(buf);
bufSize = 0;
needSize = 0;
}
inline char &operator[](int n) const { return buf[n]; }
inline char at(int n) const
{
if (n > (long long)dataSize)
return 0;
if (n >= 0)
return buf[n];
n += dataSize;
if (n < 0)
return 0;
return buf[n];
}
inline char *c_str(void) { return buf; }
inline const char *c_str(void) const { return buf; }
inline char *cursor(void) { return buf + dataSize; }
inline const char *cursor(void) const { return buf + dataSize; }
inline unsigned int size(void) const { return dataSize; }
inline unsigned int needed(void) const { return needSize; }
inline unsigned int remain(void) const { return bufSize - dataSize; }
inline unsigned int margin(void) const { return needSize < bufSize ? 0 : needSize - bufSize; }
inline int append(const char *data, int len)
{
if (expand(len) < 0)
return -1;
memcpy(buf + dataSize, data, len);
dataSize += len;
buf[dataSize] = 0;
return 0;
}
inline int append(const char *data)
{
return append(data, strlen(data));
}
inline int append(const buffer &data)
{
return append(data.c_str(), data.size());
}
inline int append(const buffer *data)
{
return append(data->c_str(), data->size());
}
#define __TEMPLATE_FUNC__(t) \
inline int append(t v) \
{ \
if (expand(sizeof(t)) < 0) \
return -1; \
*(t *)(buf + dataSize) = v; \
dataSize += sizeof(t); \
buf[dataSize] = 0; \
return 0; \
}
__TEMPLATE_FUNC__(char);
__TEMPLATE_FUNC__(signed char);
__TEMPLATE_FUNC__(unsigned char);
__TEMPLATE_FUNC__(short);
__TEMPLATE_FUNC__(unsigned short);
__TEMPLATE_FUNC__(int);
__TEMPLATE_FUNC__(unsigned int);
__TEMPLATE_FUNC__(long);
__TEMPLATE_FUNC__(unsigned long);
__TEMPLATE_FUNC__(long long);
__TEMPLATE_FUNC__(unsigned long long);
__TEMPLATE_FUNC__(float);
__TEMPLATE_FUNC__(double);
#undef __TEMPLATE_FUNC__
// unsafe
inline buffer &add(const char *data, int len)
{
memcpy(buf + dataSize, data, len);
dataSize += len;
return *this;
}
inline buffer &operator<<(const char *data)
{
return add(data, strlen(data));
}
inline buffer &operator<<(const buffer &data)
{
return add(data.c_str(), data.size());
}
inline buffer &operator<<(const buffer *data)
{
return add(data->c_str(), data->size());
}
#define __TEMPLATE_FUNC__(t) \
inline buffer &operator<<(t v) \
{ \
*(t *)(buf + dataSize) = v; \
dataSize += sizeof(t); \
return *this; \
}
__TEMPLATE_FUNC__(char);
__TEMPLATE_FUNC__(signed char);
__TEMPLATE_FUNC__(unsigned char);
__TEMPLATE_FUNC__(short);
__TEMPLATE_FUNC__(unsigned short);
__TEMPLATE_FUNC__(int);
__TEMPLATE_FUNC__(unsigned int);
__TEMPLATE_FUNC__(long);
__TEMPLATE_FUNC__(unsigned long);
__TEMPLATE_FUNC__(long long);
__TEMPLATE_FUNC__(unsigned long long);
__TEMPLATE_FUNC__(float);
__TEMPLATE_FUNC__(double);
#undef __TEMPLATE_FUNC__
int bprintf(const char *format, ...);
int vbprintf(const char *format, va_list ap);
};
#endif

135
src/search_local/index_storage/common/buffer_error.h Normal file
View File

@ -0,0 +1,135 @@
/*
* =====================================================================================
*
* Filename: buffer_error.h
*
* Description: cache error code list.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CH_CACHE_ERRR_H__
#define __CH_CACHE_ERRR_H__
enum
{
EC_ERROR_BASE = 2000,
EC_BAD_COMMAND, // unsupported command
EC_MISSING_SECTION, // missing mandatory section
EC_EXTRA_SECTION, // incompatible section present
EC_DUPLICATE_TAG, // same tag appear twice
EC_DUPLICATE_FIELD, //5: same field appear twice in .Need()
EC_BAD_SECTION_LENGTH, // section length too short
EC_BAD_VALUE_LENGTH, // value length not allow
EC_BAD_STRING_VALUE, // string value w/o NULL
EC_BAD_FLOAT_VALUE, // invalid float format
EC_BAD_FIELD_NUM, //10: invalid total field#
EC_EXTRA_SECTION_DATA, // section length too large
EC_BAD_VALUE_TYPE, // incompatible value type
EC_BAD_OPERATOR, // incompatible operator/comparison
EC_BAD_FIELD_ID, // invalid field ID
EC_BAD_FIELD_NAME, //15: invalid field name
EC_BAD_FIELD_TYPE, // invalid field type
EC_BAD_FIELD_SIZE, // invalid field size
EC_TABLE_REDEFINED, // table defined twice
EC_TABLE_MISMATCH, // request table != server table
EC_VERSION_MISMATCH, //20: unsupported protocol version
EC_CHECKSUM_MISMATCH, // table hash not equal
EC_NO_MORE_DATA, // End of Result
EC_NEED_FULL_FIELDSET, // only full field set accepted by helper
EC_BAD_KEY_TYPE, // key type incompatible
EC_BAD_KEY_SIZE, // 25: key size incompatible
EC_SERVER_BUSY, //server error
EC_BAD_SOCKET, // network failed
EC_NOT_INITIALIZED, // object didn't initialized
EC_BAD_HOST_STRING,
EC_BAD_TABLE_NAME, // 30
EC_TASK_NEED_DELETE,
EC_KEY_NEEDED,
EC_SERVER_ERROR,
EC_UPSTREAM_ERROR,
EC_KEY_OVERFLOW, // 35
EC_BAD_MULTIKEY,
EC_READONLY_FIELD,
EC_BAD_ASYNC_CMD,
EC_OUT_OF_KEY_RANGE,
EC_REQUEST_ABORTED, // 40
EC_PARALLEL_MODE,
EC_KEY_NOTEXIST,
EC_SERVER_READONLY,
EC_BAD_INVALID_FIELD,
EC_DUPLICATE_KEY, // 45
EC_TOO_MANY_KEY_VALUE,
EC_BAD_KEY_NAME,
EC_BAD_RAW_DATA,
EC_BAD_HOTBACKUP_JID,
EC_FULL_SYNC_COMPLETE, //50
EC_FULL_SYNC_STAGE,
EC_INC_SYNC_STAGE,
EC_ERR_SYNC_STAGE,
EC_NOT_ALLOWED_INSERT,
EC_COMPRESS_ERROR, //55
EC_UNCOMPRESS_ERROR,
EC_TASKPOOL,
EC_STATE_ERROR,
EC_DATA_NEEDED,
EC_BAD_ACCESS_KEY, //60
EC_TASK_TIMEOUT,
EC_BUSINESS_WITHOUT_EXPIRETIME, //62
EC_EMPTY_TBDEF, //63
EC_INVALID_KEY_VALUE, //64
EC_INVALID_EXPIRETIME, //65
EC_GET_EXPIRETIME_END_OF_RESULT, //66
EC_ERR_MIGRATEDB_ILLEGAL,
EC_ERR_MIGRATEDB_DUPLICATE,
EC_ERR_MIGRATEDB_HELPER,
EC_ERR_MIGRATEDB_MIGRATING, // 70
EC_ERR_MIGRATEDB_NOT_START,
EC_ERR_MIGRATEDB_DISTINCT,
EC_ERR_HOTBACK_WRITELRU,
EC_ERR_HOTBACK_WRITEUPDATE,
EC_ERR_COL_EXPANDING, // 75
EC_ERR_COL_EXPAND_DUPLICATE,
EC_ERR_COL_EXPAND_DONE_DUPLICATE,
EC_ERR_COL_EXPAND_DONE_DISTINCT,
EC_ERR_COL_EXPAND_NO_MEM,
EC_ERR_COL_EXPAND_KEY_ERROR, // 80
EC_ERR_COL_NOT_EXPANDING,
EC_ERR_COL_NO_KEY,
EC_GET_EXPIRETIME_RESULT_NULL,
EC_GET_EXPIRETIME_FIELD_EXPIRETIME_NULL,
EC_BAD_FIELD_SIZE_ON_RESOLVE, // 85
EC_EXCEPTION_ON_RESOLVE,
EC_BAD_FIELD_SIZE_ON_CHECKKEY,
EC_EXCEPTION_ON_CHECKKEY,
EC_NOT_HAVE_AVAILABLE_HELPERGROUP,
EC_NOT_HAVE_AVAILABLE_HELPERCLIENT, // 90
EC_RELOAD_CONFIG_FAILED,
};
#endif

107
src/search_local/index_storage/common/chash.cc Normal file
View File

@ -0,0 +1,107 @@
/*
* =====================================================================================
*
* Filename: chash.cc
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "chash.h"
#define mix(a, b, c) \
{ \
a = a - b; \
a = a - c; \
a = a ^ (c >> 13); \
b = b - c; \
b = b - a; \
b = b ^ (a << 8); \
c = c - a; \
c = c - b; \
c = c ^ (b >> 13); \
a = a - b; \
a = a - c; \
a = a ^ (c >> 12); \
b = b - c; \
b = b - a; \
b = b ^ (a << 16); \
c = c - a; \
c = c - b; \
c = c ^ (b >> 5); \
a = a - b; \
a = a - c; \
a = a ^ (c >> 3); \
b = b - c; \
b = b - a; \
b = b ^ (a << 10); \
c = c - a; \
c = c - b; \
c = c ^ (b >> 15); \
}
typedef unsigned int u4;
unsigned int chash(const char *k, int length)
{
unsigned int a, b, c; /* the internal state */
u4 len; /* how many key bytes still need mixing */
/* Set up the internal state */
len = length;
a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */
// 'TMJR'
c = 0x544D4A52; /* variable initialization of internal state */
/*---------------------------------------- handle most of the key */
while (len >= 12)
{
a = a + (k[0] + ((u4)k[1] << 8) + ((u4)k[2] << 16) + ((u4)k[3] << 24));
b = b + (k[4] + ((u4)k[5] << 8) + ((u4)k[6] << 16) + ((u4)k[7] << 24));
c = c + (k[8] + ((u4)k[9] << 8) + ((u4)k[10] << 16) + ((u4)k[11] << 24));
mix(a, b, c);
k = k + 12;
len = len - 12;
}
/*------------------------------------- handle the last 11 bytes */
c = c + length;
switch (len) /* all the case statements fall through */
{
case 11:
c = c + ((u4)k[10] << 24);
case 10:
c = c + ((u4)k[9] << 16);
case 9:
c = c + ((u4)k[8] << 8);
/* the first byte of c is reserved for the length */
case 8:
b = b + ((u4)k[7] << 24);
case 7:
b = b + ((u4)k[6] << 16);
case 6:
b = b + ((u4)k[5] << 8);
case 5:
b = b + k[4];
case 4:
a = a + ((u4)k[3] << 24);
case 3:
a = a + ((u4)k[2] << 16);
case 2:
a = a + ((u4)k[1] << 8);
case 1:
a = a + k[0];
/* case 0: nothing left to add */
}
mix(a, b, c);
/*-------------------------------------------- report the result */
return c;
}

27
src/search_local/index_storage/common/chash.h Normal file
View File

@ -0,0 +1,27 @@
/*
* =====================================================================================
*
* Filename: chash.h
*
* Description:
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef BITMAP_C_HASH_H__
#define BITMAP_C_HASH_H__
#include <stdint.h>
//hash function for consistent hash
//the algorithm is the same as new_hash, but use another initial value
uint32_t chash(const char *data, int len);
#endif

294
src/search_local/index_storage/common/client_async.cc Normal file
View File

@ -0,0 +1,294 @@
/*
* =====================================================================================
*
* Filename: client_async.cc
*
* Description: client async operation.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdio.h>
#include <sys/un.h>
#include <sys/socket.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include "client_async.h"
#include "client_unit.h"
#include "poll_thread.h"
#include "log.h"
static int statEnable = 0;
static StatItemU32 statAcceptCount;
static StatItemU32 statCurConnCount;
inline AsyncInfo::AsyncInfo(ClientAsync *c, TaskRequest *r)
: cli(c),
req(r),
pkt(NULL)
{
c->numReq++;
/* move to waiting list */
ListAddTail(&c->waitList);
}
inline AsyncInfo::~AsyncInfo()
{
if (req)
req->clear_owner_info();
DELETE(pkt);
if (cli)
cli->numReq--;
}
class CReplyAsync : public ReplyDispatcher<TaskRequest>
{
public:
CReplyAsync(void) {}
virtual ~CReplyAsync(void);
virtual void reply_notify(TaskRequest *task);
};
CReplyAsync::~CReplyAsync(void) {}
void CReplyAsync::reply_notify(TaskRequest *task)
{
AsyncInfo *info = task->OwnerInfo<AsyncInfo>();
if (info == NULL)
{
delete task;
}
else if (info->cli == NULL)
{
log_error("info->cli is NULL, possible memory corrupted");
delete task;
}
else
{
info->cli->queue_result(info);
}
}
static CReplyAsync replyAsync;
ClientAsync::ClientAsync(DTCDecoderUnit *o, int fd, int m, void *peer, int ps)
: PollerObject(o->owner_thread(), fd),
owner(o),
receiver(fd),
curReq(NULL),
curRes(NULL),
maxReq(m),
numReq(0)
{
addrLen = ps;
addr = MALLOC(ps);
memcpy((char *)addr, (char *)peer, ps);
if (!statEnable)
{
statAcceptCount = statmgr.get_item_u32(ACCEPT_COUNT);
statCurConnCount = statmgr.get_item_u32(CONN_COUNT);
statEnable = 1;
}
++statAcceptCount;
++statCurConnCount;
}
ClientAsync::~ClientAsync()
{
while (!doneList.ListEmpty())
{
AsyncInfo *info = doneList.NextOwner();
delete info;
}
while (!waitList.ListEmpty())
{
AsyncInfo *info = waitList.NextOwner();
delete info;
}
--statCurConnCount;
FREE_IF(addr);
}
int ClientAsync::Attach()
{
enable_input();
if (attach_poller() == -1)
return -1;
return 0;
}
//server peer
int ClientAsync::recv_request()
{
if (curReq == NULL)
{
curReq = new TaskRequest(owner->owner_table());
if (NULL == curReq)
{
log_error("%s", "create TaskRequest object failed, msg[no enough memory]");
return -1;
}
curReq->set_hotbackup_table(owner->admin_table());
receiver.erase();
}
int ret = curReq->Decode(receiver);
switch (ret)
{
default:
case DecodeFatalError:
if (errno != 0)
log_notice("decode fatal error, ret = %d msg = %m", ret);
DELETE(curReq);
return -1;
case DecodeDataError:
curReq->response_timer_start();
curReq->mark_as_hit();
if (curReq->result_code() < 0)
log_notice("DecodeDataError, role=%d, fd=%d, result=%d",
curReq->Role(), netfd, curReq->result_code());
return queue_error();
case DecodeDone:
if (curReq->prepare_process() < 0)
return queue_error();
curReq->set_owner_info(new AsyncInfo(this, curReq), 0, (struct sockaddr *)addr);
curReq->push_reply_dispatcher(&replyAsync);
owner->task_dispatcher(curReq);
curReq = NULL;
}
return 0;
}
int ClientAsync::queue_error(void)
{
AsyncInfo *info = new AsyncInfo(this, curReq);
curReq = NULL;
return queue_result(info);
}
int ClientAsync::queue_result(AsyncInfo *info)
{
if (info->req == NULL)
{
delete info;
return 0;
}
TaskRequest *const cur = info->req;
owner->record_request_time(curReq);
/* move to sending list */
info->list_move_tail(&doneList);
/* convert request to result */
info->pkt = new Packet;
if (info->pkt == NULL)
{
delete info->req;
info->req = NULL;
delete info;
log_error("create Packet object failed");
return 0;
}
cur->versionInfo.set_keep_alive_timeout(owner->idle_time());
info->pkt->encode_result(info->req);
DELETE(info->req);
Response();
adjust_events();
return 0;
}
int ClientAsync::response_one(void)
{
if (curRes == NULL)
return 0;
int ret = curRes->Send(netfd);
switch (ret)
{
case SendResultMoreData:
return 0;
case SendResultDone:
DELETE(curRes);
numReq--;
return 0;
default:
log_notice("send failed, return = %d, error = %m", ret);
return -1;
}
return 0;
}
int ClientAsync::Response(void)
{
do
{
int ret;
if (curRes == NULL)
{
// All result sent
if (doneList.ListEmpty())
break;
AsyncInfo *info = doneList.NextOwner();
curRes = info->pkt;
info->pkt = NULL;
delete info;
numReq++;
if (curRes == NULL)
continue;
}
ret = response_one();
if (ret < 0)
return ret;
} while (curRes == NULL);
return 0;
}
void ClientAsync::input_notify(void)
{
if (recv_request() < 0)
delete this;
else
adjust_events();
}
void ClientAsync::output_notify(void)
{
if (Response() < 0)
delete this;
else
adjust_events();
}
int ClientAsync::adjust_events(void)
{
if (curRes == NULL && doneList.ListEmpty())
disable_output();
else
enable_output();
if (numReq >= maxReq)
DisableInput();
else
enable_input();
return apply_events();
}

83
src/search_local/index_storage/common/client_async.h Normal file
View File

@ -0,0 +1,83 @@
/*
* =====================================================================================
*
* Filename: client_async.h
*
* Description: client async operation.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CLIENT__ASYNC_H__
#define __CLIENT__ASYNC_H__
#include "poller.h"
#include "task_request.h"
#include "packet.h"
#include "timer_list.h"
#include "list.h"
class DTCDecoderUnit;
class ClientAsync;
class AsyncInfo : private ListObject<AsyncInfo>
{
public:
AsyncInfo(ClientAsync *c, TaskRequest *r);
~AsyncInfo();
void list_move_tail(ListObject<AsyncInfo> *a)
{
ListObject<AsyncInfo>::list_move_tail(a);
}
ClientAsync *cli;
TaskRequest *req;
Packet *pkt;
};
class ClientAsync : public PollerObject
{
public:
friend class AsyncInfo;
DTCDecoderUnit *owner;
ClientAsync(DTCDecoderUnit *, int fd, int depth, void *peer, int ps);
virtual ~ClientAsync();
virtual int Attach(void);
int queue_result(AsyncInfo *);
int queue_error(void);
int flush_result(void);
int adjust_events(void);
virtual void input_notify(void);
private:
virtual void output_notify(void);
int recv_request(void);
int Response(void);
int response_one(void);
protected:
SimpleReceiver receiver;
TaskRequest *curReq; // decoding
Packet *curRes; // sending
ListObject<AsyncInfo> waitList;
ListObject<AsyncInfo> doneList;
void *addr;
int addrLen;
int maxReq;
int numReq;
};
#endif

299
src/search_local/index_storage/common/client_dgram.cc Normal file
View File

@ -0,0 +1,299 @@
/*
* =====================================================================================
*
* Filename: client_dgram.cc
*
* Description: client dgram.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdio.h>
#include <sys/un.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <linux/sockios.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include "client_dgram.h"
#include "task_request.h"
#include "client_unit.h"
#include "protocol.h"
#include "log.h"
class PollThread;
static int GetSocketFamily(int fd)
{
struct sockaddr addr;
bzero(&addr, sizeof(addr));
socklen_t alen = sizeof(addr);
getsockname(fd, &addr, &alen);
return addr.sa_family;
}
class ReplyDgram : public ReplyDispatcher<TaskRequest>
{
public:
ReplyDgram(void) {}
virtual ~ReplyDgram(void);
virtual void reply_notify(TaskRequest *task);
};
ReplyDgram::~ReplyDgram(void) {}
void ReplyDgram::reply_notify(TaskRequest *task)
{
DgramInfo *info = task->OwnerInfo<DgramInfo>();
if (info == NULL)
{
delete task;
}
else if (info->cli == NULL)
{
log_error("info->cli is NULL, possible memory corrupted");
FREE(info);
delete task;
}
else
{
info->cli->send_result(task, info->addr, info->len);
FREE(info);
}
}
static ReplyDgram replyDgram;
ClientDgram::ClientDgram(DTCDecoderUnit *o, int fd)
: PollerObject(o->owner_thread(), fd),
owner(o),
hastrunc(0),
mru(0),
mtu(0),
alen(0),
abuf(NULL)
{
}
ClientDgram::~ClientDgram()
{
FREE_IF(abuf);
}
int ClientDgram::init_socket_info(void)
{
switch (GetSocketFamily(netfd))
{
default:
mru = 65508;
mtu = 65507;
alen = sizeof(struct sockaddr);
break;
case AF_UNIX:
mru = 16 << 20;
mtu = 16 << 20;
alen = sizeof(struct sockaddr_un);
break;
case AF_INET:
hastrunc = 1;
mru = 65508;
mtu = 65507;
alen = sizeof(struct sockaddr_in);
break;
case AF_INET6:
hastrunc = 1;
mru = 65508;
mtu = 65507;
alen = sizeof(struct sockaddr_in6);
break;
}
return 0;
}
int ClientDgram::allocate_dgram_info(void)
{
if (abuf != NULL)
return 0;
abuf = (DgramInfo *)MALLOC(sizeof(DgramInfo) + alen);
if (abuf == NULL)
return -1;
abuf->cli = this;
return 0;
}
int ClientDgram::Attach()
{
init_socket_info();
enable_input();
if (attach_poller() == -1)
return -1;
return 0;
}
// server peer
// return value: -1, fatal error or no more packets
// return value: 0, more packet may be pending
int ClientDgram::recv_request(int noempty)
{
if (allocate_dgram_info() < 0)
{
log_error("%s", "create DgramInfo object failed, msg[no enough memory]");
return -1;
}
int dataLen;
if (hastrunc)
{
char dummy[1];
abuf->len = alen;
dataLen = recvfrom(netfd, dummy, 1, MSG_DONTWAIT | MSG_TRUNC | MSG_PEEK, (sockaddr *)abuf->addr, &abuf->len);
if (dataLen < 0)
{
// NO ERROR, and assume NO packet pending
return -1;
}
}
else
{
dataLen = -1;
ioctl(netfd, SIOCINQ, &dataLen);
if (dataLen < 0)
{
log_error("%s", "next packet size unknown");
return -1;
}
if (dataLen == 0)
{
if (noempty)
{
// NO ERROR, and assume NO packet pending
return -1;
}
/* treat 0 as max packet, because we can't ditiguish the no-packet & zero-packet */
dataLen = mru;
}
}
char *buf = (char *)MALLOC(dataLen);
if (buf == NULL)
{
log_error("allocate packet buffer[%d] failed, msg[no enough memory]", dataLen);
return -1;
}
abuf->len = alen;
dataLen = recvfrom(netfd, buf, dataLen, MSG_DONTWAIT, (sockaddr *)abuf->addr, &abuf->len);
if (abuf->len <= 1)
{
log_notice("recvfrom error: no source address");
free(buf);
// more packet pending
return 0;
}
if (dataLen <= (int)sizeof(PacketHeader))
{
int err = dataLen == -1 ? errno : 0;
if (err != EAGAIN)
log_notice("recvfrom error: size=%d errno=%d", dataLen, err);
free(buf);
// more packet pending
return 0;
}
TaskRequest *task = new TaskRequest(owner->owner_table());
if (NULL == task)
{
log_error("%s", "create TaskRequest object failed, msg[no enough memory]");
return -1;
}
task->set_hotbackup_table(owner->admin_table());
int ret = task->Decode(buf, dataLen, 1 /*eat*/);
switch (ret)
{
default:
case DecodeFatalError:
if (errno != 0)
log_notice("decode fatal error, ret = %d msg = %m", ret);
free(buf); // buf not eatten
break;
case DecodeDataError:
task->response_timer_start();
if (task->result_code() < 0)
log_notice("DecodeDataError, role=%d, fd=%d, result=%d",
task->Role(), netfd, task->result_code());
send_result(task, (void *)abuf->addr, abuf->len);
break;
case DecodeDone:
if (task->prepare_process() < 0)
{
send_result(task, (void *)abuf->addr, abuf->len);
break;
}
task->set_owner_info(abuf, 0, (sockaddr *)abuf->addr);
abuf = NULL; // eat abuf
task->push_reply_dispatcher(&replyDgram);
owner->task_dispatcher(task);
}
return 0;
}
int ClientDgram::send_result(TaskRequest *task, void *addr, int len)
{
if (task == NULL)
return 0;
owner->record_request_time(task);
Packet *reply = new Packet;
if (reply == NULL)
{
delete task;
log_error("create Packet object failed");
return 0;
}
task->versionInfo.set_keep_alive_timeout(owner->idle_time());
reply->encode_result(task, mtu);
delete task;
int ret = reply->send_to(netfd, (struct sockaddr *)addr, len);
delete reply;
if (ret != SendResultDone)
{
log_notice("send failed, return = %d, error = %m", ret);
return 0;
}
return 0;
}
void ClientDgram::input_notify(void)
{
const int batchsize = 64;
for (int i = 0; i < batchsize; ++i)
{
if (recv_request(i) < 0)
break;
}
}

64
src/search_local/index_storage/common/client_dgram.h Normal file
View File

@ -0,0 +1,64 @@
/*
* =====================================================================================
*
* Filename: client_dgram.h
*
* Description: client dgram.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CLIENT__DGRAM_H__
#define __CLIENT__DGRAM_H__
#include <sys/socket.h>
#include "poller.h"
#include "task_request.h"
#include "packet.h"
#include "timer_list.h"
class DTCDecoderUnit;
class ClientDgram;
struct DgramInfo
{
ClientDgram *cli;
socklen_t len;
char addr[0];
};
class ClientDgram : public PollerObject
{
public:
DTCDecoderUnit *owner;
ClientDgram(DTCDecoderUnit *, int fd);
virtual ~ClientDgram();
virtual int Attach(void);
int send_result(TaskRequest *, void *, int);
protected:
// recv on empty&no packets
int recv_request(int noempty);
private:
int hastrunc;
int mru;
int mtu;
int alen; // address length
DgramInfo *abuf; // current packet address
virtual void input_notify(void);
int allocate_dgram_info(void);
int init_socket_info(void);
};
#endif

263
src/search_local/index_storage/common/client_resource_pool.cc Normal file
View File

@ -0,0 +1,263 @@
/*
* =====================================================================================
*
* Filename: client_resource_pool.cc
*
* Description: client resource pool.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdio.h>
#include <stdlib.h>
#include "client_resource_pool.h"
#include "packet.h"
#include "task_request.h"
#include "log.h"
ClientResourceSlot::~ClientResourceSlot()
{
DELETE(task);
DELETE(packet);
}
ClientResourcePool::~ClientResourcePool()
{
}
/* rules: */
/* head free slot: freeprev == 0
* tail free slot: freenext == 0
* used slot: freeprev == 0, freenext == 0
* non-empty list: freehead, freetail not 0
* empty list: freehead, freetail == 0
* */
int ClientResourcePool::Init()
{
taskslot = (ClientResourceSlot *)CALLOC(256, sizeof(ClientResourceSlot));
if (taskslot == NULL)
return -1;
total = 256;
used = 1;
freehead = 1;
freetail = total - 1;
/* 1(freehead), 2..., total - 1(freetail) */
taskslot[1].freenext = 2;
taskslot[1].freeprev = 0;
if ((taskslot[1].task = new TaskRequest) == NULL)
return -1;
if ((taskslot[1].packet = new Packet) == NULL)
return -1;
taskslot[1].seq = 0;
for (unsigned int i = 2; i < total - 1; i++)
{
taskslot[i].freenext = i + 1;
taskslot[i].freeprev = i - 1;
if ((taskslot[i].task = new TaskRequest) == NULL)
return -1;
if ((taskslot[i].packet = new Packet) == NULL)
return -1;
taskslot[i].seq = 0;
}
taskslot[total - 1].freenext = 0;
taskslot[total - 1].freeprev = total - 2;
if ((taskslot[total - 1].task = new TaskRequest) == NULL)
return -1;
if ((taskslot[total - 1].packet = new Packet) == NULL)
return -1;
taskslot[total - 1].seq = 0;
return 0;
}
int ClientResourcePool::Alloc(unsigned int &id, uint32_t &seq)
{
if (freehead == 0)
{
if (Enlarge() < 0)
return -1;
}
used++;
id = freehead;
freehead = taskslot[id].freenext;
if (freehead == 0)
freetail = 0;
taskslot[id].freenext = 0;
if (freetail != 0)
taskslot[freehead].freeprev = 0;
if (taskslot[id].task == NULL)
{
if ((taskslot[id].task = new TaskRequest) == NULL)
{
used--;
if (freetail == 0)
freetail = id;
taskslot[id].freenext = freehead;
taskslot[freehead].freeprev = id;
freehead = id;
return -1;
}
}
if (taskslot[id].packet == NULL)
{
if ((taskslot[id].packet = new Packet) == NULL)
{
used--;
if (freetail == 0)
freetail = id;
taskslot[id].freenext = freehead;
taskslot[freehead].freeprev = id;
freehead = id;
return -1;
}
}
taskslot[id].seq++;
seq = taskslot[id].seq;
return 0;
}
void ClientResourcePool::Clean(unsigned int id)
{
if (taskslot[id].task)
taskslot[id].task->Clean();
if (taskslot[id].packet)
taskslot[id].packet->Clean();
}
void ClientResourcePool::Free(unsigned int id, uint32_t seq)
{
if (taskslot[id].seq != seq)
{
log_crit("client resource seq not right: %u, %u, not free", taskslot[id].seq, seq);
return;
}
used--;
taskslot[freehead].freeprev = id;
taskslot[id].freenext = freehead;
freehead = id;
if (freetail == 0)
freetail = id;
}
int ClientResourcePool::Fill(unsigned int id)
{
if (taskslot[id].task && taskslot[id].packet)
return 0;
if ((taskslot[id].task == NULL) && ((taskslot[id].task = new TaskRequest) == NULL))
return -1;
if ((taskslot[id].packet == NULL) && ((taskslot[id].packet = new Packet) == NULL))
return -1;
return 0;
}
/* enlarge when freehead == 0, empty list */
int ClientResourcePool::Enlarge()
{
void *p;
unsigned int old = total;
p = REALLOC(taskslot, sizeof(ClientResourceSlot) * 2 * total);
if (!p)
{
log_warning("enlarge taskslot failed, current CTaskRequestSlot %u", total);
return -1;
}
total *= 2;
freehead = old;
freetail = total - 1;
taskslot[old].freenext = old + 1;
taskslot[old].freeprev = 0;
if ((taskslot[old].task = new TaskRequest) == NULL)
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
if ((taskslot[old].packet = new Packet) == NULL)
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
taskslot[old].seq = 0;
for (unsigned int i = old + 1; i < total - 1; i++)
{
taskslot[i].freenext = i + 1;
taskslot[i].freeprev = i - 1;
if ((taskslot[i].task = new TaskRequest) == NULL)
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
if ((taskslot[i].packet = new Packet) == NULL)
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
taskslot[i].seq = 0;
}
taskslot[total - 1].freenext = 0;
taskslot[total - 1].freeprev = total - 2;
if (!(taskslot[total - 1].task = new TaskRequest))
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
if (!(taskslot[total - 1].packet = new Packet))
{
log_warning("no mem for new task slot, current CTaskRequestSlot %u", total);
return -1;
}
taskslot[total - 1].seq = 0;
return 0;
}
void ClientResourcePool::Shrink()
{
int shrink_cnt = (total - used) / 2;
int slot = freetail;
if (freetail == 0)
return;
for (int i = 0; i < shrink_cnt; i++)
{
if (taskslot[slot].task)
{
delete taskslot[slot].task;
taskslot[slot].task = NULL;
}
if (taskslot[slot].packet)
{
delete taskslot[slot].packet;
taskslot[slot].packet = NULL;
}
slot = taskslot[slot].freeprev;
if (slot == 0)
break;
}
}

81
src/search_local/index_storage/common/client_resource_pool.h Normal file
View File

@ -0,0 +1,81 @@
/*
* =====================================================================================
*
* Filename: client_resource_pool.h
*
* Description: client resource pool.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef CLIENT_RESOURCE_POOL_H
#define CLIENT_RESOURCE_POOL_H
#include <stdint.h>
class TaskRequest;
class Packet;
class ClientResourceSlot
{
public:
ClientResourceSlot() : freenext(0),
freeprev(0),
task(NULL),
packet(NULL),
seq(0)
{
}
~ClientResourceSlot();
int freenext;
/*only used if prev slot free too*/
int freeprev;
TaskRequest *task;
Packet *packet;
uint32_t seq;
};
/* automaticaly change size according to usage status */
/* resource from pool need */
class ClientResourcePool
{
public:
ClientResourcePool() : total(0),
used(0),
freehead(0),
freetail(0),
taskslot(NULL)
{
}
~ClientResourcePool();
int Init();
inline ClientResourceSlot *Slot(unsigned int id) { return &taskslot[id]; }
/* clean resource allocated */
int Alloc(unsigned int &id, uint32_t &seq);
int Fill(unsigned int id);
void Clean(unsigned int id);
/* free, should clean first */
void Free(unsigned int id, uint32_t seq);
private:
unsigned int total;
unsigned int used;
unsigned int freehead;
unsigned int freetail;
ClientResourceSlot *taskslot;
int Enlarge();
void Shrink();
inline int half_use()
{
return used <= total / 2 ? 1 : 0;
}
};
#endif

318
src/search_local/index_storage/common/client_sync.cc Normal file
View File

@ -0,0 +1,318 @@
/*
* =====================================================================================
*
* Filename: client_sync.cc
*
* Description: client sync.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <stdio.h>
#include <sys/un.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include "client_sync.h"
#include "task_request.h"
#include "packet.h"
#include "client_resource_pool.h"
#include "client_unit.h"
#include "log.h"
#include <stat_dtc.h>
class PollThread;
static int statEnable = 0;
static StatItemU32 statAcceptCount;
static StatItemU32 statCurConnCount;
class ReplySync : public ReplyDispatcher<TaskRequest>
{
public:
ReplySync(void) {}
virtual ~ReplySync(void) {}
virtual void reply_notify(TaskRequest *task);
};
void ReplySync::reply_notify(TaskRequest *task)
{
ClientSync *cli = task->OwnerInfo<ClientSync>();
DTCDecoderUnit *resourceOwner = (DTCDecoderUnit *)task->resourceOwner;
/* cli not exist, clean&free resource */
if (cli == NULL)
{
if (task->resourceId != 0)
{
resourceOwner->clean_resource(task->resourceId);
resourceOwner->unregist_resource(task->resourceId, task->resourceSeq);
}
}
/* normal case */
else if (cli->send_result() == 0)
{
cli->delay_apply_events();
}
else /* send error, delete client, resource will be clean&free there */
{
// delete client apply delete task
delete cli;
}
}
static ReplySync syncReplyObject;
ClientSync::ClientSync(DTCDecoderUnit *o, int fd, void *peer, int ps) : PollerObject(o->owner_thread(), fd),
owner(o),
receiver(fd),
stage(IdleState),
task(NULL),
reply(NULL)
{
addrLen = ps;
addr = MALLOC(ps);
resourceId = 0;
resource = NULL;
if (addr == NULL)
throw(int) - ENOMEM;
memcpy((char *)addr, (char *)peer, ps);
if (!statEnable)
{
statAcceptCount = statmgr.get_item_u32(ACCEPT_COUNT);
statCurConnCount = statmgr.get_item_u32(CONN_COUNT);
statEnable = 1;
}
/* ClientSync deleted if allocate resource failed. clean resource allocated */
get_resource();
if (resource)
{
task = resource->task;
reply = resource->packet;
}
else
{
throw(int) - ENOMEM;
}
rscStatus = RscClean;
++statAcceptCount;
++statCurConnCount;
}
ClientSync::~ClientSync()
{
if (task)
{
if (stage == ProcReqState)
{
/* task in use, save resource to reply */
task->clear_owner_info();
task->resourceId = resourceId;
task->resourceOwner = owner;
task->resourceSeq = resourceSeq;
}
else
{
/* task not in use, clean resource, free resource */
if (resource)
{
clean_resource();
rscStatus = RscClean;
free_resource();
}
}
}
//DELETE(reply);
FREE_IF(addr);
--statCurConnCount;
}
int ClientSync::Attach()
{
enable_input();
if (attach_poller() == -1)
return -1;
attach_timer(owner->idle_list());
stage = IdleState;
return 0;
}
//server peer
int ClientSync::recv_request()
{
/* clean task from pool, basic init take place of old construction */
if (RscClean == rscStatus)
{
task->set_data_table(owner->owner_table());
task->set_hotbackup_table(owner->admin_table());
task->set_role_as_server();
task->BeginStage();
receiver.erase();
rscStatus = RscDirty;
}
disable_timer();
int ret = task->Decode(receiver);
switch (ret)
{
default:
case DecodeFatalError:
if (errno != 0)
log_notice("decode fatal error, ret = %d msg = %m", ret);
return -1;
case DecodeDataError:
task->response_timer_start();
task->mark_as_hit();
if (task->result_code() < 0)
log_notice("DecodeDataError, role=%d, fd=%d, result=%d", task->Role(), netfd, task->result_code());
return send_result();
case DecodeIdle:
attach_timer(owner->idle_list());
stage = IdleState;
rscStatus = RscClean;
break;
case DecodeWaitData:
stage = RecvReqState;
break;
case DecodeDone:
if ((ret = task->prepare_process()) < 0)
{
log_debug("build packed key error: %d", ret);
return send_result();
}
#if 0
/* 处理任务时，如果client关闭连接，server也应该关闭连接, 所以依然enable_input */
DisableInput();
#endif
disable_output();
task->set_owner_info(this, 0, (struct sockaddr *)addr);
stage = ProcReqState;
task->push_reply_dispatcher(&syncReplyObject);
owner->task_dispatcher(task);
}
return 0;
}
/* keep task in resource slot */
int ClientSync::send_result(void)
{
stage = SendRepState;
owner->record_request_time(task);
if (task->flag_keep_alive())
task->versionInfo.set_keep_alive_timeout(owner->idle_time());
reply->encode_result(task);
return Response();
}
int ClientSync::Response(void)
{
int ret = reply->Send(netfd);
switch (ret)
{
case SendResultMoreData:
enable_output();
return 0;
case SendResultDone:
clean_resource();
rscStatus = RscClean;
stage = IdleState;
disable_output();
enable_input();
attach_timer(owner->idle_list());
return 0;
default:
log_notice("send failed, return = %d, error = %m", ret);
return -1;
}
return 0;
}
void ClientSync::input_notify(void)
{
if (stage == IdleState || stage == RecvReqState)
{
if (recv_request() < 0)
{
delete this;
return;
}
}
/* receive input events again. */
else
{
/* check whether client close connection. */
if (check_link_status())
{
log_debug("client close connection, delete ClientSync obj, stage=%d", stage);
delete this;
return;
}
else
{
DisableInput();
}
}
delay_apply_events();
}
void ClientSync::output_notify(void)
{
if (stage == SendRepState)
{
if (Response() < 0)
delete this;
}
else
{
disable_output();
log_info("Spurious ClientSync::output_notify, stage=%d", stage);
}
}
int ClientSync::get_resource()
{
return owner->regist_resource(&resource, resourceId, resourceSeq);
}
void ClientSync::free_resource()
{
task = NULL;
reply = NULL;
owner->unregist_resource(resourceId, resourceSeq);
}
void ClientSync::clean_resource()
{
owner->clean_resource(resourceId);
}

80
src/search_local/index_storage/common/client_sync.h Normal file
View File

@ -0,0 +1,80 @@
/*
* =====================================================================================
*
* Filename: client_sync.h
*
* Description: client sync.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CLIENT_SYNC_H__
#define __CLIENT_SYNC_H__
#include "poller.h"
#include "timer_list.h"
#include "receiver.h"
class DTCDecoderUnit;
class TaskRequest;
class Packet;
class ClientResourceSlot;
class ClientSync : public PollerObject, private TimerObject
{
public:
DTCDecoderUnit *owner;
void *addr;
int addrLen;
unsigned int resourceId;
ClientResourceSlot *resource;
uint32_t resourceSeq;
enum RscStatus
{
RscClean,
RscDirty,
};
RscStatus rscStatus;
ClientSync(DTCDecoderUnit *, int fd, void *, int);
virtual ~ClientSync();
virtual int Attach(void);
int send_result(void);
virtual void input_notify(void);
private:
virtual void output_notify(void);
int get_resource();
void free_resource();
void clean_resource();
protected:
enum ClientState
{
IdleState,
RecvReqState, //wait for recv request, server side
SendRepState, //wait for send response, server side
ProcReqState, // IN processing
};
SimpleReceiver receiver;
ClientState stage;
TaskRequest *task;
Packet *reply;
int recv_request(void);
int Response(void);
void adjust_events(void);
};
#endif

177
src/search_local/index_storage/common/client_unit.cc Normal file
View File

@ -0,0 +1,177 @@
/*
* =====================================================================================
*
* Filename: client_unit.cc
*
* Description: client uint class definition.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <errno.h>
#include <unistd.h>
#include "client_unit.h"
#include "client_sync.h"
#include "client_async.h"
#include "client_dgram.h"
#include "poll_thread.h"
#include "task_request.h"
#include "log.h"
DTCDecoderUnit::DTCDecoderUnit(PollThread *o, DTCTableDefinition **tdef, int it) : DecoderUnit(o, it),
tableDef(tdef),
output(o)
{
statRequestTime[0] = statmgr.get_sample(REQ_USEC_ALL);
statRequestTime[1] = statmgr.get_sample(REQ_USEC_GET);
statRequestTime[2] = statmgr.get_sample(REQ_USEC_INS);
statRequestTime[3] = statmgr.get_sample(REQ_USEC_UPD);
statRequestTime[4] = statmgr.get_sample(REQ_USEC_DEL);
statRequestTime[5] = statmgr.get_sample(REQ_USEC_FLUSH);
statRequestTime[6] = statmgr.get_sample(REQ_USEC_HIT);
statRequestTime[7] = statmgr.get_sample(REQ_USEC_REPLACE);
if (clientResourcePool.Init() < 0)
throw(int) - ENOMEM;
}
DTCDecoderUnit::~DTCDecoderUnit()
{
}
void DTCDecoderUnit::record_request_time(int hit, int type, unsigned int usec)
{
static const unsigned char cmd2type[] =
{
/*Nop*/ 0,
/*result_code*/ 0,
/*DTCResultSet*/ 0,
/*HelperAdmin*/ 0,
/*Get*/ 1,
/*Purge*/ 5,
/*Insert*/ 2,
/*Update*/ 3,
/*Delete*/ 4,
/*Other*/ 0,
/*Other*/ 0,
/*Other*/ 0,
/*Replace*/ 7,
/*Flush*/ 5,
/*Other*/ 0,
/*Other*/ 0,
};
statRequestTime[0].push(usec);
unsigned int t = hit ? 6 : cmd2type[type];
if (t)
statRequestTime[t].push(usec);
}
void DTCDecoderUnit::record_request_time(TaskRequest *req)
{
record_request_time(req->flag_is_hit(), req->request_code(), req->responseTimer.live());
}
int DTCDecoderUnit::process_stream(int newfd, int req, void *peer, int peerSize)
{
if (req <= 1)
{
ClientSync *cli = NULL;
try
{
cli = new ClientSync(this, newfd, peer, peerSize);
}
catch (int err)
{
DELETE(cli);
return -1;
}
if (0 == cli)
{
log_error("create CClient object failed, errno[%d], msg[%m]", errno);
return -1;
}
if (cli->Attach() == -1)
{
log_error("Invoke CClient::Attach() failed");
delete cli;
return -1;
}
/* accept唤醒后立即recv */
cli->input_notify();
}
else
{
ClientAsync *cli = new ClientAsync(this, newfd, req, peer, peerSize);
if (0 == cli)
{
log_error("create CClient object failed, errno[%d], msg[%m]", errno);
return -1;
}
if (cli->Attach() == -1)
{
log_error("Invoke CClient::Attach() failed");
delete cli;
return -1;
}
/* accept唤醒后立即recv */
cli->input_notify();
}
return 0;
}
int DTCDecoderUnit::process_dgram(int newfd)
{
ClientDgram *cli = new ClientDgram(this, newfd);
if (0 == cli)
{
log_error("create CClient object failed, errno[%d], msg[%m]", errno);
return -1;
}
if (cli->Attach() == -1)
{
log_error("Invoke CClient::Attach() failed");
delete cli;
return -1;
}
return 0;
}
int DTCDecoderUnit::regist_resource(ClientResourceSlot **res, unsigned int &id, uint32_t &seq)
{
if (clientResourcePool.Alloc(id, seq) < 0)
{
id = 0;
*res = NULL;
return -1;
}
*res = clientResourcePool.Slot(id);
return 0;
}
void DTCDecoderUnit::unregist_resource(unsigned int id, uint32_t seq)
{
clientResourcePool.Free(id, seq);
}
void DTCDecoderUnit::clean_resource(unsigned int id)
{
clientResourcePool.Clean(id);
}

61
src/search_local/index_storage/common/client_unit.h Normal file
View File

@ -0,0 +1,61 @@
/*
* =====================================================================================
*
* Filename: client_unit.h
*
* Description: client uint class definition.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __H_DTC_CLIENT_UNIT_H__
#define __H_DTC_CLIENT_UNIT_H__
#include "request_base_all.h"
#include "stat_dtc.h"
#include "decoder_base.h"
#include "stop_watch.h"
#include "client_resource_pool.h"
class TaskRequest;
class DTCTableDefinition;
class DTCDecoderUnit : public DecoderUnit
{
public:
DTCDecoderUnit(PollThread *, DTCTableDefinition **, int);
virtual ~DTCDecoderUnit();
virtual int process_stream(int fd, int req, void *, int);
virtual int process_dgram(int fd);
void bind_dispatcher(TaskDispatcher<TaskRequest> *p) { output.bind_dispatcher(p); }
void task_dispatcher(TaskRequest *task) { output.task_notify(task); }
DTCTableDefinition *owner_table(void) const { return tableDef[0]; }
DTCTableDefinition *admin_table(void) const { return tableDef[1]; }
void record_request_time(int hit, int type, unsigned int usec);
// TaskRequest *p must nonnull
void record_request_time(TaskRequest *p);
void record_rcv_cnt(void);
void record_snd_cnt(void);
int regist_resource(ClientResourceSlot **res, unsigned int &id, uint32_t &seq);
void unregist_resource(unsigned int id, uint32_t seq);
void clean_resource(unsigned int id);
private:
DTCTableDefinition **tableDef;
RequestOutput<TaskRequest> output;
ClientResourcePool clientResourcePool;
StatSample statRequestTime[8];
};
#endif

50
src/search_local/index_storage/common/compiler.h Normal file
View File

@ -0,0 +1,50 @@
/*
* =====================================================================================
*
* Filename: compiler.h
*
* Description: macro definition.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __HIDDEN
#if __GNUC__ >= 4
#define __HIDDEN __attribute__((__visibility__("hidden")))
#define __EXPORT __attribute__((__visibility__("default")))
#define __INLINE inline __attribute__((__always_inline__))
#define HAS_TLS 1
#if __PIC__ || __pic__
#define __TLS_MODEL __attribute__((tls_model("initial-exec")))
#else
#define __TLS_MODEL __attribute__((tls_model("local-exec")))
#endif
#else // __GNUC__
#define __HIDDEN /* */
#define __EXPORT /* */
#define __INLINE inline
#endif // __GNUC__
#ifndef likely
#if __GCC_MAJOR >= 3
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#else
#define likely(x) (x)
#define unlikely(x) (x)
#endif
#endif
#endif // __HIDDEN

97
src/search_local/index_storage/common/compress.cc Normal file
View File

@ -0,0 +1,97 @@
/*
* =====================================================================================
*
* Filename: compress.cc
*
* Description: add field compression.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include "compress.h"
#include "zlib.h"
#include "mem_check.h"
#include <errno.h>
#include <string.h>
#include <stdio.h>
DTCCompress::DTCCompress()
{
_level = 1;
_buflen = 0;
_len = 0;
_buf = NULL;
}
DTCCompress::~DTCCompress()
{
if (_buf)
FREE(_buf);
}
void DTCCompress::set_compress_level(int level)
{
_level = level < 1 ? 1 : (level > 9 ? 9 : level);
}
int DTCCompress::set_buffer_len(unsigned long len)
{
if (_buf == NULL)
{
_buflen = len;
_buf = (unsigned char *)MALLOC(len);
}
else if (_buflen < len)
{
_buflen = len;
FREE(_buf);
_buf = (unsigned char *)MALLOC(len);
}
if (_buf == NULL)
return -ENOMEM;
return 0;
}
//source 被压缩的缓冲区 sourcelen 被压缩缓冲区的原始长度
//dest 压缩后的缓冲区 destlen 被压缩后的缓冲区长度
//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度
int DTCCompress::compress(const char *source, unsigned long sourceLen)
{
if (_buf == NULL || source == NULL)
{
return -111111;
}
_len = _buflen;
return compress2(_buf, &_len, (Bytef *)source, sourceLen, _level);
}
//source 待解压的缓冲区 sourcelen 待解压缓冲区的原始长度
//dest 解压后的缓冲区 destlen 解缩后的缓冲区长度
//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度
int DTCCompress::UnCompress(char **buf, int *lenp, const char *source, unsigned long sourceLen)
{
if (_buf == NULL || source == NULL)
{
snprintf(_errmsg, sizeof(_errmsg), "input buffer or uncompress buffer is null");
return -111111;
}
_len = _buflen;
int iret = uncompress(_buf, &_len, (Bytef *)source, sourceLen);
if (iret)
{
snprintf(_errmsg, sizeof(_errmsg), "uncompress error,error code is:%d.check it in /usr/include/zlib.h", iret);
return -111111;
}
*buf = (char *)MALLOC(_len);
if (*buf == NULL)
return -ENOMEM;
memcpy(*buf, _buf, _len);
*lenp = _len;
return 0;
}

47
src/search_local/index_storage/common/compress.h Normal file
View File

@ -0,0 +1,47 @@
/*
* =====================================================================================
*
* Filename: compress.h
*
* Description: add field compression.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
class DTCCompress
{
public:
DTCCompress();
virtual ~DTCCompress();
void set_compress_level(int level);
int set_buffer_len(unsigned long len);
const char *error_message(void) const { return _errmsg; }
//source 被压缩的缓冲区 sourcelen 被压缩缓冲区的原始长度
//dest 压缩后的缓冲区 destlen 被压缩后的缓冲区长度
//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度
int compress(const char *source, unsigned long sourceLen);
//source 待解压的缓冲区 sourcelen 待解压缓冲区的原始长度
//dest 解压后的缓冲区 destlen 解缩后的缓冲区长度
//注意调用该函数时， destlen 首先要设置为dest缓冲区最大可以容纳的长度
int UnCompress(char **dest, int *destlen, const char *source, unsigned long sourceLen);
unsigned long get_len(void) { return _len; }
char *get_buf(void) { return (char *)_buf; }
char _errmsg[512];
private:
unsigned long _buflen;
unsigned char *_buf;
unsigned long _len;
int _level;
};

553
src/search_local/index_storage/common/config.cc Normal file
View File

@ -0,0 +1,553 @@
/*
* =====================================================================================
*
* Filename: config.cc
*
* Description: configuration acquiry and store.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <ctype.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <ctype.h>
#include <stdio.h>
#include <pthread.h>
#include <sys/mman.h>
#include "mem_check.h"
#include "config.h"
#include "log.h"
int str2int(const char *strval, int def)
{
int ret_code = def;
if (isdigit(strval[0]) || (strval[0] == '-' && isdigit(strval[1])))
return atoi(strval);
if (!strcasecmp(strval, "On"))
ret_code = 1;
else if (!strcasecmp(strval, "Off"))
ret_code = 0;
else if (!strcasecmp(strval, "Yes"))
ret_code = 1;
else if (!strcasecmp(strval, "No"))
ret_code = 0;
else if (!strcasecmp(strval, "True"))
ret_code = 1;
else if (!strcasecmp(strval, "False"))
ret_code = 0;
else if (!strcasecmp(strval, "Enable"))
ret_code = 1;
else if (!strcasecmp(strval, "Disable"))
ret_code = 0;
else if (!strcasecmp(strval, "Enabled"))
ret_code = 1;
else if (!strcasecmp(strval, "Disabled"))
ret_code = 0;
return ret_code;
}
void DTCConfig::Dump(FILE *fp, bool dec)
{
fprintf(fp, "##### ORIGINAL FILE %s #####\n", filename.c_str());
for (secmap_t::iterator i = smap.begin(); i != smap.end(); i++)
{
const char *sec = i->first.c_str();
keymap_t &kmap = i->second;
if (kmap.size() <= 0)
continue;
fprintf(fp, "\n[%s]\n", sec);
for (keymap_t::iterator j = kmap.begin(); j != kmap.end(); j++)
{
const char *key = j->first.c_str();
const char *val = j->second.c_str();
char flag = val[0];
if (flag == '\0')
{
if (dec == true)
fprintf(fp, "# %s NOT SET\n", key);
}
else
{
val++;
if (flag == ' ' || dec == false)
fprintf(fp, "%s = %s\n", key, val);
else
fprintf(fp, "%c %s = %s\n", flag, key, val);
}
}
}
}
int DTCConfig::Dump(const char *fn, bool dec)
{
FILE *fp = fopen(fn, "w");
if (fp == NULL)
return -1;
Dump(fp, dec);
fclose(fp);
return 0;
}
static char *skip_blank(char *p)
{
while (isblank(*p))
p++;
return p;
}
int DTCConfig::parse_buffered_config(char *buf, const char *fn, const char *defsec, bool bakconfig)
{
int len = strlen(buf), ln = 0, ret_code = 0;
char *start = buf, *end = buf;
if (defsec == NULL)
defsec = "";
if (fn == NULL)
fn = "buffered config";
keymap_t *kmap = &smap[defsec];
for (start = buf, ln = 0; end && buf + len > start; start = end + 1)
{
end = strchr(start, '\n');
if (end)
{
if (*end)
*end = '\0';
if (end > start && end[-1] == '\r')
end[-1] = '\0';
}
std::string val("?");
keymap_t *m = kmap;
ln++;
char *v = NULL;
char *key = skip_blank(start);
// blank or comment line
if (key[0] == '\0' || key[0] == '#' || key[0] == '?' || key[0] == ';')
continue;
// key must printable
if (!isprint(key[0]))
{
log_warning("%s(%d): invalid line: %s", fn, ln, key);
ret_code = -1;
continue;
}
// find the equation
start = key + strcspn(key, "= \t");
if (*start != '\0')
{
char *p = start[0] == '=' ? start : skip_blank(start + 1);
if (*p == '=')
{
v = skip_blank(p + 1);
}
else if (key[0] == '[')
/*OK*/;
else
{
log_warning("%s(%d): invalid line: %s", fn, ln, key);
ret_code = -1;
continue;
}
*start = '\0';
}
else if (key[0] != '[')
{
log_warning("%s(%d): invalid line: %s", fn, ln, key);
ret_code = -1;
continue;
}
if (key[0] == '[')
{
char *r = strchr(key, ']');
if (!r)
{
log_warning("%s(%d): invalid section: %s", fn, ln, key);
ret_code = -1;
continue;
}
else if (!((r[1] == '\0' && v == NULL) || (r[1] == '.' && v != NULL)))
{
log_warning("%s(%d): invalid section: %s", fn, ln, key);
ret_code = -1;
continue;
}
else
{
*r = '\0';
m = &smap[key + 1];
if (r[1] == '\0')
{
kmap = m;
continue;
}
key = r + 2;
}
}
if (v == NULL)
continue;
switch (v[0])
{
case '(':
start = strchr(v, ')');
if (start == NULL)
goto error;
break;
case '[':
start = strchr(v, ']');
if (start == NULL)
goto error;
break;
case '{':
start = strchr(v, '}');
if (start == NULL)
goto error;
break;
case '"':
start = strrchr(v + 1, '"');
if (start == NULL)
goto error;
break;
case '\'':
start = strrchr(v + 1, '\'');
if (start == NULL)
goto error;
break;
default:
start = end ? end - 1 : v + strlen(v) - 1;
if (*start == '\0')
start--;
while (start > v && isblank(*start))
start--;
break;
error:
log_warning("%s(%d): unended quote or bracket", fn, ln);
ret_code = -1;
continue;
}
start[1] = '\0';
if (v[0])
val.append(v);
(*m)[key] = val;
}
if (bakconfig)
{
char bak_config[1024];
int err;
snprintf(bak_config, sizeof(bak_config), "cp %s ../stat/", fn);
err = system(bak_config);
}
return ret_code;
}
int DTCConfig::parse_config(const char *fn, const char *defsec, bool bakconfig)
{
char *buf;
int len, fd, ret_code = -1;
if (fn == NULL)
fn = filename.c_str();
else if (filename.size() <= 0)
{
filename = fn;
}
if (filename[0] == '\0')
return -1;
fd = open(fn, O_RDONLY);
if (fd < 0)
{
log_warning("open %s error, cwd=%s, %m\n", fn, get_current_dir_name());
return -1;
}
len = lseek(fd, 0L, SEEK_END);
lseek(fd, 0L, SEEK_SET);
buf = (char *)MALLOC(len + 1);
int unused;
unused = read(fd, buf, len);
buf[len] = '\0';
ret_code = len + 1;
close(fd);
ret_code = parse_buffered_config(buf, fn, defsec, bakconfig);
FREE(buf);
return ret_code;
}
bool DTCConfig::has_section(const char *sec)
{
secmap_t::iterator n = smap.find(sec);
return n != smap.end() && n->second.size() > 0 ? true : false;
}
bool DTCConfig::has_key(const char *sec, const char *key)
{
secmap_t::iterator n = smap.find(sec);
if (n == smap.end())
return false;
keymap_t &kmap = n->second;
keymap_t::iterator m = kmap.find(key);
return m != kmap.end() && m->second.size() > 0 ? true : false;
}
const char *DTCConfig::get_str_val(const char *sec, const char *key)
{
keymap_t &kmap = smap[sec];
std::string &v = kmap[key];
if (v.size() == 0)
return NULL;
v[0] = ' ';
return v.c_str() + 1;
}
int DTCConfig::get_int_val(const char *sec, const char *key, int def)
{
const char *val = get_str_val(sec, key);
if (val == NULL)
return def;
return str2int(val, def);
}
unsigned long long DTCConfig::get_size_val(
const char *sec,
const char *key,
unsigned long long def,
char unit)
{
const char *val = get_str_val(sec, key);
if (val == NULL || !isdigit(val[0]))
return def;
const char *p;
double a = strtod(val, (char **)&p);
if (*p)
unit = *p;
switch (unit)
{
case 'b':
break;
case 'B':
break;
case 'k':
a *= 1000;
break;
case 'K':
a *= 1 << 10;
break;
case 'm':
a *= 1000000;
break;
case 'M':
a *= 1 << 20;
break;
case 'g':
a *= 1000000000;
break;
case 'G':
a *= 1 << 30;
break;
case 't':
a *= 1000000000000LL;
break;
case 'T':
a *= 1ULL << 40;
break;
}
return (unsigned long long)a;
}
int DTCConfig::get_idx_val(
const char *sec, const char *key,
const char *const *array, int nDefault)
{
const char *val = get_str_val(sec, key);
if (val == NULL)
return nDefault;
if (isdigit(val[0]))
{
char *p;
int n = strtol(val, &p, 0);
if (*p == '\0')
{
for (int i = 0; array[i]; i++)
{
if (n == i)
return i;
}
}
}
for (int i = 0; array[i]; i++)
{
if (!strcasecmp(val, array[i]))
return i;
}
return -1;
}
class AutoConfigSection : public AutoConfig
{
private:
static pthread_mutex_t glock;
void GlobalLock(void) { pthread_mutex_lock(&glock); }
void GlobalUnlock(void) { pthread_mutex_unlock(&glock); }
private:
DTCConfig *parent;
char *section;
// buf must have enough room place composed key name
char buf[256];
char *last;
public:
AutoConfigSection(DTCConfig *p, const char *sec);
~AutoConfigSection();
virtual int get_int_val(const char *key, const char *inst, int def = 0);
virtual unsigned long long get_size_val(const char *key, const char *inst, unsigned long long def = 0, char unit = 0);
virtual int get_idx_val(const char *, const char *, const char *const *, int = 0);
virtual const char *get_str_val(const char *key, const char *inst);
private:
// return composed key, or vanilla key, always non-null
const char *findkey(const char *key, const char *inst);
// strip suffix digits
int stripnumber(void);
// strip suffix alphaphetic
int stripalpha(void);
// strip suffix punct
int strippunct(void);
};
pthread_mutex_t AutoConfigSection::glock = PTHREAD_MUTEX_INITIALIZER;
AutoConfigSection::AutoConfigSection(DTCConfig *p, const char *sec)
{
this->parent = p;
this->section = STRDUP(sec);
}
AutoConfigSection::~AutoConfigSection()
{
FREE_CLEAR(section);
}
int AutoConfigSection::stripnumber(void)
{
int n = 0;
while (last >= buf && isdigit(*last))
{
last--;
n++;
}
last[1] = 0;
strippunct();
return n;
}
int AutoConfigSection::stripalpha(void)
{
int n = 0;
while (last >= buf && isalpha(*last))
{
last--;
n++;
}
last[1] = 0;
strippunct();
return n;
}
int AutoConfigSection::strippunct(void)
{
int n = 0;
while (last >= buf && *last != '@' && !isalnum(*last))
{
last--;
n++;
}
last[1] = 0;
return n;
}
const char *AutoConfigSection::findkey(const char *key, const char *inst)
{
snprintf(buf, sizeof(buf), "%s@%s", key, inst);
last = buf + strlen(buf) - 1;
strippunct();
do
{
if (parent->get_str_val(section, buf) != NULL)
{
return buf;
}
} while (isdigit(*last) ? stripnumber() : stripalpha());
return key;
}
int AutoConfigSection::get_int_val(const char *key, const char *inst, int def)
{
int ret;
GlobalLock();
ret = parent->get_int_val(section, findkey(key, inst), def);
GlobalUnlock();
return ret;
}
unsigned long long AutoConfigSection::get_size_val(const char *key, const char *inst, unsigned long long def, char unit)
{
unsigned long long ret;
GlobalLock();
ret = parent->get_size_val(section, findkey(key, inst), def, unit);
GlobalUnlock();
return ret;
}
int AutoConfigSection::get_idx_val(const char *key, const char *inst, const char *const *idxval, int def)
{
int ret;
GlobalLock();
ret = parent->get_idx_val(section, findkey(key, inst), idxval, def);
GlobalUnlock();
return ret;
}
const char *AutoConfigSection::get_str_val(const char *key, const char *inst)
{
const char *ret;
GlobalLock();
ret = parent->get_str_val(section, findkey(key, inst));
GlobalUnlock();
return ret;
}
AutoConfig *DTCConfig::get_auto_config_instance(const char *section)
{
AutoConfigSection *inst;
NEW(AutoConfigSection(this, section), inst);
return inst;
}

72
src/search_local/index_storage/common/config.h Normal file
View File

@ -0,0 +1,72 @@
/*
* =====================================================================================
*
* Filename: config.h
*
* Description: configuration acquiry and store.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef __CONFIG_H__
#define __CONFIG_H__
#include <stdio.h>
#include <string.h>
#include <string>
#include <map>
class AutoConfig
{
public:
AutoConfig(){};
virtual ~AutoConfig(){};
virtual int get_int_val(const char *key, const char *inst, int def = 0) = 0;
virtual unsigned long long get_size_val(const char *key, const char *inst, unsigned long long def = 0, char unit = 0) = 0;
virtual int get_idx_val(const char *, const char *, const char *const *, int = 0) = 0;
virtual const char *get_str_val(const char *key, const char *inst) = 0;
};
class DTCConfig
{
public:
DTCConfig(){};
~DTCConfig(){};
AutoConfig *get_auto_config_instance(const char *);
int get_int_val(const char *sec, const char *key, int def = 0);
unsigned long long get_size_val(const char *sec, const char *key, unsigned long long def = 0, char unit = 0);
int get_idx_val(const char *, const char *, const char *const *, int = 0);
const char *get_str_val(const char *sec, const char *key);
bool has_section(const char *sec);
bool has_key(const char *sec, const char *key);
void Dump(FILE *fp, bool dec = false);
int Dump(const char *fn, bool dec = false);
int parse_config(const char *f = 0, const char *s = 0, bool bakconfig = false);
int parse_buffered_config(char *buf, const char *fn = 0, const char *s = 0, bool bakconfig = false);
private:
struct nocase
{
bool operator()(const std::string &a, const std::string &b) const
{
return strcasecmp(a.c_str(), b.c_str()) < 0;
}
};
typedef std::map<std::string, std::string, nocase> keymap_t;
typedef std::map<std::string, keymap_t, nocase> secmap_t;
private:
std::string filename;
secmap_t smap;
};
#endif

63
src/search_local/index_storage/common/consistent_hash_selector.cc Normal file
View File

@ -0,0 +1,63 @@
/*
* =====================================================================================
*
* Filename: consistent_hash_selector.cc
*
* Description: consistent hash selector.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#include <alloca.h>
#include <cstring>
#include <cstdio>
#include <cassert>
#include <algorithm>
#include "consistent_hash_selector.h"
#include "log.h"
const std::string &ConsistentHashSelector::Select(uint32_t hash)
{
static std::string empty;
if (m_nodes.empty())
return empty;
std::map<uint32_t, int>::iterator iter = m_nodes.upper_bound(hash);
if (iter != m_nodes.end())
return m_nodeNames[iter->second];
return m_nodeNames[m_nodes.begin()->second];
}
void ConsistentHashSelector::add_node(const char *name)
{
if (find(m_nodeNames.begin(), m_nodeNames.end(), name) != m_nodeNames.end())
{
log_crit("duplicate node name: %s in ClusterConfig", name);
abort();
}
m_nodeNames.push_back(name);
int index = m_nodeNames.size() - 1;
char *buf = (char *)alloca(strlen(name) + 16);
for (int i = 0; i < VIRTUAL_NODE_COUNT; ++i)
{
snprintf(buf, strlen(name) + 16, "%s#%d", name, i);
uint32_t value = Hash(buf, strlen(buf));
std::map<uint32_t, int>::iterator iter = m_nodes.find(value);
if (iter != m_nodes.end())
{
//hash值冲突，选取字符串中较小者
if (m_nodeNames[iter->second] < name)
continue;
}
m_nodes[value] = index;
}
}

48
src/search_local/index_storage/common/consistent_hash_selector.h Normal file
View File

@ -0,0 +1,48 @@
/*
* =====================================================================================
*
* Filename: consistent_hash_selector.h
*
* Description: consistent hash selector.
*
* Version: 1.0
* Created: 09/08/2020 10:02:05 PM
* Revision: none
* Compiler: gcc
*
* Author: qiulu, choulu@jd.com
* Company: JD.com, Inc.
*
* =====================================================================================
*/
#ifndef CONSISTENT_HASH_SELECTOR_H__
#define CONSISTENT_HASH_SELECTOR_H__
#include <stdint.h>
#include <map>
#include <string>
#include <vector>
#include "chash.h"
class ConsistentHashSelector
{
public:
uint32_t Hash(const char *key, int len) { return chash(key, len); }
const std::string &Select(uint32_t hash);
const std::string &Select(const char *key, int len) { return Select(Hash(key, len)); }
static const int VIRTUAL_NODE_COUNT = 100;
void add_node(const char *name);
void Clear()
{
m_nodes.clear();
m_nodeNames.clear();
}
private:
std::map<uint32_t, int> m_nodes;
std::vector<std::string> m_nodeNames;
};
#endif

Some files were not shown because too many files have changed in this diff Show More