diff --git a/source/tools/monitor/unity/Dockerfile b/source/tools/monitor/unity/Dockerfile index 767101102f9d26ef57c5f01d46f553be42dec093..dbaf30641e3483f49d8f4606c25ef0940ec7af82 100644 --- a/source/tools/monitor/unity/Dockerfile +++ b/source/tools/monitor/unity/Dockerfile @@ -2,7 +2,7 @@ FROM registry.cn-hangzhou.aliyuncs.com/sysom/lcc MAINTAINER "liaozhaoyan " WORKDIR /root/ RUN source /opt/rh/devtoolset-9/enable && \ - yum install -y make wget * unzip git && \ + yum install -y make wget lua-devel unzip git && \ mkdir /root/build && \ cd /root/build && \ git clone https://gitee.com/chuyansz/sysak.git && \ @@ -19,6 +19,10 @@ RUN source /opt/rh/devtoolset-9/enable && \ cd yaml-0.2.5 && \ ./configure && make && make install && \ cd ../ && \ + tar curl-7.87.0.tar.gz && \ + cd curl-7.87.0 && \ + ./configure --with-openssl && make && make install && \ + cd ../ && \ luarocks install luasec && \ luarocks install lua-cjson && \ luarocks install luasocket && \ @@ -26,9 +30,9 @@ RUN source /opt/rh/devtoolset-9/enable && \ luarocks install lbase64 && \ luarocks install lua-csnappy && \ luarocks install lua-protobuf && \ - luarocks install luaposix && \ luarocks install sha1 && \ luarocks install md5 && \ + luarocks install luaposix 35.1-1 && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ && \ cd ../beeQ/ && \ make \ No newline at end of file diff --git a/source/tools/monitor/unity/ReadMe.md b/source/tools/monitor/unity/ReadMe.md index 05a4137090d1cba82915b468369018ccecedc06e..9b10a3a74ad6fe5826aa3b71a2547979a10fe206 100644 --- a/source/tools/monitor/unity/ReadMe.md +++ b/source/tools/monitor/unity/ReadMe.md @@ -26,7 +26,7 @@ docker exec -it unity-devel bash 如果采用已经好的镜像 ```bash -docker run -itd --net=host --name unity-devel registry.cn-hangzhou.aliyuncs.com/sysom/unity:v1.0 +docker run -itd --net=host --name unity-devel registry.cn-hangzhou.aliyuncs.com/sysom/unity:v1.1 docker exec -it unity-devel bash ``` diff --git a/source/tools/monitor/unity/beaver/Makefile b/source/tools/monitor/unity/beaver/Makefile index 4968b522f36f375ee9c2644dd92ed663d0173746..eff2c0b32e1ec949c0d2b476e683e82433d63b5a 100644 --- a/source/tools/monitor/unity/beaver/Makefile +++ b/source/tools/monitor/unity/beaver/Makefile @@ -3,17 +3,22 @@ AR := ar CFLAG := -g OBJS := beaver.o echos.o LIB := libbeaver.a +DEPMOD = native -all: $(LIB) +all: $(DEPMOD) $(LIB) -echos.o: echos.c beaver.h echos.h - $(CC) -c $< -o $@ $(CFLAG) -beaver.o: beaver.c beaver.h echos.h +beaver.o: beaver.c beaver.h $(CC) -c $< -o $@ $(CFLAG) $(LIB): $(OBJS) $(AR) cr $@ $(OBJS) +$(DEPMOD):ECHO + make -C $@ + +ECHO: + @echo $(SUBDIRS) + clean: rm -f $(EXEC) $(OBJS) diff --git a/source/tools/monitor/unity/beaver/beaver.c b/source/tools/monitor/unity/beaver/beaver.c index 53c4b068b6e125be07bdbcf9917cfc79ce9d1ec3..cb09d3171771f06179f9b33f89ac0723d77257a7 100644 --- a/source/tools/monitor/unity/beaver/beaver.c +++ b/source/tools/monitor/unity/beaver/beaver.c @@ -2,402 +2,154 @@ // Created by 廖肇燕 on 2022/12/20. // +#include "beaver.h" #include #include -#include -#include +#include #include -#include #include -#include +#include +#include #include -#include -#include -#include -#include -#include "beaver.h" -#include "echos.h" - -extern volatile int sighup_counter; - -#define FD_RATIO 2 // fd_num vs thread -#define gettidv1() syscall(__NR_gettid) -#define unlikely(x) __builtin_expect((x),0) -#define ASSERT_LOCKS(X) do{ \ -if (unlikely(X < 0)) \ -{ \ - perror("hold lock failed."); \ - exit(1); \ -}\ -}while(0) +LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); -struct beaver_message { - pthread_mutex_t pc_mutex; - pthread_cond_t pc_condp, pc_condc; - int sk_accept; - int status; // working is 0, bad is -1; - int thread; // thread num - int fd_num; // fd pool number max value. - int fd_count; // fd pool realtime count - int* fds; // fd pool -}; - -static int beaver_broad_stop(struct beaver_message* pmsg) { - int ret = 0; - pmsg->status = -1; - - ret = pthread_cond_broadcast(&pmsg->pc_condc); - ASSERT_LOCKS(ret); - ret = pthread_cond_broadcast(&pmsg->pc_condp); - ASSERT_LOCKS(ret); - - return ret; +static void report_lua_failed(lua_State *L) { + fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); } -static int fd_in_beaver(int fd, struct beaver_message* pmsg){ - if (pmsg->status) { // bad status should exit. - return 0; - } +static int call_init(lua_State *L, int tid, int backlog) { + int ret; + lua_Number lret; - if (pmsg->fd_count == pmsg->fd_num) { //fd poll may large - return 1; + lua_getglobal(L, "init"); + lua_pushinteger(L, tid); + lua_pushinteger(L, backlog); + ret = lua_pcall(L, 2, 1, 0); + if (ret) { + perror("luaL_call init func error"); + report_lua_failed(L); + goto endCall; } - for (int i = 0; i < pmsg->fd_num; ++i) { - if (pmsg->fds[i] == fd) { - return 1; - } + if (!lua_isnumber(L, -1)) { // check + errno = -EINVAL; + perror("function beaver.lua init must return a number."); + goto endReturn; + } + lret = lua_tonumber(L, -1); + lua_pop(L, 1); + if (lret < 0) { + errno = -EINVAL; + ret = -1; + perror("beaver.lua init failed."); + goto endReturn; } - return 0; + return ret; + endReturn: + endCall: + return ret; } -static int add_in_beaver(int fd, struct beaver_message* pmsg){ - for (int i = 0; i < pmsg->fd_num; i ++) { -#ifdef BEAVER_DEBUG - if (pmsg->fds[i] == fd) { - fprintf(stderr, "bug: fd %d is already in fds.\n", fd); - exit(0); - } -#endif - if (pmsg->fds[i] == -1) { - pmsg->fds[i] = fd; - pmsg->fd_count ++; - return 0; - } - } -#ifdef BEAVER_DEBUG - fprintf(stderr, "bug: add %d to fds failed..\n\t", fd); - for (int i = 0; i < pmsg->fd_num; i ++) { - fprintf(stderr, "%d ", pmsg->fds[i]); - } - fprintf(stderr, "\n"); - exit(0); -#endif - return 1; +void LuaAddPath(lua_State *L, char *name, char *value) { + char s[256]; + + lua_getglobal(L, "package"); + lua_getfield(L, -1, name); + strcpy(s, lua_tostring(L, -1)); + strcat(s, ";"); + strcat(s, value); + strcat(s, ";"); + lua_pushstring(L, s); + lua_setfield(L, -3, name); + lua_pop(L, 2); } -static int del_in_beaver(int fd, struct beaver_message* pmsg){ +static lua_State * echos_init(int port, int backlog) { int ret; - ret = pthread_mutex_lock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - for (int i = 0; i < pmsg->fd_num; ++i) { - if (pmsg->fds[i] == fd) { - pmsg->fds[i] = -1; - pmsg->fd_count --; - ret = pthread_cond_signal(&pmsg->pc_condp); - ASSERT_LOCKS(ret); - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - return 0; - } - } -#ifdef BEAVER_DEBUG - fprintf(stderr, "bug: fd %d is not in fds.\n", fd); - exit(0); -#endif - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - return 1; -} -static void * beaver_threads(void * arg) { - int ret; - int fd; - int tid = (int)gettidv1(); - struct beaver_message * pmsg = (struct beaver_message *)arg; - lua_State *L; - int count = sighup_counter; - - // init is here. - L = echos_init(tid); + /* create a state and load standard library. */ + lua_State *L = luaL_newstate(); if (L == NULL) { - exit(1); + perror("new lua failed."); + goto endNew; } - while (1) { - ret = pthread_mutex_lock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - - while (pmsg->sk_accept == 0 || pmsg->status != 0){ - ret = pthread_cond_wait(&pmsg->pc_condc, &pmsg->pc_mutex); - ASSERT_LOCKS(ret); - } - if (pmsg->status != 0) { - goto endStatus; - } - fd = pmsg->sk_accept; - pmsg->sk_accept = 0; - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - - if (count != sighup_counter) { // need to reload ? - lua_close(L); - L = NULL; + /* opens all standard Lua libraries into the given state. */ + luaL_openlibs(L); - L = echos_init(tid); - if (L == NULL) { - exit(1); - } - count = sighup_counter; - } - - //work here - ret = echos(L, fd); - close(fd); - del_in_beaver(fd, pmsg); + LuaAddPath(L, "path", "../beaver/?.lua"); - if (ret) { - goto endThread; + ret = luaL_loadfile(L, "../beaver/beaver.lua"); + printf("ret: %d\n", ret); + ret = lua_pcall(L, 0, LUA_MULTRET, 0); + if (ret) { + const char *msg = lua_tostring(L, -1); + perror("luaL_dofile error"); + if (msg) { + luaL_traceback(L, L, msg, 0); + fprintf(stderr, "FATAL ERROR:%s\n\n", msg); } + goto endLoad; } - endStatus: - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - endThread: - lua_close(L); - beaver_broad_stop(pmsg); - pthread_exit(NULL); -} - -static int beaver_setup_message(struct beaver_message* pmsg) { - int ret; - int fret; - pmsg->sk_accept = 0; - pmsg->status = 0; - - ret = pthread_mutex_init(&pmsg->pc_mutex, NULL); - if (ret < 0) { - perror("pc_mutex create failed."); - goto endMutex; - } - ret = pthread_cond_init(&pmsg->pc_condp, NULL); + ret = call_init(L, port, backlog); if (ret < 0) { - perror("pc_condp create failed."); - goto endCondp; - } - ret = pthread_cond_init(&pmsg->pc_condc, NULL); - if (ret < 0) { - perror("pc_condc create failed."); - goto endCondc; + goto endCall; } - return ret; - endCondc: - fret = pthread_cond_destroy(&pmsg->pc_condp); - if (fret < 0){ - perror("destroy condp faild."); - exit(1); - } - endCondp: - fret = pthread_mutex_destroy(&pmsg->pc_mutex); - if (fret < 0){ - perror("destroy pc_mutex faild."); - exit(1); - } - endMutex: - return ret; + return L; + endCall: + endLoad: + lua_close(L); + endNew: + return NULL; } -static void beaver_destroy_message(struct beaver_message* pmsg) { +static int echos(lua_State *L) { int ret; + lua_Number lret; - ret = pthread_cond_destroy(&pmsg->pc_condc); - if (ret < 0){ - perror("destroy condc faild."); - exit(1); - } - ret = pthread_cond_destroy(&pmsg->pc_condp); - if (ret < 0){ - perror("destroy condp faild."); - exit(1); + lua_getglobal(L, "echo"); + ret = lua_pcall(L, 0, 1, 0); + if (ret) { + perror("lua call error"); + report_lua_failed(L); + goto endCall; } - ret = pthread_mutex_destroy(&pmsg->pc_mutex); - if (ret < 0){ - perror("destroy pc_mutex faild."); - exit(1); - } -} - -static int beaver_threads_start(int thread, pthread_t ** tid_arr, struct beaver_message* pmsg) { - int ret = 0; - pthread_t *tids; - - tids = (pthread_t *) malloc(thread * sizeof (pthread_t)); - if (tids == NULL) { - perror("beaver thread malloc."); - ret = -ENOMEM; - goto endMalloc; - } - for (int i = 0; i < thread; i ++) { - tids[i] = 0; - } - *tid_arr = tids; - ret = beaver_setup_message(pmsg); - if (ret < 0) { - goto endMessage; + if (!lua_isnumber(L, -1)) { // check + errno = -EINVAL; + perror("function beaver.lua init must return a number."); + goto endReturn; } - - for (int i = 0; i < thread; i ++) { - ret = pthread_create(&tids[i], NULL, beaver_threads, pmsg); - ASSERT_LOCKS(ret); + lret = lua_tonumber(L, -1); + lua_pop(L, 1); + if (lret < 0) { + errno = -EINVAL; + ret = -1; + perror("beaver.lua init failed."); + goto endReturn; } return ret; - endMessage: - free(tids); - endMalloc: + endReturn: + endCall: return ret; } -static int beaver_thread_stop(int thread, pthread_t ** tid_arr, struct beaver_message* pmsg) { +int beaver_init(int port, int backlog) { int ret = 0; - pthread_t * tids = *tid_arr; - beaver_broad_stop(pmsg); - for (int i = 0; i < thread; i ++) { - if (tids[i] > 0) { - ret = pthread_join(tids[i], NULL); - ASSERT_LOCKS(ret); + while (ret == 0) { + lua_State *L = echos_init(port, backlog); + if (L == NULL) { + break; } + ret = echos(L); + lua_close(L); + sleep(5); // to release port } - free(tids); - *tid_arr = NULL; - return 0; -} - -static int beaver_accept(int sk_listen, struct beaver_message* pmsg) { - struct sockaddr_in cli_addr; - socklen_t len = sizeof(cli_addr); - int sk_accept; - int ret; - - ret = pthread_mutex_lock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - pmsg->fds = malloc(sizeof (int) * pmsg->fd_num); - if (pmsg->fds == NULL) { - ret = -ENOMEM; - goto endMem; - } - for (int i = 0; i < pmsg->fd_num; i ++) { - pmsg->fds[i] = -1; - } - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - - while (1) { - sk_accept = accept(sk_listen, (struct sockaddr *)&cli_addr, &len); - if (sk_accept < 0) { - perror("accept failed."); - ret = sk_accept; - goto endAccept; - } - - ret = pthread_mutex_lock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - while (fd_in_beaver(sk_accept, pmsg)){ // new fd may re - ret = pthread_cond_wait(&pmsg->pc_condp, &pmsg->pc_mutex); - ASSERT_LOCKS(ret); - } - if (pmsg->status) { - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - goto endStatus; - } - add_in_beaver(sk_accept, pmsg); - - pmsg->sk_accept = sk_accept; - ret = pthread_cond_signal(&pmsg->pc_condc); - ASSERT_LOCKS(ret); - - ret = pthread_mutex_unlock(&pmsg->pc_mutex); - ASSERT_LOCKS(ret); - } - endStatus: - endAccept: - free(pmsg->fds); - endMem: - return ret; -} - -int beaver_init(int port, int thread) { - int ret; - int sk_listen; - int sockopt = 1; - struct sockaddr_in srvaddr; - pthread_t * tids; - struct beaver_message msg; - - msg.thread = thread; - msg.fd_num = thread * FD_RATIO; - msg.fd_count = 0; - - sk_listen = socket(AF_INET, SOCK_STREAM, 0); - if (sk_listen < 0) { - perror("create socket failed."); - ret = -ENOENT; - goto endSocket; - } - - srvaddr.sin_family = AF_INET; - srvaddr.sin_port = htons(port); - srvaddr.sin_addr.s_addr = htonl(INADDR_ANY); - - ret = setsockopt(sk_listen, SOL_SOCKET, SO_REUSEADDR, &sockopt, sizeof(int)); - if (ret < 0) { - perror("setsockopt socket failed."); - goto endSetoption; - } - - ret = bind(sk_listen, (struct sockaddr *)&srvaddr, sizeof(srvaddr)); - if (ret < 0) { - perror("bind socket failed."); - goto endBind; - } - - ret = listen(sk_listen, thread); - if (ret < 0) { - perror("listen socket failed."); - goto endListen; - } - - ret = beaver_threads_start(thread, &tids, &msg); - if (ret < 0) { - goto endThread; - } - - printf("listen %d for %d threads.\n", port, thread); - beaver_accept(sk_listen, &msg); - beaver_thread_stop(thread, &tids, &msg); - return ret; - endThread: - endListen: - endBind: - endSetoption: - close(sk_listen); - endSocket: - return ret; + exit(1); } diff --git a/source/tools/monitor/unity/beaver/beaver.lua b/source/tools/monitor/unity/beaver/beaver.lua index 2567102d978de4ce2b56ea2c01407f11361a4f58..47f60506c8b2772557af38a707a130b1856abe93 100644 --- a/source/tools/monitor/unity/beaver/beaver.lua +++ b/source/tools/monitor/unity/beaver/beaver.lua @@ -4,36 +4,39 @@ --- DateTime: 2022/12/21 11:44 AM --- -package.path = package.path .. ";../common/?.lua;" -package.path = package.path .. ";../httplib/?.lua;" -package.path = package.path .. ";../tsdb/?.lua;" -package.path = package.path .. ";../tsdb/native/?.lua;" - -local Cframe = require("frame") -local CurlApi = require("url_api") -local CurlRpc = require("url_rpc") -local CurlIndex = require("index") -local Cexport = require("export") -local CurlGuide = require("url_guide") -local CurlExportHtml = require("url_export_html") -local CurlExportRaw = require("url_export_raw") - -local web = Cframe.new() - -function init(tid) - print("hello beaver " .. tid) +package.path = package.path .. ";../?.lua;" + +local Cframe = require("beaver.frame") +local CurlApi = require("beaver.url_api") +local CurlRpc = require("beaver.url_rpc") +local CurlIndex = require("beaver.index") +local Cexport = require("beaver.export") +local CurlGuide = require("beaver.url_guide") +local CurlExportHtml = require("beaver.url_export_html") +local CurlExportRaw = require("beaver.url_export_raw") +local CLocalBeaver = require("beaver.localBeaver") + +local lb = nil + +function init(port, backlog) + local web = Cframe.new() + CurlIndex.new(web) CurlApi.new(web) CurlRpc.new(web) CurlGuide.new(web) - local export = Cexport.new("12345abdc") + local Cidentity = require("beaver.identity") + local inst = Cidentity.new() + local export = Cexport.new(inst:id()) CurlExportHtml.new(web, export) CurlExportRaw.new(web, export) + + lb = CLocalBeaver.new(web, port, nil, backlog) return 0 end -function echo(fd) - web:proc(fd) +function echo() + lb:poll() return 0 end diff --git a/source/tools/monitor/unity/beaver/echos.c b/source/tools/monitor/unity/beaver/echos.c deleted file mode 100644 index 7938c83eff8f3a554438f6edb6fdf024ede5554d..0000000000000000000000000000000000000000 --- a/source/tools/monitor/unity/beaver/echos.c +++ /dev/null @@ -1,141 +0,0 @@ -// -// Created by 廖肇燕 on 2022/12/20. -// - -#include "echos.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); - -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, int tid) { - int ret; - lua_Number lret; - - lua_getglobal(L, "init"); - lua_pushinteger(L, tid); - ret = lua_pcall(L, 1, 1, 0); - if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); - goto endCall; - } - - if (!lua_isnumber(L, -1)) { // check - errno = -EINVAL; - perror("function beaver.lua init must return a number."); - goto endReturn; - } - lret = lua_tonumber(L, -1); - lua_pop(L, 1); - if (lret < 0) { - errno = -EINVAL; - ret = -1; - perror("beaver.lua init failed."); - goto endReturn; - } - - return ret; - endReturn: - endCall: - return ret; -} - -void LuaAddPath(lua_State *L, char *name, char *value) { - char s[256]; - - lua_getglobal(L, "package"); - lua_getfield(L, -1, name); - strcpy(s, lua_tostring(L, -1)); - strcat(s, ";"); - strcat(s, value); - strcat(s, ";"); - lua_pushstring(L, s); - lua_setfield(L, -3, name); - lua_pop(L, 2); -} - -lua_State * echos_init(int tid) { - int ret; - - /* create a state and load standard library. */ - lua_State *L = luaL_newstate(); - if (L == NULL) { - perror("new lua failed."); - goto endNew; - } - - /* opens all standard Lua libraries into the given state. */ - luaL_openlibs(L); - - LuaAddPath(L, "path", "../beaver/?.lua"); - - ret = luaL_loadfile(L, "../beaver/beaver.lua"); - printf("ret: %d\n", ret); - ret = lua_pcall(L, 0, LUA_MULTRET, 0); - if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } - goto endLoad; - } - - ret = call_init(L, tid); - if (ret < 0) { - goto endCall; - } - - return L; - endCall: - endLoad: - lua_close(L); - endNew: - return NULL; -} - -int echos(lua_State *L, int fd) { - int ret; - lua_Number lret; - - lua_getglobal(L, "echo"); - lua_pushinteger(L, fd); - ret = lua_pcall(L, 1, 1, 0); - if (ret) { - perror("lua call error"); - report_lua_failed(L); - goto endCall; - } - - if (!lua_isnumber(L, -1)) { // check - errno = -EINVAL; - perror("function beaver.lua init must return a number."); - goto endReturn; - } - lret = lua_tonumber(L, -1); - lua_pop(L, 1); - if (lret < 0) { - errno = -EINVAL; - ret = -1; - perror("beaver.lua init failed."); - goto endReturn; - } - - return ret; - endReturn: - endCall: - return ret; -} \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/echos.h b/source/tools/monitor/unity/beaver/echos.h deleted file mode 100644 index a91dded3ff10e733cb5758bfef8c68b47296939f..0000000000000000000000000000000000000000 --- a/source/tools/monitor/unity/beaver/echos.h +++ /dev/null @@ -1,13 +0,0 @@ -// -// Created by 廖肇燕 on 2022/12/20. -// - -#ifndef UNITY_ECHOS_H -#define UNITY_ECHOS_H -#include - -#define BEAVER_DEBUG -lua_State * echos_init(int tid); -int echos(lua_State *L, int fd); - -#endif //UNITY_ECHOS_H diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index d12d20e1388bb6f28a5f1eeff40384baa4942b8d..f17ec27745b8f2ed543769652c187b5a175d5890 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -4,31 +4,23 @@ --- DateTime: 2022/12/25 11:20 AM --- -local system = require("system") -local pystring = require("pystring") -local CfoxTSDB = require("foxTSDB") -require("class") +local system = require("common.system") +local pystring = require("common.pystring") +local CfoxTSDB = require("tsdb.foxTSDB") +require("common.class") local Cexport = class("Cexport") function Cexport:_init_(instance, fYaml) self._instance = instance fYaml = fYaml or "../collector/plugin.yaml" - local ms = self:_load(fYaml) + local ms = system:parseYaml(fYaml) + self._freq = ms.config.freq self._tDescr = ms.metrics self._fox = CfoxTSDB.new() self._fox:_setupRead() end -function Cexport:_load(fYaml) - local lyaml = require("lyaml") - local f = io.open(fYaml,"r") - local s = f:read("*all") - f:close() - - return lyaml.load(s) -end - local function qFormData(from, tData) local res = {} local len = #tData @@ -63,7 +55,7 @@ end function Cexport:export() local qs = {} self._fox:resize() - self._fox:qlast(15, qs) + self._fox:qlast(self._freq, qs) local res = {} for _, line in ipairs(self._tDescr) do local from = line.from diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 7de16848fa8d275de25de4f09bab85c8dabe323b..0c243edd09cf599b54b9ddd2ab20b4b79e022402 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -9,10 +9,9 @@ local unistd = require("posix.unistd") local poll = require("posix.poll") -require("class") -local ChttpComm = require("httpComm") -local pystring = require("pystring") -local serpent = require("serpent") +require("common.class") +local ChttpComm = require("httplib.httpComm") +local pystring = require("common.pystring") local Cframe = class("frame", ChttpComm) function Cframe:_init_() @@ -20,27 +19,23 @@ function Cframe:_init_() self._objs = {} end -local function waitDataRest(fd, rest, tReq, tmo) - tmo = tmo or 100 -- max wait 100ms, other wise +local function waitDataRest(fread, rest, tReq) local len = 0 - local tStream = {} + local tStream = {tReq.data} while len < rest do - local r = poll.rpoll(fd, tmo) - if r == 0 then -- wait time out - return -1 - elseif r == 1 then - local s = unistd.read(fd, 4096) - table.insert(tStream, s) + local s = fread() + if s then len = len + #s - else -- bad socket - return -2 + table.insert(tStream, s) + else + return -1 end end - tReq.data = tReq.data .. pystring:join("", tStream) + tReq.data = pystring:join("", tStream) return 0 end -local function waitHttpRest(fd, tReq) +local function waitHttpRest(fread, tReq) if tReq.header["content-length"] then local lenData = #tReq.data local lenInfo = tonumber(tReq.header["content-length"]) @@ -50,39 +45,29 @@ local function waitHttpRest(fd, tReq) return -1 end - if waitDataRest(fd, rest, tReq) < 0 then + if waitDataRest(fread, rest, tReq) < 0 then return -2 end end return 0 end -local function waitHttpHead(fd, tmo, maxLen) - tmo = tmo or 100 -- max wait 100ms, - maxLen = maxLen or 8192 - local use = 0 +local function waitHttpHead(fread) local stream = "" - while tmo > 0 do - local r = poll.rpoll(fd, tmo) - if r == 0 then - return nil - elseif r == 1 then - local s = unistd.read(fd, maxLen - use) - if type(s) == "string" then - stream = stream .. s - use = use + #stream - if string.find(stream, "\r\n\r\n") then -- http head end with \r\n\r\n - return stream - end + while true do + local s = fread() + if s then + stream = stream .. s + if string.find(stream, "\r\n\r\n") then + return stream end - tmo = tmo - 1 -- time quota weill - else -- bad socket + else return nil end end end -function Cframe:parse(fd, stream) +function Cframe:parse(fread, stream) local tStatus = pystring:split(stream, "\r\n", 1) if #tStatus < 2 then print("bad stream format.") @@ -121,7 +106,7 @@ function Cframe:parse(fd, stream) end tReq.header = header tReq.data = data - if waitHttpRest(fd, tReq) < 0 then + if waitHttpRest(fread, tReq) < 0 then return nil end return tReq @@ -138,27 +123,26 @@ function Cframe:echo404() return pystring:join("\r\n", tHttp) end -function Cframe:proc(fd) - local stream = waitHttpHead(fd) - if type(stream) ~= "string" then -- read return stream or error code or nil - return stream +function Cframe:proc(fread) + local stream = waitHttpHead(fread) + if stream == nil then -- read return stream or error code or nil + return nil end - local tReq = self:parse(fd, stream) + local tReq = self:parse(fread, stream) if tReq then if self._objs[tReq.path] then local obj = self._objs[tReq.path] - local res = obj:call(tReq) - unistd.write(fd, res) + local res, keep = obj:call(tReq) + return res, keep else print("show all path.") for k, _ in pairs(self._objs) do print("path:", k) end - unistd.write(fd, self:echo404()) + return self:echo404(), false end end - collectgarbage("collect") end function Cframe:register(path, obj) diff --git a/source/tools/monitor/unity/beaver/guide/dev_proc.md b/source/tools/monitor/unity/beaver/guide/dev_proc.md index 17b627de17a37f2ec1ac49bd6c952b0feb988fa6..3c006d0dfb0e7bca223ec63064099788ffe118e1 100644 --- a/source/tools/monitor/unity/beaver/guide/dev_proc.md +++ b/source/tools/monitor/unity/beaver/guide/dev_proc.md @@ -251,7 +251,7 @@ Active(file): 522084 kB Inactive(file): 522000 kB …… ``` -对应处理代码说明,重点需要关注** readKV**函数实现。 +对应处理代码说明,重点需要关注**readKV**函数实现。 ``` local system = require("system") diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua new file mode 100644 index 0000000000000000000000000000000000000000..0775da44068362ec9758ce3a0d8651d60e03f5cd --- /dev/null +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -0,0 +1,84 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/10 4:07 PM +--- + +require("common.class") +local system = require("common.system") +local socket = require("socket") + +local Cidentity = class("identity") + +function Cidentity:_init_(fYaml) + fYaml = fYaml or "../collector/plugin.yaml" + local conf = system:parseYaml(fYaml) + self._opts = conf.config.identity + self._funcs = { + hostip = function() return self:hostip() end, + curl = function() return self:curl() end, + hostname = function() return self:hostname() end, + file = function() return self:file() end, + specify = function() return self:specify() end + } +end + +local function getAdd(hostName) + local _, resolved = socket.dns.toip(hostName) + local listTab = {} + for _, v in pairs(resolved.ip) do + table.insert(listTab, v) + end + return listTab +end + +function Cidentity:hostip() + local res = getAdd(socket.dns.gethostname()) + if type(res) == "table" and #res >= 1 then + return res[1] + else + return "None" + end +end + +function Cidentity:curl() + if self._opts.curl then + local ChttpCli = require("httplib.httpCli") + + local cli = ChttpCli.new() + local res = cli:get(self._opts.curl) + return res.body + else + return "None" + end +end + +function Cidentity:hostname() + return socket.dns.gethostname() +end + +function Cidentity:file() + if self._opts.path then + local file = io.open(self._opts.path, "r") + io.input(file) + local res = io.read() + io.close(file) + return res + else + return "None" + end +end + +function Cidentity:specify() + if self._opts.name then + return self._opts.name + else + return "None" + end +end + +function Cidentity:id() + return self._funcs[self._opts.mode]() +end + +return Cidentity \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/index.lua b/source/tools/monitor/unity/beaver/index.lua index 1ac711309a0dc8b0cc44f7706296c861f200f6d0..c3ce944ce09128543ae7765c8adc1984cbed0d5e 100644 --- a/source/tools/monitor/unity/beaver/index.lua +++ b/source/tools/monitor/unity/beaver/index.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/24 9:06 PM --- -require("class") +require("common.class") local unistd = require("posix.unistd") -local ChttpHtml = require("httpHtml") +local ChttpHtml = require("httplib.httpHtml") local CurlIndex = class("CurlIndex", ChttpHtml) diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua new file mode 100644 index 0000000000000000000000000000000000000000..b482c399efaba4587df75b4c3ba04f8af569b224 --- /dev/null +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -0,0 +1,217 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/14 3:14 PM +--- + +local unistd = require("posix.unistd") +local socket = require("posix.sys.socket") +local system = require("common.system") +require("common.class") + +local CLocalBeaver = class("poBeaver") + +function CLocalBeaver:_init_(frame, port, ip, backlog) + port = port or 8400 + + self._bfd = self:_install_fd(port, ip, backlog) + self._efd = self:_installFFI() + + self._cos = {} + self._last = os.time() + self._tmos = {} + + self._once = true + self._frame = frame +end + +function CLocalBeaver:_del_() + if self._efd then + self._cffi.deinit(self._efd) + end + if self._bfd then + unistd.close(self._bfd) + end +end + +local function posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + +function CLocalBeaver:_installTmo(fd) + self._tmos[fd] = os.time() +end + +function CLocalBeaver:_checkTmo() + local now = os.time() + if now - self._last >= 30 then + -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety + local tmos = system:dictCopy(self._tmos) + for fd, t in pairs(tmos) do + if now - t >= 60 then + local e = self._ffi.new("native_event_t") + e.ev_close = 1 + e.fd = fd + local co = self._cos[fd] + print("close " .. fd) + coroutine.resume(co, e) + end + end + self._last = now + end +end + +function CLocalBeaver:_installFFI() + local ffi = require("beaver.native.beavercffi") + + self._ffi = ffi.ffi + self._cffi = ffi.cffi + + local efd = self._cffi.init(self._bfd) + assert(efd > 0) + return efd +end + +function CLocalBeaver:_install_fd(port, ip, backlog) + ip = ip or "0.0.0.0" + backlog = backlog or 100 + + local fd, res, err, errno + fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + if fd then -- for socket + res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) + if res then -- for bind + res, err, errno = socket.listen(fd, backlog) + if res then -- for listen + return fd + else + posixError("socket listen failed", err, errno) + end + else -- for bind failed + unistd.close(fd) + posixError("socket bind failed", err, errno) + end + else -- socket failed + posixError("create socket failed", err, errno) + end +end + +function CLocalBeaver:read(fd, maxLen) + maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max + local function readFd() + local e = coroutine.yield() + if e.ev_close > 0 then + return nil + elseif e.ev_in > 0 then + local s, err, errno + s, err, errno = socket.recv(fd, maxLen) + if s then + if #s > 0 then + maxLen = maxLen - #s + return s + else + return nil + end + else + posixError("socket recv error", err, errno) + end + else + print(system:dump(e)) + end + return nil + end + return readFd +end + +function CLocalBeaver:_proc(fd) + local fread = self:read(fd) + while true do + local res, alive = self._frame:proc(fread) + if res then + socket.send(fd, res) + + if not alive then + self:co_exit(fd) + break + end + else + self:co_exit(fd) + break + end + end +end + +function CLocalBeaver:co_add(fd) + local res = self._cffi.add_fd(self._efd, fd) + assert(res >= 0) + + local co = coroutine.create(function(o, fd) self._proc(o, fd) end) + self._cos[fd] = co + local res, msg = coroutine.resume(co, self, fd) + assert(res, msg) +end + +function CLocalBeaver:co_exit(fd) + local res = self._cffi.del_fd(self._efd, fd) + assert(res >= 0) + + self._cos[fd] = nil + self._tmos[fd] = nil +end + +function CLocalBeaver:accept(fd, e) + if e.ev_close > 0 then + error("should close bind fd.") + else + local nfd, err, errno = socket.accept(fd) + if nfd then + self:co_add(nfd) + self:_installTmo(nfd) + else + posixError("accept new socket failed", err, errno) + end + end +end + +function CLocalBeaver:_poll(bfd, nes) + for i = 0, nes.num - 1 do + local e = nes.evs[i]; + local fd = e.fd + if fd == bfd then + self:accept(fd, e) + else + local co = self._cos[fd] + assert(co, string.format("fd: %d not setup.", fd)) + self:_installTmo(fd) + local res, msg = coroutine.resume(co, e) + assert(res, msg) + end + end + self:_checkTmo() +end + +function CLocalBeaver:poll() + assert(self._once, "poll loop only run once time.") + self._once = false + + local bfd = self._bfd + local efd = self._efd + while true do + local nes = self._ffi.new("native_events_t") + local res = self._cffi.poll_fds(efd, 10, nes) + + if res < 0 then + break + end + + self:_poll(bfd, nes) + end + + for fd in pairs(self._cos) do + local res = self._cffi.del_fd(self._efd, fd) + assert(res >= 0) + end +end + +return CLocalBeaver \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/main.c b/source/tools/monitor/unity/beaver/main.c deleted file mode 100644 index 42dd21ade1014aca577f55abfd7f164ee3363d32..0000000000000000000000000000000000000000 --- a/source/tools/monitor/unity/beaver/main.c +++ /dev/null @@ -1,19 +0,0 @@ -// -// Created by 廖肇燕 on 2022/12/20. -// - -#include -#include -#include "beaver.h" - -int main(int argc, char** argv) { - int port = 8400; - int thread = 3; - if (argc >= 3) { - char *ptr; - port = strtol(argv[1], &ptr, 10); - thread = strtol(argv[2], &ptr, 10); - } - beaver_init(port, thread); -} - diff --git a/source/tools/monitor/unity/beaver/make.bak b/source/tools/monitor/unity/beaver/make.bak deleted file mode 100644 index edee774c86013f8e5e7abcc285794e8b4ea383c4..0000000000000000000000000000000000000000 --- a/source/tools/monitor/unity/beaver/make.bak +++ /dev/null @@ -1,22 +0,0 @@ -CC := gcc -CFLAG := -g -LDFLAG := -g -lm -ldl -lpthread -lluajit-5.1 -OBJS := beaver.o main.o echos.o -EXEC := beaver - -all: $(EXEC) - -echos.o: echos.c beaver.h echos.h - $(CC) -c $< -o $@ $(CFLAG) - -beaver.o: beaver.c beaver.h echos.h - $(CC) -c $< -o $@ $(CFLAG) - -main.o: main.c beaver.h - $(CC) -c $< -o $@ $(CFLAG) - -$(EXEC): $(OBJS) - $(CC) -o $@ $(OBJS) $(LDFLAG) - -clean: - rm -f $(EXEC) $(OBJS) diff --git a/source/tools/monitor/unity/beaver/native/Makefile b/source/tools/monitor/unity/beaver/native/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..583c418ad408c5e7ed0d7ad46d6625b3132c74a8 --- /dev/null +++ b/source/tools/monitor/unity/beaver/native/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := local_beaver.o +SO := liblbeaver.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../collector/native/ + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/native/beavercffi.lua b/source/tools/monitor/unity/beaver/native/beavercffi.lua new file mode 100644 index 0000000000000000000000000000000000000000..b8b7fa043c977d8aea210b4f5c099a484e23d82a --- /dev/null +++ b/source/tools/monitor/unity/beaver/native/beavercffi.lua @@ -0,0 +1,28 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/14 2:59 PM +--- + +local ffi = require("ffi") +local cffi = ffi.load('lbeaver') +ffi.cdef [[ +typedef struct native_event { + int fd; + short int ev_in; + short int ev_close; +}native_event_t; + +typedef struct native_events { + int num; + native_event_t evs[64]; +}native_events_t; + +int init(int listen_fd); +int add_fd(int efd, int fd); +int del_fd(int efd, int fd); +int poll_fds(int efd, int tmo, native_events_t* nes); +void deinit(int efd); +]] + +return {ffi = ffi, cffi=cffi} diff --git a/source/tools/monitor/unity/beaver/native/local_beaver.c b/source/tools/monitor/unity/beaver/native/local_beaver.c new file mode 100644 index 0000000000000000000000000000000000000000..4d8f56ed31bee80721314239a7096e2d25879ada --- /dev/null +++ b/source/tools/monitor/unity/beaver/native/local_beaver.c @@ -0,0 +1,132 @@ +// +// Created by 廖肇燕 on 2023/2/14. +// + +#include "local_beaver.h" +#include +#include +#include +#include +#include +#include +#include + +static int socket_non_blocking(int sfd) +{ + int flags, res; + + flags = fcntl(sfd, F_GETFL); + if (flags < 0) { + perror("error : cannot get socket flags!\n"); + return flags; + } + + flags |= O_NONBLOCK; + res = fcntl(sfd, F_SETFL, flags); + if (res < 0) { + perror("error : cannot set socket flags!\n"); + return res; + } + + return 0; +} + +static int epoll_add(int efd, int fd) { + struct epoll_event event; + int res; + + event.events = EPOLLIN; + event.data.fd = fd; + + res = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &event); + if (res < 0) { + perror("error : can not add event to epoll!\n"); + } + return res; +} + +static int epoll_del(int efd, int fd) { + int res; + + res = epoll_ctl(efd, EPOLL_CTL_DEL, fd, NULL); + if (res < 0) { + perror("error : can not del event to epoll!\n"); + } + return res; +} + +int init(int listen_fd) { + int efd; + int res; + + efd = epoll_create(NATIVE_EVENT_MAX); + if (efd < 0) { + perror("error : cannot create epoll!\n"); + exit(1); + } + + res = epoll_add(efd, listen_fd); + if (res < 0) { + goto end_epoll_add; + } + return efd; + + end_epoll_add: + return res; +} + +int add_fd(int efd, int fd) { + int res; + + res = socket_non_blocking(fd); + if (res < 0) { + goto end_socket_non_blocking; + } + + res = epoll_add(efd, fd); + if (res < 0) { + goto end_epoll_add; + } + return res; + + end_socket_non_blocking: + end_epoll_add: + return res; +} + +int del_fd(int efd, int fd) { + int res; + res = epoll_del(efd, fd); + + close(fd); + return res; +} + +int poll_fds(int efd, int tmo, native_events_t* nes) { + struct epoll_event events[NATIVE_EVENT_MAX]; + int i, res; + + res = epoll_wait(efd, events, NATIVE_EVENT_MAX, tmo * 1000); + if (res < 0) { + perror("error : epoll failed!\n"); + return res; + } + nes->num = res; + for (i = 0; i < res; i ++) { + nes->evs[i].fd = events[i].data.fd; + + if ( (events[i].events & EPOLLERR) || + (events[i].events & EPOLLHUP) || + !(events[i].events & EPOLLIN) ) { + nes->evs[i].ev_close = 1; + } + if (events[i].events & EPOLLIN) { + nes->evs[i].ev_in = 1; + } + } + return 0; +} + +void deinit(int efd) { + close(efd); +} diff --git a/source/tools/monitor/unity/beaver/native/local_beaver.h b/source/tools/monitor/unity/beaver/native/local_beaver.h new file mode 100644 index 0000000000000000000000000000000000000000..39410f8c5ce858117d306ba312d7f1b3571c0546 --- /dev/null +++ b/source/tools/monitor/unity/beaver/native/local_beaver.h @@ -0,0 +1,21 @@ +// +// Created by 廖肇燕 on 2023/2/14. +// + +#ifndef UNITY_LOCAL_BEAVER_H +#define UNITY_LOCAL_BEAVER_H + +#define NATIVE_EVENT_MAX 64 + +typedef struct native_event { + int fd; + short int ev_in; + short int ev_close; +}native_event_t; + +typedef struct native_events { + int num; + native_event_t evs[NATIVE_EVENT_MAX]; +}native_events_t; + +#endif //UNITY_LOCAL_BEAVER_H diff --git a/source/tools/monitor/unity/beaver/url_api.lua b/source/tools/monitor/unity/beaver/url_api.lua index 1d307bdeeb3dcb0d6e3973032fc2226395f1224f..4eab10dd5bba1622c3d1314fcdbd16fae2abd7d0 100644 --- a/source/tools/monitor/unity/beaver/url_api.lua +++ b/source/tools/monitor/unity/beaver/url_api.lua @@ -4,10 +4,10 @@ --- DateTime: 2022/12/22 12:08 PM --- -require("class") -local system = require("system") -local ChttpApp = require("httpApp") -local CfoxTSDB = require("foxTSDB") +require("common.class") +local system = require("common.system") +local ChttpApp = require("httplib.httpApp") +local CfoxTSDB = require("tsdb.foxTSDB") local CurlApi = class("urlApi", ChttpApp) function CurlApi:_init_(frame) @@ -31,7 +31,7 @@ end function CurlApi:sub(tReq) local stat, tJson = pcall(self.getJson, self, tReq) if stat then - return {sum=tJson.num1 - tJson.num2} + return {sub=tJson.num1 - tJson.num2} else return {} end diff --git a/source/tools/monitor/unity/beaver/url_export_html.lua b/source/tools/monitor/unity/beaver/url_export_html.lua index 5b46b0bdcc49460808d9f54f08ff1e773e90cf7e..bc3b953d0f71f9773ab5464c1f5f54c5cde4d949 100644 --- a/source/tools/monitor/unity/beaver/url_export_html.lua +++ b/source/tools/monitor/unity/beaver/url_export_html.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/25 11:19 AM --- -local pystring = require("pystring") -require("class") -local ChttpHtml = require("httpHtml") +local pystring = require("common.pystring") +require("common.class") +local ChttpHtml = require("httplib.httpHtml") local CurlExportHtml = class("CurlExportHtml", ChttpHtml) diff --git a/source/tools/monitor/unity/beaver/url_export_raw.lua b/source/tools/monitor/unity/beaver/url_export_raw.lua index d8147be280f5ace3ec85871c17b96a115722e6a3..b6c904bf177105d9f26c650d81090f0133d0348f 100644 --- a/source/tools/monitor/unity/beaver/url_export_raw.lua +++ b/source/tools/monitor/unity/beaver/url_export_raw.lua @@ -4,8 +4,8 @@ --- DateTime: 2022/12/25 11:20 AM --- -require("class") -local ChttpPlain = require("httpPlain") +require("common.class") +local ChttpPlain = require("httplib.httpPlain") local CurlExportRaw = class("CurlExportRaw", ChttpPlain) diff --git a/source/tools/monitor/unity/beaver/url_guide.lua b/source/tools/monitor/unity/beaver/url_guide.lua index 5cd838d98579482e8f087dcc77d48c0fe7476827..1362b01e3c9530432e360dcadbee6c942fe7330a 100644 --- a/source/tools/monitor/unity/beaver/url_guide.lua +++ b/source/tools/monitor/unity/beaver/url_guide.lua @@ -4,8 +4,8 @@ --- DateTime: 2023/1/2 12:05 PM --- -require("class") -local ChttpHtml = require("httpHtml") +require("common.class") +local ChttpHtml = require("httplib.httpHtml") local CurlGuide = class("CurlIndex", ChttpHtml) diff --git a/source/tools/monitor/unity/beaver/url_rpc.lua b/source/tools/monitor/unity/beaver/url_rpc.lua index 2fd784295f912225f8528771ce94e7d8655d0a0b..7fbb8aa1dab52847da5791acf360faa1cccbac4f 100644 --- a/source/tools/monitor/unity/beaver/url_rpc.lua +++ b/source/tools/monitor/unity/beaver/url_rpc.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/29 10:17 PM --- -- refer to https://www.cnblogs.com/cielosun/p/6762550.html -require("class") -local ChttpApp = require("httpApp") -local system = require("system") +require("common.class") +local ChttpApp = require("httplib.httpApp") +local system = require("common.system") local CurlRpc = class("urlRpc", ChttpApp) diff --git a/source/tools/monitor/unity/beeQ/Makefile b/source/tools/monitor/unity/beeQ/Makefile index 71bc614c1461ee48d8a5f1e4a2cba92412774083..4f1feee7ea0b16b677c5e3f3e6596fa392b20869 100644 --- a/source/tools/monitor/unity/beeQ/Makefile +++ b/source/tools/monitor/unity/beeQ/Makefile @@ -25,3 +25,6 @@ clean: @echo "Removing linked and compiled files......" $(foreach N, $(DEPMOD),make clean -C $(N);) rm -f $(OBJ) $(PRG) + +dist:$(PRG) + ./pack.sh \ No newline at end of file diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index 0ff7aca369c67a6f3c8533bbd70d028d8bbd5074..27243841a7c80e1e5aef2f67b78514a534907f13 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -12,6 +12,7 @@ #include #define gettidv1() syscall(__NR_gettid) +static int sample_period = 0; LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); @@ -217,7 +218,7 @@ static lua_State * app_collector_init(void* q, void* proto_q, int delta) { lua_pushlightuserdata(L, proto_q); lua_pushinteger(L, delta); ret = lua_pcall(L, 3, 1, 0); - if (ret) { + if (ret < 0) { perror("luaL_call init func error"); report_lua_failed(L); goto endCall; @@ -236,6 +237,8 @@ static lua_State * app_collector_init(void* q, void* proto_q, int delta) { perror("collectors.lua init failed."); goto endReturn; } + sample_period = lret; + printf("setup sample period %ds\n", sample_period); return L; endReturn: @@ -265,7 +268,7 @@ static int app_collector_work(lua_State **pL, void* q, void* proto_q, int delta) } lua_getglobal(L, "work"); - lua_pushinteger(L, 15); + lua_pushinteger(L, sample_period); ret = lua_pcall(L, 1, 1, 0); if (ret) { perror("luaL_call init func error"); @@ -296,7 +299,6 @@ static int app_collector_work(lua_State **pL, void* q, void* proto_q, int delta) #include #include typedef long bee_time_t; -#define APP_LOOP_PERIOD 15 static bee_time_t local_time(void) { int ret; struct timespec tp; @@ -317,7 +319,7 @@ int app_collector_run(struct beeQ* q, void* arg) { lua_State **pL; struct beeQ* proto_que = (struct beeQ* )arg; - L = app_collector_init(q, proto_que, APP_LOOP_PERIOD); + L = app_collector_init(q, proto_que, sample_period); if (L == NULL) { ret = -1; goto endInit; @@ -327,13 +329,13 @@ int app_collector_run(struct beeQ* q, void* arg) { while (1) { bee_time_t t1, t2, delta; t1 = local_time(); - ret = app_collector_work(pL, q, proto_que, APP_LOOP_PERIOD); + ret = app_collector_work(pL, q, proto_que, sample_period); if (ret < 0) { goto endLoop; } t2 = local_time(); - delta = t1 + APP_LOOP_PERIOD * 1000000 - t2; + delta = t1 + sample_period * 1000000 - t2; if (delta > 0) { usleep(delta); diff --git a/source/tools/monitor/unity/beeQ/bees.c b/source/tools/monitor/unity/beeQ/bees.c index 6e2904a1f4972f7ae17de9c1c49c8a2b159a8009..091bf37b7e9eb71dda38eb409b1dde8162a71696 100644 --- a/source/tools/monitor/unity/beeQ/bees.c +++ b/source/tools/monitor/unity/beeQ/bees.c @@ -46,7 +46,7 @@ int main(int argc, char *argv[]) { } beeQ_send_thread(q, proto_que, app_collector_run); - beaver_init(8400, 3); + beaver_init(8400, 64); pause(); fprintf(stderr, "test exit."); beeQ_stop(q); diff --git a/source/tools/monitor/unity/beeQ/bees.lua b/source/tools/monitor/unity/beeQ/bees.lua index eea0a53662261e29feffc180aaf0613f2308ba60..2c2e9353db7ca33c06285f87e64100c7131a0d05 100644 --- a/source/tools/monitor/unity/beeQ/bees.lua +++ b/source/tools/monitor/unity/beeQ/bees.lua @@ -4,11 +4,9 @@ --- DateTime: 2022/12/26 4:02 PM --- -package.path = package.path .. ";../common/?.lua;" -package.path = package.path .. ";../tsdb/?.lua;" -package.path = package.path .. ";../tsdb/native/?.lua;" +package.path = package.path .. ";../?.lua;" -local CfoxRecv = require("foxRecv") +local CfoxRecv = require("beeQ.foxRecv") local unistd = require("posix.unistd") --local proto = require("protoData") --local system = require("system") diff --git a/source/tools/monitor/unity/beeQ/collectors.lua b/source/tools/monitor/unity/beeQ/collectors.lua index 7d4f649cc58b9b89d818664356fb5fffad07449c..e42b0a1c0ed238ef8a6de15d2a4d984d3963c80d 100644 --- a/source/tools/monitor/unity/beeQ/collectors.lua +++ b/source/tools/monitor/unity/beeQ/collectors.lua @@ -3,19 +3,33 @@ --- Created by liaozhaoyan. --- DateTime: 2022/12/26 11:26 PM --- -package.path = package.path .. ";../common/?.lua;" -package.path = package.path .. ";../collector/?.lua;" -package.path = package.path .. ";../collector/native/?.lua;" +package.path = package.path .. ";../?.lua;" -local Cloop = require("loop") -local system = require("system") +local Cloop = require("collector.loop") +local system = require("common.system") workLoop = nil +local function setupFreq(fYaml) + local conf = system:parseYaml(fYaml) + if conf then + local ret = tonumber(conf.config.freq) + if ret > 5 then + return conf.config.freq + else + return 5 + end + else + error("load yaml file failed.") + return -1 + end +end + function init(que, proto_q, t) - local work = Cloop.new(que, proto_q) + local fYaml = "../collector/plugin.yaml" + local work = Cloop.new(que, proto_q, fYaml) workLoop = work - return 0 + return setupFreq(fYaml) end function work(t) diff --git a/source/tools/monitor/unity/beeQ/foxRecv.lua b/source/tools/monitor/unity/beeQ/foxRecv.lua index a77b9dcd3f671536bd07a5e42c736944176a919a..dfb52a3964e429e2e9543629de4e8ad67fea253c 100644 --- a/source/tools/monitor/unity/beeQ/foxRecv.lua +++ b/source/tools/monitor/unity/beeQ/foxRecv.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/26 3:57 PM --- -require("class") +require("common.class") -local CfoxTSDB = require("foxTSDB") +local CfoxTSDB = require("tsdb.foxTSDB") local CfoxRecv = class("CfoxRecv") diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh new file mode 100755 index 0000000000000000000000000000000000000000..b2d311023fa5e5273e3f6e17133b41a72d3271eb --- /dev/null +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -0,0 +1,53 @@ +#!/bin/bash +DIST=dist +APP=${DIST}/app + +echo $DIST +echo $APP +cd ../ +rm -rf $DIST +mkdir $DIST + +mkdir ${DIST}/install +cp /usr/local/lib/libyaml-0.so* ${DIST}/install/ +cp /usr/local/lib/libluajit-5.1.so* ${DIST}/install/ +cp /usr/local/lib/libyaml.so* ${DIST}/install/ + +mkdir ${DIST}/lib +cp -r /usr/local/lib/lua/5.1/* ${DIST}/lib/ + +mkdir ${DIST}/lua +cp -r /usr/local/share/lua/5.1/* ${DIST}/lua/ + +mkdir ${APP} +mkdir ${APP}/beaver +cp -r beaver/guide ${APP}/beaver/ +cp beaver/*.lua ${APP}/beaver/ + +mkdir ${APP}/beeQ/ +mkdir ${APP}/beeQ/lib +cp beeQ/lib/*.so* ${APP}/beeQ/lib/ +cp beeQ/*.lua ${APP}/beeQ/ +cp beeQ/unity-mon ${APP}/beeQ/ +cp beeQ/run.sh ${APP}/beeQ/ + +mkdir ${APP}/collector +mkdir ${APP}/collector/native +cp collector/native/*.so* ${APP}/collector/native/ +cp collector/native/*.lua ${APP}/collector/native/ +cp collector/*.lua ${APP}/collector/ +cp collector/plugin.yaml ${APP}/collector/ + +mkdir ${APP}/common +cp common/*.lua ${APP}/common/ + +mkdir ${APP}/httplib +cp httplib/*.lua ${APP}/httplib/ + +mkdir ${APP}/tsdb +mkdir ${APP}/tsdb/native +cp tsdb/native/*.so* ${APP}/tsdb/native/ +cp tsdb/native/*.lua ${APP}/tsdb/native/ +cp tsdb/*.lua ${APP}/tsdb + +tar zcv -f dist.tar.gz dist/ diff --git a/source/tools/monitor/unity/beeQ/proto_queue.lua b/source/tools/monitor/unity/beeQ/proto_queue.lua index 9dc82efae8bfad9bd921fdabd87197ba05bb08ca..f1988d89f251ea5a18d6e96a06de31248f91ac7a 100644 --- a/source/tools/monitor/unity/beeQ/proto_queue.lua +++ b/source/tools/monitor/unity/beeQ/proto_queue.lua @@ -4,15 +4,15 @@ --- DateTime: 2023/1/12 3:28 PM --- -require("class") +require("common.class") -local CprotoData = require("protoData") +local CprotoData = require("common.protoData") local CprotoQueue = class("loop") -local system = require("system") +local system = require("common.system") function CprotoQueue:_init_(que) self._proto = CprotoData.new(que) - self._ffi = require("plugincffi") + self._ffi = require("collector.native.plugincffi") self._que = que end diff --git a/source/tools/monitor/unity/beeQ/proto_send.lua b/source/tools/monitor/unity/beeQ/proto_send.lua index 5288aab2f14a9e854f891aa8a62c1d570bc57673..892cce620409119789366757d45d75e6f2dab518 100644 --- a/source/tools/monitor/unity/beeQ/proto_send.lua +++ b/source/tools/monitor/unity/beeQ/proto_send.lua @@ -4,10 +4,9 @@ --- DateTime: 2023/1/12 2:53 PM --- -package.path = package.path .. ";../common/?.lua;" -package.path = package.path .. ";../collector/native/?.lua;" +package.path = package.path .. ";../?.lua;" -local CprotoQueue = require("proto_queue") +local CprotoQueue = require("beeQ.proto_queue") local workLoop = nil diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..8be5ea21e2711de6015811b8f17cf4ebefa662a4 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../collector/native/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../beaver/ + +if [ ! -f "/lib64/libyaml-0.so.2" ]; then + echo "install libyaml." + cp ../../install/libyaml-0.so* /lib64 + cp ../../install/libyaml.so* /lib64 + cp ../../install/libluajit-5.1.so* /lib64 +fi + +if [ ! -d "/usr/local/lib/lua/5.1/" ]; then + echo "install libs." + mkdir -p /usr/local/lib/lua/5.1/ + cp -r ../../lib/* /usr/local/lib/lua/5.1/ +fi + +if [ ! -d "/usr/local/share/lua/5.1/" ]; then + echo "install luas." + mkdir -p /usr/local/share/lua/5.1/ + cp -r ../../lua/* /usr/local/share/lua/5.1/ +fi + +./unity-mon diff --git a/source/tools/monitor/unity/collector/kvProc.lua b/source/tools/monitor/unity/collector/kvProc.lua index e60d8e31c1cd437a271ffc0b540c94641ebe9f9f..6f3bc3e4dfd462bb62a53ad89b2682be14e5c88b 100644 --- a/source/tools/monitor/unity/collector/kvProc.lua +++ b/source/tools/monitor/unity/collector/kvProc.lua @@ -4,14 +4,14 @@ --- DateTime: 2022/12/16 11:30 PM --- -local system = require("system") -require("class") -local CvProc = require("vproc") +local system = require("common.system") +require("common.class") +local CvProc = require("collector.vproc") local CkvProc = class("kvProc", CvProc) -function CkvProc:_init_(proto, pffi, pFile, tName) - CvProc._init_(self, proto, pffi, pFile) +function CkvProc:_init_(proto, pffi, mnt, pFile, tName) + CvProc._init_(self, proto, pffi, mnt, pFile) self._protoTable = { line = tName, ls = nil, diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 069829aea82eef2b9d4a2b8a1073776ad52b99c9..0121ec01981cb9037e13afce895d1de86c16e7ab 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -4,37 +4,41 @@ --- DateTime: 2022/12/16 10:39 PM --- -require("class") -local CprotoData = require("protoData") -local procffi = require("procffi") +require("common.class") +local CprotoData = require("common.protoData") +local procffi = require("collector.native.procffi") -local CprocStat = require("proc_stat") -local CprocMeminfo = require("proc_meminfo") -local CprocVmstat = require("proc_vmstat") -local CprocNetdev = require("proc_netdev") -local CprocDiskstats = require("proc_diskstats") -local CprocSockStat = require("proc_sockstat") -local CprocSnmpStat = require("proc_snmp_stat") -local CprocMounts = require("proc_mounts") -local CprocStatm = require("proc_statm") -local Cplugin = require("plugin") +local CprocStat = require("collector.proc_stat") +local CprocMeminfo = require("collector.proc_meminfo") +local CprocVmstat = require("collector.proc_vmstat") +local CprocNetdev = require("collector.proc_netdev") +local CprocDiskstats = require("collector.proc_diskstats") +local CprocSockStat = require("collector.proc_sockstat") +local CprocSnmpStat = require("collector.proc_snmp_stat") +local CprocMounts = require("collector.proc_mounts") +local CprocStatm = require("collector.proc_statm") +local Cplugin = require("collector.plugin") + +local system = require("common.system") local Cloop = class("loop") -function Cloop:_init_(que, proto_q) +function Cloop:_init_(que, proto_q, fYaml) + local res = system:parseYaml(fYaml) self._proto = CprotoData.new(que) self._procs = { - CprocStat.new(self._proto, procffi), - CprocMeminfo.new(self._proto, procffi), - CprocVmstat.new(self._proto, procffi), - CprocNetdev.new(self._proto, procffi), - CprocDiskstats.new(self._proto, procffi), - CprocSockStat.new(self._proto, procffi), - CprocSnmpStat.new(self._proto, procffi), - CprocMounts.new(self._proto, procffi), - CprocStatm.new(self._proto, procffi), + CprocStat.new(self._proto, procffi, res.config.proc_path), + CprocMeminfo.new(self._proto, procffi, res.config.proc_path), + CprocVmstat.new(self._proto, procffi, res.config.proc_path), + CprocNetdev.new(self._proto, procffi, res.config.proc_path), + CprocDiskstats.new(self._proto, procffi, res.config.proc_path), + CprocSockStat.new(self._proto, procffi, res.config.proc_path), + CprocSnmpStat.new(self._proto, procffi, res.config.proc_path), + CprocMounts.new(self._proto, procffi, res.config.proc_path), + CprocStatm.new(self._proto, procffi, res.config.proc_path), } - self._plugin = Cplugin.new(self._proto, procffi, que, proto_q) + print("proc new stop.") + self._plugin = Cplugin.new(self._proto, procffi, que, proto_q, fYaml) end function Cloop:work(t) diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 20b1511343cf7687dab151d971e3ac36e8c3aeb0..107946fbfa7cdd84be3944489f75a0d4c471489a 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -4,17 +4,17 @@ --- DateTime: 2022/12/30 10:20 AM --- -require("class") - +require("common.class") +local system = require("common.system") local Cplugin = class("plugin") function Cplugin:_init_(proto, procffi, que, proto_q, fYaml) self._proto = proto - fYaml = fYaml or "../collector/plugin.yaml" - self._ffi = require("plugincffi") + local res = system:parseYaml(fYaml) + + self._ffi = require("collector.native.plugincffi") self._sig_cffi = procffi["cffi"] - local res = self:loadYaml(fYaml) self._sig_cffi.plugin_init() self:setup(res.plugins, proto_q) end @@ -27,20 +27,12 @@ function Cplugin:_del_() end end -function Cplugin:loadYaml(fYaml) - local lyaml = require("lyaml") - local f = io.open(fYaml,"r") - local s = f:read("*all") - f:close() - - return lyaml.load(s) -end - function Cplugin:setup(plugins, proto_q) self._plugins = {} for _, plugin in ipairs(plugins) do local so = plugin.so if so then + print(so) local cffi = self._ffi.load(so) local plugin = { so = plugin.so, diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 2b935c7daec7dff539b12b29fc15e078606f2d99..727afd15cf28003b3e6a030b00c2b53e7450789b 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -1,3 +1,12 @@ +config: + freq: 20 # unit second + identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg) + mode: specify + name: test_specify +# mode: hostip + proc_path: /mnt/home/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ +# proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ + plugins: - so: kmsg description: "collect dmesg info." @@ -7,10 +16,9 @@ plugins: - so: sample_threads description: "threads example." - - - - so: bpfsample2 - description: "bpf threads example." +# - +# so: bpfsample2 +# description: "bpf threads example." - so: proc_schedstat diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h new file mode 100644 index 0000000000000000000000000000000000000000..6acf256f68c6902a7ea1604f3df1d4a5586788ad --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -0,0 +1,60 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +#ifndef UNITY_BPF_HEAD_H +#define UNITY_BPF_HEAD_H + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name) + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + if (!perf_args) \ + { \ + __ret = -ENOMEM; \ + printf("failed to allocate memory: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + perf_args->mapfd = bpf_map__fd(skel_name->maps.perf); \ + perf_args->sample_cb = handle_event; \ + perf_args->lost_cb = handle_lost_events; \ + perf_args->ctx = arg; \ + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); \ + load_bpf_skel_out: \ + __ret; \ + }) + +#endif //UNITY_BPF_HEAD_H diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/Makefile b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..82662e5131730820a689b6767ee18fc2af6ecb23 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := cpudist.bpf.c +csrcs := cpudist.c +target := cpudist.so + +include $(SRC)/mk/bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..e4f92481842be657ee8b647ec2d9405e592ede31 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c @@ -0,0 +1,34 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +struct sched_switch_args { + u16 type; + u8 flag; + u8 preeempt; + u32 c_pid; + char prev_comm[16]; + u32 prev_pid; + u32 prev_prio; + u64 prev_state; + char next_comm[16]; + u32 next_pid; + u32 next_prio; +}; +SEC("tracepoint/sched/sched_switch") +int sched_switch_hook(struct sched_switch_args *args){ + u64 ts = bpf_ktime_get_ns(); + u64 *pv; + u32 prev = args->prev_pid; + u32 next = args->next_pid; + + if (next > 0) { + bpf_map_update_elem(&start, &next, &ts, BPF_ANY); + } + pv = bpf_map_lookup_elem(&start, &prev); + if (pv && ts > *pv) { + hist10_push(&cpudist, (ts - *pv) / 1000); + } +} +char _license[] SEC("license") = "GPL"; + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c new file mode 100644 index 0000000000000000000000000000000000000000..625f510722bac41aad995b38e7a8bbe95eb33e33 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c @@ -0,0 +1,5 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +#include "cpudist.h" diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h new file mode 100644 index 0000000000000000000000000000000000000000..88782b99b8026105b0980374e3ff8e71353d4f2c --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +#ifndef UNITY_CPUDIST_H +#define UNITY_CPUDIST_H + +#endif //UNITY_CPUDIST_H diff --git a/source/tools/monitor/unity/collector/plugin/virtout/Makefile b/source/tools/monitor/unity/collector/plugin/virtout/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..bb7e3544dc9623594db420e5c3f40519024be05b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := virtout.bpf.c +csrcs := virtout.c +target := virtout.so + +include $(SRC)/mk/bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c b/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..8bdc14ab6df5627dd782e54c5331d1fb6e475985 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c @@ -0,0 +1,156 @@ +#include +#include +#include "virtout.h" + +static inline int fast_log2(long value) +{ + int n = 0; + int i; + + if (value < 1) { + goto end; + } + +#pragma unroll + for (i = 32; i > 0; i /= 2) { + long v = 1ULL << i; + if (value >= v) { + n += i; + value = value >> i; + } + } + end: + return n; +} + +#define NUM_E16 10000000000000000ULL +#define NUM_E8 100000000ULL +#define NUM_E4 10000ULL +#define NUM_E2 100ULL +#define NUM_E1 10ULL +static inline int fast_log10(long v) +{ + int n = 0; + if (v >= NUM_E16) {n += 16; v /= NUM_E16;} + if (v >= NUM_E8) {n += 8; v /= NUM_E8;} + if (v >= NUM_E4) {n += 4; v /= NUM_E4;} + if (v >= NUM_E2) {n += 2; v /= NUM_E2;} + if (v >= NUM_E1) {n += 1;} + return n; +} + +static inline void add_hist(struct bpf_map_def* maps, int k, int v) { + u64 *pv = bpf_map_lookup_elem(maps, &k); + if (pv) { + __sync_fetch_and_add(pv, v); + } +} + +#define incr_hist(maps, k) add_hist(maps, k, 1) + +static inline void hist2_push(struct bpf_map_def* maps, long v) { + int k = fast_log2(v); + incr_hist(maps, k); +} + +static inline void hist10_push(struct bpf_map_def* maps, long v) { + int k = fast_log10(v); + incr_hist(maps, k); +} + +#define LBC_HIST10(MAPS) \ + struct bpf_map_def SEC("maps") MAPS = { \ + .type = BPF_MAP_TYPE_ARRAY, \ + .key_size = sizeof(int), \ + .value_size = sizeof(long), \ + .max_entries = 20, \ + } + +#define PERIOD_TIME (10 * 1000 * 1000ULL) +#define THRESHOLD_TIME (200 * 1000 * 1000ULL) + +struct bpf_map_def SEC("maps") virtHist = { \ + .type = BPF_MAP_TYPE_ARRAY, \ + .key_size = sizeof(int), \ + .value_size = sizeof(long), \ + .max_entries = 20, \ + }; + +struct bpf_map_def SEC("maps") e_sw = { \ + .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, \ + .key_size = sizeof(int), \ + .value_size = sizeof(struct data_t), \ + .max_entries = 16, \ + +}; +struct bpf_map_def SEC("maps") call_stack = { \ + .type = BPF_MAP_TYPE_STACK_TRACE, \ + .key_size = sizeof(u32), \ + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), \ + .max_entries = 1024, \ + }; + +struct bpf_map_def SEC("maps") perRec = { \ + .type = BPF_MAP_TYPE_PERCPU_ARRAY, \ + .key_size = sizeof(int), \ + .value_size = sizeof(u64), \ + .max_entries = 2, \ + }; + +static void store_con(char* con, struct task_struct *p) +{ + struct cgroup_name *cname; + cname = BPF_CORE_READ(p, cgroups, subsys[0], cgroup, name); + if (cname != NULL) { + bpf_core_read(con, CON_NAME_LEN, &cname->name[0]); + } else { + con[0] = '\0'; + } +} + +static inline u64 get_last(int index) { + u64 *pv = bpf_map_lookup_elem(&perRec, &index); + if (pv) { + return *pv; + } + return 0; +} + +static inline void save_last(int index, u64 ns) { + bpf_map_update_elem(&perRec, &index, &ns, BPF_ANY); +} + +static inline void check_time(struct bpf_perf_event_data *ctx, + int index, + struct bpf_map_def* event) { + u64 ns = bpf_ktime_get_ns(); + u64 last = get_last(index); + s64 delta; + + save_last(index, ns); + delta = ns - last; + + if (last && delta > 2 * PERIOD_TIME) { + delta -= PERIOD_TIME; + hist10_push(&virtHist, delta / PERIOD_TIME); + if (delta >= THRESHOLD_TIME) { + struct task_struct* task = bpf_get_current_task(); + struct data_t data = {}; + + data.pid = bpf_get_current_pid_tgid() >> 32; + data.stack_id = bpf_get_stackid(ctx, &call_stack, KERN_STACKID_FLAGS); + data.delta = delta; + bpf_get_current_comm(&data.comm, TASK_COMM_LEN); + store_con(&data.con[0], task); + + bpf_perf_event_output(ctx, event, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + } +} + +SEC("perf_event") +int sw_clock(struct bpf_perf_event_data *ctx) +{ + check_time(ctx, 0, &e_sw); + return 0; +} \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.c b/source/tools/monitor/unity/collector/plugin/virtout/virtout.c new file mode 100644 index 0000000000000000000000000000000000000000..0596578e564591096480f80d122122da8c774c08 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.c @@ -0,0 +1,49 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +#include "virtout.h" +#include "virtout.skel.h" + +#include +#include + +DEFINE_SEKL_OBJECT(virtout); + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event *e = (struct event *)data; + struct unity_line *line; + struct unity_lines *lines = unity_new_lines(); + struct beeQ *q = (struct beeQ *)ctx; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample2"); + unity_set_index(line, 0, "index", "value"); + unity_set_value(line, 0, "ns", e->ns); + unity_set_value(line, 1, "cpu", e->cpu); + beeQ_send(q, lines); +} + +void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) +{ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); +} + +int init(void *arg) +{ + printf("bpfsample2 plugin install.\n"); + return LOAD_SKEL_OBJECT(virtout, e_sw); +} + +int call(int t, struct unity_lines *lines) +{ + +} + +void deinit(void) +{ + DESTORY_SKEL_BOJECT(virtout); +} + diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.h b/source/tools/monitor/unity/collector/plugin/virtout/virtout.h new file mode 100644 index 0000000000000000000000000000000000000000..0c966df0621cf97e4f264f17dc6ece44ae8805dd --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.h @@ -0,0 +1,25 @@ +// +// Created by 廖肇燕 on 2023/2/7. +// + +#ifndef UNITY_VIRTOUT_H +#define UNITY_VIRTOUT_H + +#define TASK_COMM_LEN 16 +#define CON_NAME_LEN 80 +struct data_t { + int pid; + u32 stack_id; + u64 delta; + char comm[TASK_COMM_LEN]; + char con[CON_NAME_LEN]; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" +#include "../bpf_head.h" + +#endif + +#endif //UNITY_VIRTOUT_H diff --git a/source/tools/monitor/unity/collector/proc_diskstats.lua b/source/tools/monitor/unity/collector/proc_diskstats.lua index 9465231ca801fc3b19d28dc0dedc46b700c59062..45badf6b35b2847a7f6d108738d2b856f6525343 100644 --- a/source/tools/monitor/unity/collector/proc_diskstats.lua +++ b/source/tools/monitor/unity/collector/proc_diskstats.lua @@ -4,14 +4,14 @@ --- DateTime: 2022/12/16 11:49 PM --- -require("class") -local system = require("system") -local CvProc = require("vproc") +require("common.class") +local system = require("common.system") +local CvProc = require("collector.vproc") local CprocDiskstats = class("proc_diskstats", CvProc) -function CprocDiskstats:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/diskstats") +function CprocDiskstats:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/diskstats") self._lastData = {} self._lastDisk = {} self._diskVNum = 11 diff --git a/source/tools/monitor/unity/collector/proc_meminfo.lua b/source/tools/monitor/unity/collector/proc_meminfo.lua index 51d34785efe99bfda799e867e60817f135e8161a..ab89cfa18ede52cb24d9685d01c117ea5913bce4 100644 --- a/source/tools/monitor/unity/collector/proc_meminfo.lua +++ b/source/tools/monitor/unity/collector/proc_meminfo.lua @@ -4,13 +4,13 @@ --- DateTime: 2022/12/16 11:36 PM --- -require("class") -local CkvProc = require("kvProc") +require("common.class") +local CkvProc = require("collector.kvProc") local CprocMeminfo = class("proc_meminfo", CkvProc) -function CprocMeminfo:_init_(proto, pffi, pFile) - CkvProc._init_(self, proto, pffi, pFile or "/proc/meminfo", "meminfo") +function CprocMeminfo:_init_(proto, pffi, mnt, pFile) + CkvProc._init_(self, proto, pffi, mnt,pFile or "proc/meminfo", "meminfo") end return CprocMeminfo \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 3bef202ff70e24452ddffc2613b6cd163cc10935..4de16c327bcba74a277713bff7996ea95b40c3e0 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -4,16 +4,16 @@ --- DateTime: 2023/1/29 5:18 PM --- -require("class") +require("common.class") local statvfs = require "posix.sys.statvfs".statvfs -local pystring = require("pystring") -local system = require("system") -local CvProc = require("vproc") +local pystring = require("common.pystring") +local system = require("common.system") +local CvProc = require("collector.vproc") local CprocMounts = class("proc_mounts", CvProc) -function CprocMounts:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/mounts") +function CprocMounts:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/mounts") self._counter = 0 self._mpoints = {} end @@ -45,7 +45,7 @@ end local function get_point(fName) local lines = get_lines(fName) - local lOut = {"devtmpfs", "tmpfs", "shm"} + local lOut = {"devtmpfs", "tmpfs"} local tDev = {} local ret = {} for _, line in ipairs(lines) do diff --git a/source/tools/monitor/unity/collector/proc_netdev.lua b/source/tools/monitor/unity/collector/proc_netdev.lua index d0fe256df265e1c3cc423d96820a9937d8e6bf20..7a2f09543dad999dfd1da4b41e020dd027ef0e57 100644 --- a/source/tools/monitor/unity/collector/proc_netdev.lua +++ b/source/tools/monitor/unity/collector/proc_netdev.lua @@ -4,14 +4,14 @@ --- DateTime: 2022/12/16 11:57 PM --- -require("class") -local CvProc = require("vproc") -local system = require("system") +require("common.class") +local CvProc = require("collector.vproc") +local system = require("common.system") local CprocNetdev = class("proc_netdev", CvProc) -function CprocNetdev:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/net/dev") +function CprocNetdev:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/net/dev") self._lastData = {} self._lastIfNames = {} end diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index 184870e8a03b992bc25eb80a1f8788682e5cbc24..95cb88fe5ac008547d654a373960860ecb1fb2f9 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -4,15 +4,15 @@ --- DateTime: 2023/1/19 11:19 PM --- -require("class") -local CvProc = require("vproc") -local system = require("system") -local pystring = require("pystring") +require("common.class") +local CvProc = require("collector.vproc") +local system = require("common.system") +local pystring = require("common.pystring") local CprocSnmpStat = class("proc_snmp_stat", CvProc) -function CprocSnmpStat:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or nil) +function CprocSnmpStat:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or nil) self._cellsMon = {'retrans', 'abort', 'paws', 'err', 'fail', 'drop', 'overflow'} self._rec = nil end @@ -102,8 +102,8 @@ end function CprocSnmpStat:proc(elapsed, lines) CvProc.proc(self) local now = {} - self:_proc("/proc/net/snmp", now) - self:_proc("/proc/net/netstat", now) + self:_proc(self.pFile .. "proc/net/snmp", now) + self:_proc(self.pFile .. "proc/net/netstat", now) self:check(now) return self:push(lines) end diff --git a/source/tools/monitor/unity/collector/proc_sockstat.lua b/source/tools/monitor/unity/collector/proc_sockstat.lua index 06991a9cb783c6388403b2483016d9993b05b26d..d906ae97bf744e556c50846b5a3419d18c012d26 100644 --- a/source/tools/monitor/unity/collector/proc_sockstat.lua +++ b/source/tools/monitor/unity/collector/proc_sockstat.lua @@ -4,14 +4,14 @@ --- DateTime: 2023/1/17 12:27 AM --- -require("class") -local pystring = require("pystring") -local CvProc = require("vproc") +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") local CprocSockStat = class("procsockstat", CvProc) -function CprocSockStat:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/net/sockstat") +function CprocSockStat:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/net/sockstat") end function CprocSockStat:proc(elapsed, lines) @@ -28,8 +28,8 @@ function CprocSockStat:proc(elapsed, lines) for i = 1, len do local title = string.format("%s_%s", head, bodies[2 * i - 1]) local v = { - name=title, - value=tonumber(bodies[2 * i]) + name = title, + value = tonumber(bodies[2 * i]) } table.insert(vs, v) end diff --git a/source/tools/monitor/unity/collector/proc_stat.lua b/source/tools/monitor/unity/collector/proc_stat.lua index 966e7e5718ec5fb92fdac30ae638d26d13c84367..a7096896be758fd600aa9bab2c84ef54b894cb5c 100644 --- a/source/tools/monitor/unity/collector/proc_stat.lua +++ b/source/tools/monitor/unity/collector/proc_stat.lua @@ -4,14 +4,14 @@ --- DateTime: 2022/12/16 10:27 PM --- -require("class") -local pystring = require("pystring") -local CvProc = require("vproc") +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") local CprocStat = class("procstat", CvProc) -function CprocStat:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/stat") +function CprocStat:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt,pFile or "proc/stat") self._funs = self:setupTable() self._cpuArr = {} end diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index 1bbf69fb93856610388f6242b94f6c9a0ebffd8f..d3ebeb6083b24bff86186f1536c45c3e3684b68c 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -4,19 +4,19 @@ --- DateTime: 2023/1/29 6:07 PM --- -require("class") -local CvProc = require("vproc") +require("common.class") +local CvProc = require("collector.vproc") local CprocStatm = class("procStatm", CvProc) -function CprocStatm:_init_(proto, pffi, pFile) - CvProc._init_(self, proto, pffi, pFile or "/proc/self/statm") +function CprocStatm:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or nil) end function CprocStatm:proc(elapsed, lines) CvProc.proc(self) local heads = {"size", "resident", "shared", "text", "lib", "data", "dt"} - for line in io.lines(self.pFile) do + for line in io.lines("/proc/self/statm") do local vs = {} local data = self._ffi.new("var_long_t") assert(self._cffi.var_input_long(self._ffi.string(line), data) == 0) diff --git a/source/tools/monitor/unity/collector/proc_vmstat.lua b/source/tools/monitor/unity/collector/proc_vmstat.lua index 594ec2a4c9531c623bec10e719b59318f088a875..af65c9a4ac528826a3b034870a1111135f9ce46f 100644 --- a/source/tools/monitor/unity/collector/proc_vmstat.lua +++ b/source/tools/monitor/unity/collector/proc_vmstat.lua @@ -4,13 +4,13 @@ --- DateTime: 2022/12/16 11:40 PM --- -require("class") -local CkvProc = require("kvProc") +require("common.class") +local CkvProc = require("collector.kvProc") local CprocVmstat = class("proc_vmstat", CkvProc) -function CprocVmstat:_init_(proto, pffi, pFile) - CkvProc._init_(self, proto, pffi, pFile or "/proc/vmstat", "vmstat") +function CprocVmstat:_init_(proto, pffi, mnt, pFile) + CkvProc._init_(self, proto, pffi, mnt,pFile or "proc/vmstat","vmstat") end return CprocVmstat \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 215c50539c98f40ea775a17b8a73a5b909d24802..33c016a5551bd4dff1cecd579e9e2272c80cdb05 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -4,16 +4,18 @@ --- DateTime: 2022/12/16 10:12 PM --- -require("class") -local system = require("system") +require("common.class") +local system = require("common.system") -local CvProc = class("vproc") +local CvProc = class("collector.vproc") -function CvProc:_init_(proto, pffi, pFile) +function CvProc:_init_(proto, pffi, mnt, pFile) self._proto = proto self._cffi = pffi["cffi"] self._ffi = pffi["ffi"] - self.pFile = pFile + mnt = mnt or "/" + pFile = pFile or "" + self.pFile = mnt .. pFile end function CvProc:proc(elapsed) diff --git a/source/tools/monitor/unity/common/lineParse.lua b/source/tools/monitor/unity/common/lineParse.lua new file mode 100644 index 0000000000000000000000000000000000000000..35a4ff945f79ed0f0c166759d2a3c077cc6fea61 --- /dev/null +++ b/source/tools/monitor/unity/common/lineParse.lua @@ -0,0 +1,104 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/9 11:24 PM +--- + +local cjson = require("cjson.safe") +local pystring = require("common.pystring") +local system = require("common.system") +local module = {} +local json = cjson.new() + +local function parseLabel(sls, ls) + local lss = pystring:split(sls, ",") + for _, cell in ipairs(lss) do + local kv = pystring:split(cell, "=", 1) + ls[kv[1]] = kv[2] + end +end + +local function getNextQuote(vStr) + local quote = string.byte('"') + local slash = string.byte('\\') + for idx = 2, #vStr do + if string.byte(vStr, idx) == quote then + if string.byte(vStr, idx - 1) ~= slash then + return idx + end + end + end + return -1 +end + +local function parseValue(svs, vs) + local flag = true + local vStr = svs + while flag do + local kvs = pystring:split(vStr, "=", 1) + local k = kvs[1] + vStr = kvs[2] + local quote = string.byte('"') + local dot = string.byte(',') + if string.byte(vStr, 1) == quote then -- '"' + local idx = getNextQuote(vStr) + assert(idx > 0, "bad string, " .. vStr) + + idx = idx + local s = string.sub(vStr, 1, idx) + vs[k] = json.decode(s) + if string.byte(vStr, idx + 1) == dot then + vStr = string.sub(vStr, idx + 2) + else + flag = false + end + else -- number + local vss = pystring:split(vStr, ",", 1) + local v = vss[1] + vStr = vss[2] + vs[k] = tonumber(v) + if not vStr then + flag = false + end + end + end +end + +function module.parse(line) + local hvs = pystring:split(line, " ", 1) + local heads, svs = hvs[1], hvs[2] + local ths = pystring:split(heads, ",", 1) + local title, ls, vs = ths[1], {}, {} + if #ths > 1 then + local sls = ths[2] + parseLabel(sls, ls) + end + parseValue(svs, vs) + return title, ls, vs +end + +function module.pack(title, ls, vs) + local line = title + if system:keyCount(ls) > 0 then + local lss = {} + for k, v in pairs(ls) do + table.insert(lss, k .. "=" .. v) + end + line = line .. ',' .. pystring:join(",", lss) + end + local vss = {} + for k, v in pairs(vs) do + local tStr = type(v) + if tStr == "number" then + table.insert(vss, k .. '=' .. tostring(v)) + elseif tStr == "string" then + table.insert(vss, k .. '=' .. json.encode(v)) + else + error("bad value type for " .. tStr) + end + end + line = line .. ' ' .. pystring:join(",", vss) + return line +end + +return module diff --git a/source/tools/monitor/unity/common/lmd.lua b/source/tools/monitor/unity/common/lmd.lua index ccd0a6d0c1d7121d72ea347757e01861b02731d5..67a0e1bf8693ec77bb142e8b913530e272ade0b6 100644 --- a/source/tools/monitor/unity/common/lmd.lua +++ b/source/tools/monitor/unity/common/lmd.lua @@ -6,8 +6,8 @@ -- for markdown trans -require("class") -local pystring = require("pystring") +require("common.class") +local pystring = require("common.pystring") local Clmd = class("lmd") diff --git a/source/tools/monitor/unity/common/protoData.lua b/source/tools/monitor/unity/common/protoData.lua index 82bdaadf625a13d2e5159235a6c8ebb0b3e8516d..93517f9b3c03fd27714ed5ad18b74f6cebe16233 100644 --- a/source/tools/monitor/unity/common/protoData.lua +++ b/source/tools/monitor/unity/common/protoData.lua @@ -4,7 +4,7 @@ --- DateTime: 2022/12/16 6:05 PM --- -require("class") +require("common.class") local pb = require("pb") local protoc = require("protoc") local serpent = require("serpent") diff --git a/source/tools/monitor/unity/common/slsProto.lua b/source/tools/monitor/unity/common/slsProto.lua new file mode 100644 index 0000000000000000000000000000000000000000..be3711b469d204c5cb1e619523c8c32d6244add1 --- /dev/null +++ b/source/tools/monitor/unity/common/slsProto.lua @@ -0,0 +1,54 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/3 2:14 PM +--- + +require("common.class") +local system = require("common.system") + +local CslsProto = class("CslsProto") + +function CslsProto:_init_() + self._pb = require("sls_pb") +end + +function CslsProto:pack(lines) + local logList = self._pb.LogGroupList() + for _, line in ipairs(lines.logGroupList) do + local log = logList.logGroupList:add() + + if line.Reserved then + log.Reserved = line.Reserved + end + if line.Topic then + log.Topic = line.Topic + end + if line.Source then + log.Source = line.Source + end + + if line.Logs then + for _, l_log in ipairs(line.Logs) do + local cell = log.Logs:add() + cell.Time = l_log.Time + for _, l_con in ipairs(l_log.Contents) do + local con = cell.Contents:add() + con.Key = l_con.Key + con.Value = l_con.Value + end + end + end + + if line.LogTags then + for _, l_tag in ipairs(line.LogTags) do + local tag = log.LogTags:add() + tag.Key = l_tag.Key + tag.Value = l_tag.Value + end + end + end + return logList:SerializeToString() +end + +return CslsProto diff --git a/source/tools/monitor/unity/common/sls_pb.lua b/source/tools/monitor/unity/common/sls_pb.lua new file mode 100644 index 0000000000000000000000000000000000000000..9117c282b418e9ac159d490b9b770c0cc6bd628a --- /dev/null +++ b/source/tools/monitor/unity/common/sls_pb.lua @@ -0,0 +1,189 @@ +-- Generated By protoc-gen-lua Do not Edit +local protobuf = require "protobuf" +module('sls_pb') + + +local LOG = protobuf.Descriptor(); +local LOG_CONTENT = protobuf.Descriptor(); +local LOG_CONTENT_KEY_FIELD = protobuf.FieldDescriptor(); +local LOG_CONTENT_VALUE_FIELD = protobuf.FieldDescriptor(); +local LOG_TIME_FIELD = protobuf.FieldDescriptor(); +local LOG_CONTENTS_FIELD = protobuf.FieldDescriptor(); +local LOGTAG = protobuf.Descriptor(); +local LOGTAG_KEY_FIELD = protobuf.FieldDescriptor(); +local LOGTAG_VALUE_FIELD = protobuf.FieldDescriptor(); +local LOGGROUP = protobuf.Descriptor(); +local LOGGROUP_LOGS_FIELD = protobuf.FieldDescriptor(); +local LOGGROUP_RESERVED_FIELD = protobuf.FieldDescriptor(); +local LOGGROUP_TOPIC_FIELD = protobuf.FieldDescriptor(); +local LOGGROUP_SOURCE_FIELD = protobuf.FieldDescriptor(); +local LOGGROUP_LOGTAGS_FIELD = protobuf.FieldDescriptor(); +local LOGGROUPLIST = protobuf.Descriptor(); +local LOGGROUPLIST_LOGGROUPLIST_FIELD = protobuf.FieldDescriptor(); + +LOG_CONTENT_KEY_FIELD.name = "Key" +LOG_CONTENT_KEY_FIELD.full_name = ".Log.Content.Key" +LOG_CONTENT_KEY_FIELD.number = 1 +LOG_CONTENT_KEY_FIELD.index = 0 +LOG_CONTENT_KEY_FIELD.label = 2 +LOG_CONTENT_KEY_FIELD.has_default_value = false +LOG_CONTENT_KEY_FIELD.default_value = "" +LOG_CONTENT_KEY_FIELD.type = 9 +LOG_CONTENT_KEY_FIELD.cpp_type = 9 + +LOG_CONTENT_VALUE_FIELD.name = "Value" +LOG_CONTENT_VALUE_FIELD.full_name = ".Log.Content.Value" +LOG_CONTENT_VALUE_FIELD.number = 2 +LOG_CONTENT_VALUE_FIELD.index = 1 +LOG_CONTENT_VALUE_FIELD.label = 2 +LOG_CONTENT_VALUE_FIELD.has_default_value = false +LOG_CONTENT_VALUE_FIELD.default_value = "" +LOG_CONTENT_VALUE_FIELD.type = 9 +LOG_CONTENT_VALUE_FIELD.cpp_type = 9 + +LOG_CONTENT.name = "Content" +LOG_CONTENT.full_name = ".Log.Content" +LOG_CONTENT.nested_types = {} +LOG_CONTENT.enum_types = {} +LOG_CONTENT.fields = {LOG_CONTENT_KEY_FIELD, LOG_CONTENT_VALUE_FIELD} +LOG_CONTENT.is_extendable = false +LOG_CONTENT.extensions = {} +LOG_CONTENT.containing_type = LOG +LOG_TIME_FIELD.name = "Time" +LOG_TIME_FIELD.full_name = ".Log.Time" +LOG_TIME_FIELD.number = 1 +LOG_TIME_FIELD.index = 0 +LOG_TIME_FIELD.label = 2 +LOG_TIME_FIELD.has_default_value = false +LOG_TIME_FIELD.default_value = 0 +LOG_TIME_FIELD.type = 13 +LOG_TIME_FIELD.cpp_type = 3 + +LOG_CONTENTS_FIELD.name = "Contents" +LOG_CONTENTS_FIELD.full_name = ".Log.Contents" +LOG_CONTENTS_FIELD.number = 2 +LOG_CONTENTS_FIELD.index = 1 +LOG_CONTENTS_FIELD.label = 3 +LOG_CONTENTS_FIELD.has_default_value = false +LOG_CONTENTS_FIELD.default_value = {} +LOG_CONTENTS_FIELD.message_type = LOG_CONTENT +LOG_CONTENTS_FIELD.type = 11 +LOG_CONTENTS_FIELD.cpp_type = 10 + +LOG.name = "Log" +LOG.full_name = ".Log" +LOG.nested_types = {LOG_CONTENT} +LOG.enum_types = {} +LOG.fields = {LOG_TIME_FIELD, LOG_CONTENTS_FIELD} +LOG.is_extendable = false +LOG.extensions = {} +LOGTAG_KEY_FIELD.name = "Key" +LOGTAG_KEY_FIELD.full_name = ".LogTag.Key" +LOGTAG_KEY_FIELD.number = 1 +LOGTAG_KEY_FIELD.index = 0 +LOGTAG_KEY_FIELD.label = 2 +LOGTAG_KEY_FIELD.has_default_value = false +LOGTAG_KEY_FIELD.default_value = "" +LOGTAG_KEY_FIELD.type = 9 +LOGTAG_KEY_FIELD.cpp_type = 9 + +LOGTAG_VALUE_FIELD.name = "Value" +LOGTAG_VALUE_FIELD.full_name = ".LogTag.Value" +LOGTAG_VALUE_FIELD.number = 2 +LOGTAG_VALUE_FIELD.index = 1 +LOGTAG_VALUE_FIELD.label = 2 +LOGTAG_VALUE_FIELD.has_default_value = false +LOGTAG_VALUE_FIELD.default_value = "" +LOGTAG_VALUE_FIELD.type = 9 +LOGTAG_VALUE_FIELD.cpp_type = 9 + +LOGTAG.name = "LogTag" +LOGTAG.full_name = ".LogTag" +LOGTAG.nested_types = {} +LOGTAG.enum_types = {} +LOGTAG.fields = {LOGTAG_KEY_FIELD, LOGTAG_VALUE_FIELD} +LOGTAG.is_extendable = false +LOGTAG.extensions = {} +LOGGROUP_LOGS_FIELD.name = "Logs" +LOGGROUP_LOGS_FIELD.full_name = ".LogGroup.Logs" +LOGGROUP_LOGS_FIELD.number = 1 +LOGGROUP_LOGS_FIELD.index = 0 +LOGGROUP_LOGS_FIELD.label = 3 +LOGGROUP_LOGS_FIELD.has_default_value = false +LOGGROUP_LOGS_FIELD.default_value = {} +LOGGROUP_LOGS_FIELD.message_type = LOG +LOGGROUP_LOGS_FIELD.type = 11 +LOGGROUP_LOGS_FIELD.cpp_type = 10 + +LOGGROUP_RESERVED_FIELD.name = "Reserved" +LOGGROUP_RESERVED_FIELD.full_name = ".LogGroup.Reserved" +LOGGROUP_RESERVED_FIELD.number = 2 +LOGGROUP_RESERVED_FIELD.index = 1 +LOGGROUP_RESERVED_FIELD.label = 1 +LOGGROUP_RESERVED_FIELD.has_default_value = false +LOGGROUP_RESERVED_FIELD.default_value = "" +LOGGROUP_RESERVED_FIELD.type = 9 +LOGGROUP_RESERVED_FIELD.cpp_type = 9 + +LOGGROUP_TOPIC_FIELD.name = "Topic" +LOGGROUP_TOPIC_FIELD.full_name = ".LogGroup.Topic" +LOGGROUP_TOPIC_FIELD.number = 3 +LOGGROUP_TOPIC_FIELD.index = 2 +LOGGROUP_TOPIC_FIELD.label = 1 +LOGGROUP_TOPIC_FIELD.has_default_value = false +LOGGROUP_TOPIC_FIELD.default_value = "" +LOGGROUP_TOPIC_FIELD.type = 9 +LOGGROUP_TOPIC_FIELD.cpp_type = 9 + +LOGGROUP_SOURCE_FIELD.name = "Source" +LOGGROUP_SOURCE_FIELD.full_name = ".LogGroup.Source" +LOGGROUP_SOURCE_FIELD.number = 4 +LOGGROUP_SOURCE_FIELD.index = 3 +LOGGROUP_SOURCE_FIELD.label = 1 +LOGGROUP_SOURCE_FIELD.has_default_value = false +LOGGROUP_SOURCE_FIELD.default_value = "" +LOGGROUP_SOURCE_FIELD.type = 9 +LOGGROUP_SOURCE_FIELD.cpp_type = 9 + +LOGGROUP_LOGTAGS_FIELD.name = "LogTags" +LOGGROUP_LOGTAGS_FIELD.full_name = ".LogGroup.LogTags" +LOGGROUP_LOGTAGS_FIELD.number = 6 +LOGGROUP_LOGTAGS_FIELD.index = 4 +LOGGROUP_LOGTAGS_FIELD.label = 3 +LOGGROUP_LOGTAGS_FIELD.has_default_value = false +LOGGROUP_LOGTAGS_FIELD.default_value = {} +LOGGROUP_LOGTAGS_FIELD.message_type = LOGTAG +LOGGROUP_LOGTAGS_FIELD.type = 11 +LOGGROUP_LOGTAGS_FIELD.cpp_type = 10 + +LOGGROUP.name = "LogGroup" +LOGGROUP.full_name = ".LogGroup" +LOGGROUP.nested_types = {} +LOGGROUP.enum_types = {} +LOGGROUP.fields = {LOGGROUP_LOGS_FIELD, LOGGROUP_RESERVED_FIELD, LOGGROUP_TOPIC_FIELD, LOGGROUP_SOURCE_FIELD, LOGGROUP_LOGTAGS_FIELD} +LOGGROUP.is_extendable = false +LOGGROUP.extensions = {} +LOGGROUPLIST_LOGGROUPLIST_FIELD.name = "logGroupList" +LOGGROUPLIST_LOGGROUPLIST_FIELD.full_name = ".LogGroupList.logGroupList" +LOGGROUPLIST_LOGGROUPLIST_FIELD.number = 1 +LOGGROUPLIST_LOGGROUPLIST_FIELD.index = 0 +LOGGROUPLIST_LOGGROUPLIST_FIELD.label = 3 +LOGGROUPLIST_LOGGROUPLIST_FIELD.has_default_value = false +LOGGROUPLIST_LOGGROUPLIST_FIELD.default_value = {} +LOGGROUPLIST_LOGGROUPLIST_FIELD.message_type = LOGGROUP +LOGGROUPLIST_LOGGROUPLIST_FIELD.type = 11 +LOGGROUPLIST_LOGGROUPLIST_FIELD.cpp_type = 10 + +LOGGROUPLIST.name = "LogGroupList" +LOGGROUPLIST.full_name = ".LogGroupList" +LOGGROUPLIST.nested_types = {} +LOGGROUPLIST.enum_types = {} +LOGGROUPLIST.fields = {LOGGROUPLIST_LOGGROUPLIST_FIELD} +LOGGROUPLIST.is_extendable = false +LOGGROUPLIST.extensions = {} + +Log = protobuf.Message(LOG) +Log.Content = protobuf.Message(LOG_CONTENT) +LogGroup = protobuf.Message(LOGGROUP) +LogGroupList = protobuf.Message(LOGGROUPLIST) +LogTag = protobuf.Message(LOGTAG) \ No newline at end of file diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index 770295c204de3b5e39a9954b2f4cec8efb79a30c..1c083430d397f0f951fae47ee0afa152355510eb 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -5,7 +5,7 @@ --- local socket = require("socket") -local serpent = require("serpent") +local serpent = require("common.serpent") local system = {} @@ -51,7 +51,7 @@ function system:valueIsIn(tbl, value) if type(tbl) ~= "table" then return false end - for _, v in ipairs(tbl) do + for _, v in pairs(tbl) do if v == value then return true end @@ -59,6 +59,36 @@ function system:valueIsIn(tbl, value) return false end +function system:valueIndex(tbl, value) + if type(tbl) ~= "table" then + return 0 + end + for i, v in ipairs(tbl) do + if v == value then + return i + end + end + return 0 +end + +function system:keyCount(tbl) + local count = 0 + for _, _ in pairs(tbl) do + count = count + 1 + end + return count +end + +function system:dictCopy(tbl) + local cp = {} + assert(type(tbl) == "table") + + for k, v in pairs(tbl) do + cp[k] = v + end + return cp +end + function system:hex2ups(hex) return (string.gsub(hex, ".", function (c) return string.format("%02X", string.byte(c)) @@ -71,9 +101,28 @@ function system:hex2lows(hex) end)) end +function system:hexdump(buf) + for byte=1, #buf, 16 do + local chunk = buf:sub(byte, byte+15) + io.write(string.format('%08X ',byte-1)) + chunk:gsub('.', function (c) io.write(string.format('%02X ',string.byte(c))) end) + io.write(string.rep(' ',3*(16-#chunk))) + io.write(' ',chunk:gsub('%c','.'),"\n") + end +end + function system:timeRfc1123(t) t = t or os.time() return os.date("!%a, %d %b %Y %H:%M:%S GMT", t) end +function system:parseYaml(fYaml) + local lyaml = require("lyaml") + local f = io.open(fYaml,"r") + local s = f:read("*all") + f:close() + + return lyaml.load(s) +end + return system \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/httpApp.lua b/source/tools/monitor/unity/httplib/httpApp.lua index 8416d012af5e035ff5d91c46e03da2564a19513f..42995cae63540d25163f542f42b92358e4871e75 100644 --- a/source/tools/monitor/unity/httplib/httpApp.lua +++ b/source/tools/monitor/unity/httplib/httpApp.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/22 12:14 PM --- -require("class") -local ChttpBase = require("httpBase") -local pystring = require("pystring") +require("common.class") +local ChttpBase = require("httplib.httpBase") +local pystring = require("common.pystring") local ChttpApp = class("ChttpApp", ChttpBase) @@ -14,10 +14,11 @@ function ChttpApp:_init_(frame) ChttpBase._init_(self) end -function ChttpApp:echo(tRet) +function ChttpApp:echo(tRet, keep) local stat = self:packStat(200) local tHead = { ["Content-Type"] = "application/json", + ["Connection"] = (keep and "keep-alive") or "close" } local body = self:jencode(tRet) local headers = self:packHeaders(tHead, #body) diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 10ef0dd3a44e1ae78114e08605f78a27a850a35c..7ddbc24e43b3c9aa16f0eff8ef7394103590b2af 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -4,8 +4,8 @@ --- DateTime: 2022/12/24 9:52 AM --- -require("class") -local ChttpComm = require("httpComm") +require("common.class") +local ChttpComm = require("httplib.httpComm") local ChttpBase = class("ChttpBase", ChttpComm) @@ -20,13 +20,22 @@ function ChttpBase:_install(frame) end end -function ChttpBase:echo(tRet) +function ChttpBase:echo(tRet, keep) error("ChttpBase:echo is a virtual function.") end +local function checkKeep(tReq) + local conn = tReq.header["connection"] + if conn and string.lower(conn) == "keep-alive" then + return true + end + return false +end + function ChttpBase:call(tReq) local tRet = self._urlCb[tReq.path](tReq) - return self:echo(tRet) + local keep = checkKeep(tReq) + return self:echo(tRet, keep), keep end return ChttpBase diff --git a/source/tools/monitor/unity/httplib/httpCli.lua b/source/tools/monitor/unity/httplib/httpCli.lua index 550cb362b66bfe59dc32b14542aedc0848b943cf..c83888f7f7055f37379ea9ad3241f7e0db02a9af 100644 --- a/source/tools/monitor/unity/httplib/httpCli.lua +++ b/source/tools/monitor/unity/httplib/httpCli.lua @@ -4,12 +4,13 @@ --- DateTime: 2022/12/19 4:40 PM --- -require("class") -local ChttpComm = require("httpComm") +require("common.class") +local ChttpComm = require("httplib.httpComm") local ChttpCli = class("httpCli", ChttpComm) -function ChttpCli:_init_() +function ChttpCli:_init_(proxy) ChttpComm._init_(self) + self._proxy = proxy self._http = require("socket.http") self._ltn12 = require("ltn12") end @@ -18,6 +19,7 @@ function ChttpCli:get(Url) local t = {} local res, code, head= self._http.request{ url=Url, + proxy = self._proxy, sink = self._ltn12.sink.table(t) } local body = table.concat(t) @@ -38,6 +40,7 @@ function ChttpCli:post(Url, reqs, header) method = "POST", headers = headers, source = source, + proxy = self._proxy, sink = self._ltn12.sink.table(t) } local body = table.concat(t) diff --git a/source/tools/monitor/unity/httplib/httpComm.lua b/source/tools/monitor/unity/httplib/httpComm.lua index 43ed0196d754f25d48271d1815e55a24bfa14b34..12b691f318d6f72b218f1be258387b8b909245bd 100644 --- a/source/tools/monitor/unity/httplib/httpComm.lua +++ b/source/tools/monitor/unity/httplib/httpComm.lua @@ -4,13 +4,13 @@ --- DateTime: 2022/12/19 10:46 PM --- -require("class") -local pystring = require("pystring") +require("common.class") +local pystring = require("common.pystring") local sockerUrl = require("socket.url") -local ChttpComm = class("httpComm") +local ChttpComm = class("httplib.httpComm") -local cjson = require("cjson") +local cjson = require("cjson.safe") local json = cjson.new() local function codeTable() @@ -75,8 +75,7 @@ end local function originHeader() return { - connection = "close", - server = "beaver/0.0.1", + server = "beaver/0.0.2", date = os.date("%a, %d %b %Y %H:%M:%S %Z", os.time()), } end diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index f7bf570e6f3ff5b2a59034fcdf6a61e37764777c..5205af6556a44472e8fee1d3873fc0b8f1d75eaa 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/23 11:53 PM --- -require("class") -local pystring = require("pystring") -local ChttpBase = require("httpBase") +require("common.class") +local pystring = require("common.pystring") +local ChttpBase = require("httplib.httpBase") local ChttpHtml = class("ChttpHtml", ChttpBase) @@ -15,7 +15,7 @@ function ChttpHtml:_init_(frame) end function ChttpHtml:markdown(text) - local md = require("lmd") + local md = require("common.lmd") return md:toHtml(text) end @@ -41,10 +41,11 @@ local function htmlPack(title, content) return pystring:join("", bodies) end -function ChttpHtml:echo(tRet) +function ChttpHtml:echo(tRet, keep) local stat = self:packStat(200) local tHead = { ["Content-Type"] = "text/html", + ["Connection"] = (keep and "keep-alive") or "close" } local body = htmlPack(tRet.title, tRet.content) local headers = self:packHeaders(tHead, #body) diff --git a/source/tools/monitor/unity/httplib/httpPlain.lua b/source/tools/monitor/unity/httplib/httpPlain.lua index 65f58d14ee8d42deba19457e3afd3df67f6ae357..81a6b39dad355f430c12345ff4b5452b612e0d90 100644 --- a/source/tools/monitor/unity/httplib/httpPlain.lua +++ b/source/tools/monitor/unity/httplib/httpPlain.lua @@ -4,9 +4,9 @@ --- DateTime: 2022/12/24 12:32 AM --- -require("class") -local pystring = require("pystring") -local ChttpBase = require("httpBase") +require("common.class") +local pystring = require("common.pystring") +local ChttpBase = require("httplib.httpBase") local ChttpPlain = class("ChttpPlain", ChttpBase) @@ -14,10 +14,11 @@ function ChttpPlain:_init_(frame) ChttpBase._init_(self) end -function ChttpPlain:echo(tRet) +function ChttpPlain:echo(tRet, keep) local stat = self:packStat(200) local tHead = { ["Content-Type"] = "text/plain", + ["Connection"] = (keep and "keep-alive") or "close" } local body = tRet.text local headers = self:packHeaders(tHead, #body) diff --git a/source/tools/monitor/unity/httplib/slsCli.lua b/source/tools/monitor/unity/httplib/slsCli.lua index c6351ae54e2acb46d3620c891f5edd1e882faaaa..be15d31b59c8746c07eb29c8dd0bd91223699ebc 100644 --- a/source/tools/monitor/unity/httplib/slsCli.lua +++ b/source/tools/monitor/unity/httplib/slsCli.lua @@ -4,28 +4,27 @@ --- DateTime: 2023/1/30 3:06 PM --- -require("class") +require("common.class") local sha1 = require("sha1") local lz4 = require("lz4") local md5 = require("md5") local base64 = require("base64") -local system = require("system") -local pystring = require("pystring") -local pb = require("pb") -local protoc = require("protoc") +local system = require("common.system") +local pystring = require("common.pystring") -local ChttpCli = require("httpCli") +local ChttpCli = require("httplib.httpCli") +local CslsProto = require("protobuf.slsProto") local CslsCli = class("slsCli", ChttpCli) -function CslsCli:_init_(endPoint, project, store, key, pswd) - ChttpCli._init_(self) +function CslsCli:_init_(endPoint, project, store, key, pswd, proxy) + ChttpCli._init_(self, proxy) self._endPoint = endPoint self._project = project self._store = store self._key = key self._pswd = pswd - self._pc = self:_setupPb() + self._proto = CslsProto.new() end local function packLog(vm, log) @@ -42,25 +41,30 @@ local function packLog(vm, log) }, }, } + local Tag = { + Key = "log", + Value = log, + } local LogGroup = { Logs = {Log, }, Source = "sysak mon.", + LogTags = {Tag,}, } return { logGroupList = {LogGroup,}, } end -local function packHead(data, rawSize, project, endPoint) +local function packHead(data, project, endPoint) return { ["Content-Type"] = "application/x-protobuf", ["Content-Length"] = #data, ["Content-MD5"] = system:hex2ups(md5.sum(data)), ["Date"] = system:timeRfc1123(), ["Host"] = project .. "." .. endPoint, + --["Host"] = project .. "." .. "cn-heyuan-share.log.aliyuncs.com", ["x-log-apiversion"] = "0.6.0", - ["x-log-bodyrawsize"] = rawSize, - ["x-log-compresstype"] = "lz4", + ["x-log-bodyrawsize"] = #data, ["x-log-signaturemethod"] = "hmac-sha1", } end @@ -82,54 +86,18 @@ function CslsCli:signature(heads, uri, msg) end function CslsCli:putLog(vm, log) - local uri = "/logstores/" .. self._store + local uri = "/logstores/" .. self._store .. "/shards/lb" local logList = packLog(vm, log) - local msg = assert(pb.encode("LogGroupList", logList)) - local body = lz4.compress(msg) - local heads = packHead(body, #msg, self._project, self._endPoint) + print(self:jencode(logList)) + local msg = self._proto:pack(logList) + local heads = packHead(msg, self._project, self._endPoint) self:signature(heads, uri, msg) - local url = string.format("http://%s", self._endPoint) + local url = string.format("http://%s%s", self._endPoint, uri) + print(url) + print(system:hexdump(msg)) print(system:dump(heads)) - local res = self:post(url, body, heads) + local res = self:post(url, msg, heads) print(system:dump(res)) end -function CslsCli:_setupPb() - local pc = protoc:new() - local format = [[ -message Log -{ - required uint32 Time = 1; - message Content - { - required string Key = 1; - required string Value = 2; - } - repeated Content Contents = 2; -} - -message LogTag -{ - required string Key = 1; - required string Value = 2; -} - -message LogGroup -{ - repeated Log Logs= 1; - optional string Reserved = 2; - optional string Topic = 3; - optional string Source = 4; - repeated LogTag LogTags = 6; -} - -message LogGroupList -{ - repeated LogGroup logGroupList = 1; -} -]] - assert(pc:load(format)) - return pc -end - return CslsCli \ No newline at end of file diff --git a/source/tools/monitor/unity/protobuf/Makefile b/source/tools/monitor/unity/protobuf/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..59f6291e8bc16f3e0e806854c70e3d9bcd96b68c --- /dev/null +++ b/source/tools/monitor/unity/protobuf/Makefile @@ -0,0 +1,9 @@ +SRC=npb.c + +TARGET=npb.so +CFLAGS=-std=gnu99 + +all:$(TARGET) + +$(TARGET):$(SRC) + gcc -O3 -shared -fPIC $(SRC) $(CFLAGS) -o $@ diff --git a/source/tools/monitor/unity/protobuf/containers.lua b/source/tools/monitor/unity/protobuf/containers.lua new file mode 100644 index 0000000000000000000000000000000000000000..54ac71e397b7b47e1eacaa382e06314b2cd2e118 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/containers.lua @@ -0,0 +1,78 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: containers.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年08月02日 16时15分42秒 CST +-------------------------------------------------------------------------------- +-- +local setmetatable = setmetatable +local table = table +local rawset = rawset +local error = error + +module "containers" + +local _RCFC_meta = { + add = function(self) + local value = self._message_descriptor._concrete_class() + local listener = self._listener + rawset(self, #self + 1, value) + value:_SetListener(listener) + if listener.dirty == false then + listener:Modified() + end + return value + end, + remove = function(self, key) + local listener = self._listener + table.remove(self, key) + listener:Modified() + end, + __newindex = function(self, key, value) + error("RepeatedCompositeFieldContainer Can't set value directly") + end +} +_RCFC_meta.__index = _RCFC_meta + +function RepeatedCompositeFieldContainer(listener, message_descriptor) + local o = { + _listener = listener, + _message_descriptor = message_descriptor + } + return setmetatable(o, _RCFC_meta) +end + +local _RSFC_meta = { + append = function(self, value) + self._type_checker(value) + rawset(self, #self + 1, value) + self._listener:Modified() + end, + remove = function(self, key) + table.remove(self, key) + self._listener:Modified() + end, + __newindex = function(self, key, value) + error("RepeatedCompositeFieldContainer Can't set value directly") + end +} +_RSFC_meta.__index = _RSFC_meta + +function RepeatedScalarFieldContainer(listener, type_checker) + local o = {} + o._listener = listener + o._type_checker = type_checker + return setmetatable(o, _RSFC_meta) +end + + diff --git a/source/tools/monitor/unity/protobuf/decoder.lua b/source/tools/monitor/unity/protobuf/decoder.lua new file mode 100644 index 0000000000000000000000000000000000000000..e049998ea9a345f0fc64d3a70b939ac447d463b7 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/decoder.lua @@ -0,0 +1,337 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: decoder.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年07月29日 19时30分51秒 CST +-------------------------------------------------------------------------------- +-- +local string = string +local table = table +local assert = assert +local ipairs = ipairs +local error = error +local print = print + +local pb = require "npb" +local encoder = require "encoder" +local wire_format = require "wire_format" +module "decoder" + +local _DecodeVarint = pb.varint_decoder +local _DecodeSignedVarint = pb.signed_varint_decoder + +local _DecodeVarint32 = pb.varint_decoder +local _DecodeSignedVarint32 = pb.signed_varint_decoder + +ReadTag = pb.read_tag + +local function _SimpleDecoder(wire_type, decode_value) + return function(field_number, is_repeated, is_packed, key, new_default) + if is_packed then + local DecodeVarint = _DecodeVarint + return function (buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + local endpoint + endpoint, pos = DecodeVarint(buffer, pos) + endpoint = endpoint + pos + if endpoint > pend then + error('Truncated message.') + end + local element + while pos < endpoint do + element, pos = decode_value(buffer, pos) + value[#value + 1] = element + end + if pos > endpoint then + value:remove(#value) + error('Packed element was truncated.') + end + return pos + end + elseif is_repeated then + local tag_bytes = encoder.TagBytes(field_number, wire_type) + local tag_len = #tag_bytes + local sub = string.sub + return function(buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + while 1 do + local element, new_pos = decode_value(buffer, pos) + value:append(element) + pos = new_pos + tag_len + if sub(buffer, new_pos+1, pos) ~= tag_bytes or new_pos >= pend then + if new_pos > pend then + error('Truncated message.') + end + return new_pos + end + end + end + else + return function (buffer, pos, pend, message, field_dict) + field_dict[key], pos = decode_value(buffer, pos) + if pos > pend then + field_dict[key] = nil + error('Truncated message.') + end + return pos + end + end + end +end + +local function _ModifiedDecoder(wire_type, decode_value, modify_value) + local InnerDecode = function (buffer, pos) + local result, new_pos = decode_value(buffer, pos) + return modify_value(result), new_pos + end + return _SimpleDecoder(wire_type, InnerDecode) +end + +local function _StructPackDecoder(wire_type, value_size, format) + local struct_unpack = pb.struct_unpack + + function InnerDecode(buffer, pos) + local new_pos = pos + value_size + local result = struct_unpack(format, buffer, pos) + return result, new_pos + end + return _SimpleDecoder(wire_type, InnerDecode) +end + +local function _Boolean(value) + return value ~= 0 +end + +Int32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32) +EnumDecoder = Int32Decoder + +Int64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeSignedVarint) + +UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32) +UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint) + +SInt32Decoder = _ModifiedDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode32) +SInt64Decoder = _ModifiedDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode64) + +Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('I')) +Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('Q')) +SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('i')) +SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('q')) +FloatDecoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('f')) +DoubleDecoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('d')) + + +BoolDecoder = _ModifiedDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint, _Boolean) + + +function StringDecoder(field_number, is_repeated, is_packed, key, new_default) + local DecodeVarint = _DecodeVarint + local sub = string.sub + -- local unicode = unicode + assert(not is_packed) + if is_repeated then + local tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local tag_len = #tag_bytes + return function (buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + while 1 do + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated string.') + end + value:append(sub(buffer, pos+1, new_pos)) + pos = new_pos + tag_len + if sub(buffer, new_pos + 1, pos) ~= tag_bytes or new_pos == pend then + return new_pos + end + end + end + else + return function (buffer, pos, pend, message, field_dict) + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated string.') + end + field_dict[key] = sub(buffer, pos + 1, new_pos) + return new_pos + end + end +end + +function BytesDecoder(field_number, is_repeated, is_packed, key, new_default) + local DecodeVarint = _DecodeVarint + local sub = string.sub + assert(not is_packed) + if is_repeated then + local tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local tag_len = #tag_bytes + return function (buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + while 1 do + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated string.') + end + value:append(sub(buffer, pos + 1, new_pos)) + pos = new_pos + tag_len + if sub(buffer, new_pos + 1, pos) ~= tag_bytes or new_pos == pend then + return new_pos + end + end + end + else + return function(buffer, pos, pend, message, field_dict) + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated string.') + end + field_dict[key] = sub(buffer, pos + 1, new_pos) + return new_pos + end + end +end + +function MessageDecoder(field_number, is_repeated, is_packed, key, new_default) + local DecodeVarint = _DecodeVarint + local sub = string.sub + + assert(not is_packed) + if is_repeated then + local tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local tag_len = #tag_bytes + return function (buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + while 1 do + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated message.') + end + if value:add():_InternalParse(buffer, pos, new_pos) ~= new_pos then + error('Unexpected end-group tag.') + end + pos = new_pos + tag_len + if sub(buffer, new_pos + 1, pos) ~= tag_bytes or new_pos == pend then + return new_pos + end + end + end + else + return function (buffer, pos, pend, message, field_dict) + local value = field_dict[key] + if value == nil then + value = new_default(message) + field_dict[key] = value + end + local size, new_pos + size, pos = DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > pend then + error('Truncated message.') + end + if value:_InternalParse(buffer, pos, new_pos) ~= new_pos then + error('Unexpected end-group tag.') + end + return new_pos + end + end +end + +function _SkipVarint(buffer, pos, pend) + local value + value, pos = _DecodeVarint(buffer, pos) + return pos +end + +function _SkipFixed64(buffer, pos, pend) + pos = pos + 8 + if pos > pend then + error('Truncated message.') + end + return pos +end + +function _SkipLengthDelimited(buffer, pos, pend) + local size + size, pos = _DecodeVarint(buffer, pos) + pos = pos + size + if pos > pend then + error('Truncated message.') + end + return pos +end + +function _SkipFixed32(buffer, pos, pend) + pos = pos + 4 + if pos > pend then + error('Truncated message.') + end + return pos +end + +function _RaiseInvalidWireType(buffer, pos, pend) + error('Tag had invalid wire type.') +end + +function _FieldSkipper() + WIRETYPE_TO_SKIPPER = { + _SkipVarint, + _SkipFixed64, + _SkipLengthDelimited, + _SkipGroup, + _EndGroup, + _SkipFixed32, + _RaiseInvalidWireType, + _RaiseInvalidWireType, + } + +-- wiretype_mask = wire_format.TAG_TYPE_MASK + local ord = string.byte + local sub = string.sub + + return function (buffer, pos, pend, tag_bytes) + local wire_type = ord(sub(tag_bytes, 1, 1)) % 8 + 1 + return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, pend) + end +end + +SkipField = _FieldSkipper() diff --git a/source/tools/monitor/unity/protobuf/descriptor.lua b/source/tools/monitor/unity/protobuf/descriptor.lua new file mode 100644 index 0000000000000000000000000000000000000000..6b200729959aea0a81df6d35deacfdafc435cd4f --- /dev/null +++ b/source/tools/monitor/unity/protobuf/descriptor.lua @@ -0,0 +1,64 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: descriptor.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年08月11日 18时45分43秒 CST +-------------------------------------------------------------------------------- +-- + +module "descriptor" + +FieldDescriptor = { + TYPE_DOUBLE = 1, + TYPE_FLOAT = 2, + TYPE_INT64 = 3, + TYPE_UINT64 = 4, + TYPE_INT32 = 5, + TYPE_FIXED64 = 6, + TYPE_FIXED32 = 7, + TYPE_BOOL = 8, + TYPE_STRING = 9, + TYPE_GROUP = 10, + TYPE_MESSAGE = 11, + TYPE_BYTES = 12, + TYPE_UINT32 = 13, + TYPE_ENUM = 14, + TYPE_SFIXED32 = 15, + TYPE_SFIXED64 = 16, + TYPE_SINT32 = 17, + TYPE_SINT64 = 18, + MAX_TYPE = 18, + + -- Must be consistent with C++ FieldDescriptor::CppType enum in + -- descriptor.h. + -- + CPPTYPE_INT32 = 1, + CPPTYPE_INT64 = 2, + CPPTYPE_UINT32 = 3, + CPPTYPE_UINT64 = 4, + CPPTYPE_DOUBLE = 5, + CPPTYPE_FLOAT = 6, + CPPTYPE_BOOL = 7, + CPPTYPE_ENUM = 8, + CPPTYPE_STRING = 9, + CPPTYPE_MESSAGE = 10, + MAX_CPPTYPE = 10, + + -- Must be consistent with C++ FieldDescriptor::Label enum in + -- descriptor.h. + -- + LABEL_OPTIONAL = 1, + LABEL_REQUIRED = 2, + LABEL_REPEATED = 3, + MAX_LABEL = 3 +} diff --git a/source/tools/monitor/unity/protobuf/encoder.lua b/source/tools/monitor/unity/protobuf/encoder.lua new file mode 100644 index 0000000000000000000000000000000000000000..e447f33a3014f953a3015d7bd910619970b7d281 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/encoder.lua @@ -0,0 +1,461 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: encoder.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年07月29日 19时30分46秒 CST +-------------------------------------------------------------------------------- +-- +local string = string +local table = table +local ipairs = ipairs +local assert =assert + +local pb = require "npb" +local wire_format = require "wire_format" +module "encoder" + +function _VarintSize(value) + if value <= 0x7f then return 1 end + if value <= 0x3fff then return 2 end + if value <= 0x1fffff then return 3 end + if value <= 0xfffffff then return 4 end + return 5 +end + +function _SignedVarintSize(value) + if value < 0 then return 10 end + if value <= 0x7f then return 1 end + if value <= 0x3fff then return 2 end + if value <= 0x1fffff then return 3 end + if value <= 0xfffffff then return 4 end + return 5 +end + +function _TagSize(field_number) + return _VarintSize(wire_format.PackTag(field_number, 0)) +end + +function _SimpleSizer(compute_value_size) + return function(field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + if is_packed then + local VarintSize = _VarintSize + return function(value) + local result = 0 + for _, element in ipairs(value) do + result = result + compute_value_size(element) + end + return result + VarintSize(result) + tag_size + end + elseif is_repeated then + return function(value) + local result = tag_size * #value + for _, element in ipairs(value) do + result = result + compute_value_size(element) + end + return result + end + else + return function (value) + return tag_size + compute_value_size(value) + end + end + end +end + +function _ModifiedSizer(compute_value_size, modify_value) + return function (field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + if is_packed then + local VarintSize = _VarintSize + return function (value) + local result = 0 + for _, element in ipairs(value) do + result = result + compute_value_size(modify_value(element)) + end + return result + VarintSize(result) + tag_size + end + elseif is_repeated then + return function (value) + local result = tag_size * #value + for _, element in ipairs(value) do + result = result + compute_value_size(modify_value(element)) + end + return result + end + else + return function (value) + return tag_size + compute_value_size(modify_value(value)) + end + end + end +end + +function _FixedSizer(value_size) + return function (field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + if is_packed then + local VarintSize = _VarintSize + return function (value) + local result = #value * value_size + return result + VarintSize(result) + tag_size + end + elseif is_repeated then + local element_size = value_size + tag_size + return function(value) + return #value * element_size + end + else + local field_size = value_size + tag_size + return function (value) + return field_size + end + end + end +end + +Int32Sizer = _SimpleSizer(_SignedVarintSize) +Int64Sizer = Int32Sizer +EnumSizer = Int32Sizer + +UInt32Sizer = _SimpleSizer(_VarintSize) +UInt64Sizer = UInt32Sizer + +SInt32Sizer = _ModifiedSizer(_SignedVarintSize, wire_format.ZigZagEncode) +SInt64Sizer = SInt32Sizer + +Fixed32Sizer = _FixedSizer(4) +SFixed32Sizer = Fixed32Sizer +FloatSizer = Fixed32Sizer + +Fixed64Sizer = _FixedSizer(8) +SFixed64Sizer = Fixed64Sizer +DoubleSizer = Fixed64Sizer + +BoolSizer = _FixedSizer(1) + + +function StringSizer(field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + local VarintSize = _VarintSize + assert(not is_packed) + if is_repeated then + return function(value) + local result = tag_size * #value + for _, element in ipairs(value) do + local l = #element + result = result + VarintSize(l) + l + end + return result + end + else + return function(value) + local l = #value + return tag_size + VarintSize(l) + l + end + end +end + +function BytesSizer(field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + local VarintSize = _VarintSize + assert(not is_packed) + if is_repeated then + return function (value) + local result = tag_size * #value + for _,element in ipairs(value) do + local l = #element + result = result + VarintSize(l) + l + end + return result + end + else + return function (value) + local l = #value + return tag_size + VarintSize(l) + l + end + end +end + +function MessageSizer(field_number, is_repeated, is_packed) + local tag_size = _TagSize(field_number) + local VarintSize = _VarintSize + assert(not is_packed) + if is_repeated then + return function(value) + local result = tag_size * #value + for _,element in ipairs(value) do + local l = element:ByteSize() + result = result + VarintSize(l) + l + end + return result + end + else + return function (value) + local l = value:ByteSize() + return tag_size + VarintSize(l) + l + end + end +end + + +-- ==================================================================== +-- Encoders! + +local _EncodeVarint = pb.varint_encoder +local _EncodeSignedVarint = pb.signed_varint_encoder + + +function _VarintBytes(value) + local out = {} + local write = function(value) + out[#out + 1 ] = value + end + _EncodeSignedVarint(write, value) + return table.concat(out) +end + +function TagBytes(field_number, wire_type) + return _VarintBytes(wire_format.PackTag(field_number, wire_type)) +end + +function _SimpleEncoder(wire_type, encode_value, compute_value_size) + return function(field_number, is_repeated, is_packed) + if is_packed then + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + return function(write, value) + write(tag_bytes) + local size = 0 + for _, element in ipairs(value) do + size = size + compute_value_size(element) + end + EncodeVarint(write, size) + for element in value do + encode_value(write, element) + end + end + elseif is_repeated then + local tag_bytes = TagBytes(field_number, wire_type) + return function(write, value) + for _, element in ipairs(value) do + write(tag_bytes) + encode_value(write, element) + end + end + else + local tag_bytes = TagBytes(field_number, wire_type) + return function(write, value) + write(tag_bytes) + encode_value(write, value) + end + end + end +end + +function _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value) + return function (field_number, is_repeated, is_packed) + if is_packed then + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + return function (write, value) + write(tag_bytes) + local size = 0 + for _, element in ipairs(value) do + size = size + compute_value_size(modify_value(element)) + end + EncodeVarint(write, size) + for _, element in ipairs(value) do + encode_value(write, modify_value(element)) + end + end + elseif is_repeated then + local tag_bytes = TagBytes(field_number, wire_type) + return function (write, value) + for _, element in ipairs(value) do + write(tag_bytes) + encode_value(write, modify_value(element)) + end + end + else + local tag_bytes = TagBytes(field_number, wire_type) + return function (write, value) + write(tag_bytes) + encode_value(write, modify_value(value)) + end + end + end +end + +function _StructPackEncoder(wire_type, value_size, format) + return function(field_number, is_repeated, is_packed) + local struct_pack = pb.struct_pack + if is_packed then + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + return function (write, value) + write(tag_bytes) + EncodeVarint(write, #value * value_size) + for _, element in ipairs(value) do + struct_pack(write, format, element) + end + end + elseif is_repeated then + local tag_bytes = TagBytes(field_number, wire_type) + return function (write, value) + for _, element in ipairs(value) do + write(tag_bytes) + struct_pack(write, format, element) + end + end + else + local tag_bytes = TagBytes(field_number, wire_type) + return function (write, value) + write(tag_bytes) + struct_pack(write, format, value) + end + end + + end +end + +Int32Encoder = _SimpleEncoder(wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize) +Int64Encoder = Int32Encoder +EnumEncoder = Int32Encoder + +UInt32Encoder = _SimpleEncoder(wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize) +UInt64Encoder = UInt32Encoder + +SInt32Encoder = _ModifiedEncoder( + wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize, + wire_format.ZigZagEncode32) + +SInt64Encoder = _ModifiedEncoder( + wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize, + wire_format.ZigZagEncode64) + +Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('I')) +Fixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('Q')) +SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('i')) +SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('q')) +FloatEncoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, 4, string.byte('f')) +DoubleEncoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, 8, string.byte('d')) + + +function BoolEncoder(field_number, is_repeated, is_packed) + local false_byte = '\0' + local true_byte = '\1' + if is_packed then + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + return function (write, value) + write(tag_bytes) + EncodeVarint(write, #value) + for _, element in ipairs(value) do + if element then + write(true_byte) + else + write(false_byte) + end + end + end + elseif is_repeated then + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) + return function(write, value) + for _, element in ipairs(value) do + write(tag_bytes) + if element then + write(true_byte) + else + write(false_byte) + end + end + end + else + local tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) + return function (write, value) + write(tag_bytes) + if value then + return write(true_byte) + end + return write(false_byte) + end + end +end + +function StringEncoder(field_number, is_repeated, is_packed) + local tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + assert(not is_packed) + if is_repeated then + return function (write, value) + for _, element in ipairs(value) do +-- encoded = element.encode('utf-8') + write(tag) + EncodeVarint(write, #element) + write(element) + end + end + else + return function (write, value) +-- local encoded = value.encode('utf-8') + write(tag) + EncodeVarint(write, #value) + return write(value) + end + end +end + +function BytesEncoder(field_number, is_repeated, is_packed) + local tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + assert(not is_packed) + if is_repeated then + return function (write, value) + for _, element in ipairs(value) do + write(tag) + EncodeVarint(write, #element) + write(element) + end + end + else + return function(write, value) + write(tag) + EncodeVarint(write, #value) + return write(value) + end + end +end + + +function MessageEncoder(field_number, is_repeated, is_packed) + local tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local EncodeVarint = _EncodeVarint + assert(not is_packed) + if is_repeated then + return function(write, value) + for _, element in ipairs(value) do + write(tag) + EncodeVarint(write, element:ByteSize()) + element:_InternalSerialize(write) + end + end + else + return function (write, value) + write(tag) + EncodeVarint(write, value:ByteSize()) + return value:_InternalSerialize(write) + end + end +end + diff --git a/source/tools/monitor/unity/protobuf/listener.lua b/source/tools/monitor/unity/protobuf/listener.lua new file mode 100644 index 0000000000000000000000000000000000000000..1c00eaef3423cefe4c6a776ee252f6f709a9e5c8 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/listener.lua @@ -0,0 +1,50 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: listener.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年08月02日 17时35分25秒 CST +-------------------------------------------------------------------------------- +-- +local setmetatable = setmetatable + +module "listener" + +local _null_listener = { + Modified = function() + end +} + +function NullMessageListener() + return _null_listener +end + +local _listener_meta = { + Modified = function(self) + if self.dirty then + return + end + if self._parent_message then + self._parent_message:_Modified() + end + end +} +_listener_meta.__index = _listener_meta + +function Listener(parent_message) + local o = {} + o.__mode = "v" + o._parent_message = parent_message + o.dirty = false + return setmetatable(o, _listener_meta) +end + diff --git a/source/tools/monitor/unity/protobuf/npb.c b/source/tools/monitor/unity/protobuf/npb.c new file mode 100644 index 0000000000000000000000000000000000000000..3ed340c3e482af09241228d7086771ed7ff075dd --- /dev/null +++ b/source/tools/monitor/unity/protobuf/npb.c @@ -0,0 +1,474 @@ +/* + * ===================================================================================== + * + * Filename: npb.c + * + * Description: protoc-gen-lua + * Google's Protocol Buffers project, ported to lua. + * https://code.google.com/p/protoc-gen-lua/ + * + * Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com + * All rights reserved. + * + * Use, modification and distribution are subject to the "New BSD License" + * as listed at . + * + * Created: 2010年08月02日 18时04分21秒 + * + * Company: NetEase + * + * ===================================================================================== + */ +#include +#include + +#include +#include +#include + +#ifdef _ALLBSD_SOURCE +#include +#else +#include +#endif + +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define IS_LITTLE_ENDIAN +#endif + +#define IOSTRING_META "protobuf.IOString" + +#define checkiostring(L) \ + (IOString*) luaL_checkudata(L, 1, IOSTRING_META) + +#define IOSTRING_BUF_LEN 65535 + +typedef struct{ + size_t size; + char buf[IOSTRING_BUF_LEN]; +} IOString; + +static void pack_varint(luaL_Buffer *b, uint64_t value) +{ + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + if (value >= 0x80) + { + luaL_addchar(b, value | 0x80); + value >>= 7; + } + } + } + } + } + } + } + } + } + luaL_addchar(b, value); +} + +static int varint_encoder(lua_State *L) +{ + lua_Number l_value = luaL_checknumber(L, 2); + uint64_t value = (uint64_t)l_value; + + luaL_Buffer b; + luaL_buffinit(L, &b); + + pack_varint(&b, value); + + lua_settop(L, 1); + luaL_pushresult(&b); + lua_call(L, 1, 0); + return 0; +} + +static int signed_varint_encoder(lua_State *L) +{ + lua_Number l_value = luaL_checknumber(L, 2); + int64_t value = (int64_t)l_value; + + luaL_Buffer b; + luaL_buffinit(L, &b); + + if (value < 0) + { + pack_varint(&b, *(uint64_t*)&value); + }else{ + pack_varint(&b, value); + } + + lua_settop(L, 1); + luaL_pushresult(&b); + lua_call(L, 1, 0); + return 0; +} + +static int pack_fixed32(lua_State *L, uint8_t* value){ +#ifdef IS_LITTLE_ENDIAN + lua_pushlstring(L, (char*)value, 4); +#else + uint32_t v = htole32(*(uint32_t*)value); + lua_pushlstring(L, (char*)&v, 4); +#endif + return 0; +} + +static int pack_fixed64(lua_State *L, uint8_t* value){ +#ifdef IS_LITTLE_ENDIAN + lua_pushlstring(L, (char*)value, 8); +#else + uint64_t v = htole64(*(uint64_t*)value); + lua_pushlstring(L, (char*)&v, 8); +#endif + return 0; +} + +static int struct_pack(lua_State *L) +{ + uint8_t format = luaL_checkinteger(L, 2); + lua_Number value = luaL_checknumber(L, 3); + lua_settop(L, 1); + + switch(format){ + case 'i': + { + int32_t v = (int32_t)value; + pack_fixed32(L, (uint8_t*)&v); + break; + } + case 'q': + { + int64_t v = (int64_t)value; + pack_fixed64(L, (uint8_t*)&v); + break; + } + case 'f': + { + float v = (float)value; + pack_fixed32(L, (uint8_t*)&v); + break; + } + case 'd': + { + double v = (double)value; + pack_fixed64(L, (uint8_t*)&v); + break; + } + case 'I': + { + uint32_t v = (uint32_t)value; + pack_fixed32(L, (uint8_t*)&v); + break; + } + case 'Q': + { + uint64_t v = (uint64_t) value; + pack_fixed64(L, (uint8_t*)&v); + break; + } + default: + luaL_error(L, "Unknown, format"); + } + lua_call(L, 1, 0); + return 0; +} + +static size_t size_varint(const char* buffer, size_t len) +{ + size_t pos = 0; + while(buffer[pos] & 0x80){ + ++pos; + if(pos > len){ + return -1; + } + } + return pos+1; +} + +static uint64_t unpack_varint(const char* buffer, size_t len) +{ + uint64_t value = buffer[0] & 0x7f; + size_t shift = 7; + size_t pos=0; + for(pos = 1; pos < len; ++pos) + { + value |= ((uint64_t)(buffer[pos] & 0x7f)) << shift; + shift += 7; + } + return value; +} + +static int varint_decoder(lua_State *L) +{ + size_t len; + const char* buffer = luaL_checklstring(L, 1, &len); + size_t pos = luaL_checkinteger(L, 2); + + buffer += pos; + len = size_varint(buffer, len); + if(len == -1){ + luaL_error(L, "error data %s, len:%d", buffer, len); + }else{ + lua_pushnumber(L, (lua_Number)unpack_varint(buffer, len)); + lua_pushinteger(L, len + pos); + } + return 2; +} + +static int signed_varint_decoder(lua_State *L) +{ + size_t len; + const char* buffer = luaL_checklstring(L, 1, &len); + size_t pos = luaL_checkinteger(L, 2); + buffer += pos; + len = size_varint(buffer, len); + + if(len == -1){ + luaL_error(L, "error data %s, len:%d", buffer, len); + }else{ + lua_pushnumber(L, (lua_Number)(int64_t)unpack_varint(buffer, len)); + lua_pushinteger(L, len + pos); + } + return 2; +} + +static int zig_zag_encode32(lua_State *L) +{ + int32_t n = luaL_checkinteger(L, 1); + uint32_t value = (n << 1) ^ (n >> 31); + lua_pushinteger(L, value); + return 1; +} + +static int zig_zag_decode32(lua_State *L) +{ + uint32_t n = (uint32_t)luaL_checkinteger(L, 1); + int32_t value = (n >> 1) ^ - (int32_t)(n & 1); + lua_pushinteger(L, value); + return 1; +} + +static int zig_zag_encode64(lua_State *L) +{ + int64_t n = (int64_t)luaL_checknumber(L, 1); + uint64_t value = (n << 1) ^ (n >> 63); + lua_pushinteger(L, value); + return 1; +} + +static int zig_zag_decode64(lua_State *L) +{ + uint64_t n = (uint64_t)luaL_checknumber(L, 1); + int64_t value = (n >> 1) ^ - (int64_t)(n & 1); + lua_pushinteger(L, value); + return 1; +} + +static int read_tag(lua_State *L) +{ + size_t len; + const char* buffer = luaL_checklstring(L, 1, &len); + size_t pos = luaL_checkinteger(L, 2); + + buffer += pos; + len = size_varint(buffer, len); + if(len == -1){ + luaL_error(L, "error data %s, len:%d", buffer, len); + }else{ + lua_pushlstring(L, buffer, len); + lua_pushinteger(L, len + pos); + } + return 2; +} + +static const uint8_t* unpack_fixed32(const uint8_t* buffer, uint8_t* cache) +{ +#ifdef IS_LITTLE_ENDIAN + return buffer; +#else + *(uint32_t*)cache = le32toh(*(uint32_t*)buffer); + return cache; +#endif +} + +static const uint8_t* unpack_fixed64(const uint8_t* buffer, uint8_t* cache) +{ +#ifdef IS_LITTLE_ENDIAN + return buffer; +#else + *(uint64_t*)cache = le64toh(*(uint64_t*)buffer); + return cache; +#endif +} + +static int struct_unpack(lua_State *L) +{ + uint8_t format = luaL_checkinteger(L, 1); + size_t len; + const uint8_t* buffer = (uint8_t*)luaL_checklstring(L, 2, &len); + size_t pos = luaL_checkinteger(L, 3); + + buffer += pos; + uint8_t out[8]; + switch(format){ + case 'i': + { + lua_pushinteger(L, *(int32_t*)unpack_fixed32(buffer, out)); + break; + } + case 'q': + { + lua_pushnumber(L, (lua_Number)*(int64_t*)unpack_fixed64(buffer, out)); + break; + } + case 'f': + { + lua_pushnumber(L, (lua_Number)*(float*)unpack_fixed32(buffer, out)); + break; + } + case 'd': + { + lua_pushnumber(L, (lua_Number)*(double*)unpack_fixed64(buffer, out)); + break; + } + case 'I': + { + lua_pushnumber(L, *(uint32_t*)unpack_fixed32(buffer, out)); + break; + } + case 'Q': + { + lua_pushnumber(L, (lua_Number)*(uint64_t*)unpack_fixed64(buffer, out)); + break; + } + default: + luaL_error(L, "Unknown, format"); + } + return 1; +} + +static int iostring_new(lua_State* L) +{ + IOString* io = (IOString*)lua_newuserdata(L, sizeof(IOString)); + io->size = 0; + + luaL_getmetatable(L, IOSTRING_META); + lua_setmetatable(L, -2); + return 1; +} + +static int iostring_str(lua_State* L) +{ + IOString *io = checkiostring(L); + lua_pushlstring(L, io->buf, io->size); + return 1; +} + +static int iostring_len(lua_State* L) +{ + IOString *io = checkiostring(L); + lua_pushinteger(L, io->size); + return 1; +} + +static int iostring_write(lua_State* L) +{ + IOString *io = checkiostring(L); + size_t size; + const char* str = luaL_checklstring(L, 2, &size); + if(io->size + size > IOSTRING_BUF_LEN){ + luaL_error(L, "Out of range"); + } + memcpy(io->buf + io->size, str, size); + io->size += size; + return 0; +} + +static int iostring_sub(lua_State* L) +{ + IOString *io = checkiostring(L); + size_t begin = luaL_checkinteger(L, 2); + size_t end = luaL_checkinteger(L, 3); + + if(begin > end || end > io->size) + { + luaL_error(L, "Out of range"); + } + lua_pushlstring(L, io->buf + begin - 1, end - begin + 1); + return 1; +} + +static int iostring_clear(lua_State* L) +{ + IOString *io = checkiostring(L); + io->size = 0; + return 0; +} + +static const struct luaL_reg _pb [] = { + {"varint_encoder", varint_encoder}, + {"signed_varint_encoder", signed_varint_encoder}, + {"read_tag", read_tag}, + {"struct_pack", struct_pack}, + {"struct_unpack", struct_unpack}, + {"varint_decoder", varint_decoder}, + {"signed_varint_decoder", signed_varint_decoder}, + {"zig_zag_decode32", zig_zag_decode32}, + {"zig_zag_encode32", zig_zag_encode32}, + {"zig_zag_decode64", zig_zag_decode64}, + {"zig_zag_encode64", zig_zag_encode64}, + {"new_iostring", iostring_new}, + {NULL, NULL} +}; + +static const struct luaL_reg _c_iostring_m [] = { + {"__tostring", iostring_str}, + {"__len", iostring_len}, + {"write", iostring_write}, + {"sub", iostring_sub}, + {"clear", iostring_clear}, + {NULL, NULL} +}; + +int luaopen_npb (lua_State *L) +{ + luaL_newmetatable(L, IOSTRING_META); + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); + luaL_register(L, NULL, _c_iostring_m); + + luaL_register(L, "pb", _pb); + return 1; +} diff --git a/source/tools/monitor/unity/protobuf/protobuf.lua b/source/tools/monitor/unity/protobuf/protobuf.lua new file mode 100644 index 0000000000000000000000000000000000000000..321362242f73a6c6e0bc28ef96fa9e4c45a5d4f0 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/protobuf.lua @@ -0,0 +1,921 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: protobuf.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年07月29日 14时30分02秒 CST +-------------------------------------------------------------------------------- +-- + +local setmetatable = setmetatable +local rawset = rawset +local rawget = rawget +local error = error +local ipairs = ipairs +local pairs = pairs +local print = print +local table = table +local string = string +local tostring = tostring +local type = type + +local wire_format = require "wire_format" +local type_checkers = require "type_checkers" +local encoder = require "encoder" +local decoder = require "decoder" +local listener_mod = require "listener" +local containers = require "containers" +local descriptor = require "descriptor" +local FieldDescriptor = descriptor.FieldDescriptor +local text_format = require "text_format" + +module("protobuf") + +local function make_descriptor(name, descriptor, usable_key) + local meta = { + __newindex = function(self, key, value) + if usable_key[key] then + rawset(self, key, value) + else + error("error key: "..key) + end + end + }; + meta.__index = meta + meta.__call = function() + return setmetatable({}, meta) + end + + _M[name] = setmetatable(descriptor, meta); +end + + +make_descriptor("Descriptor", {}, { + name = true, + full_name = true, + filename = true, + containing_type = true, + fields = true, + nested_types = true, + enum_types = true, + extensions = true, + options = true, + is_extendable = true, + extension_ranges = true, +}) + +make_descriptor("FieldDescriptor", FieldDescriptor, { + name = true, + full_name = true, + index = true, + number = true, + type = true, + cpp_type = true, + label = true, + has_default_value = true, + default_value = true, + containing_type = true, + message_type = true, + enum_type = true, + is_extension = true, + extension_scope = true, +}) + +make_descriptor("EnumDescriptor", {}, { + name = true, + full_name = true, + values = true, + containing_type = true, + options = true +}) + +make_descriptor("EnumValueDescriptor", {}, { + name = true, + index = true, + number = true, + type = true, + options = true +}) + +-- Maps from field type to expected wiretype. +local FIELD_TYPE_TO_WIRE_TYPE = { + [FieldDescriptor.TYPE_DOUBLE] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_FLOAT] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_INT64] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_UINT64] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_INT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_FIXED64] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_FIXED32] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_BOOL] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_STRING] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_GROUP] = wire_format.WIRETYPE_START_GROUP, + [FieldDescriptor.TYPE_MESSAGE] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_BYTES] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_UINT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_ENUM] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_SFIXED32] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_SFIXED64] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_SINT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_SINT64] = wire_format.WIRETYPE_VARINT +} + +local NON_PACKABLE_TYPES = { + [FieldDescriptor.TYPE_STRING] = true, + [FieldDescriptor.TYPE_GROUP] = true, + [FieldDescriptor.TYPE_MESSAGE] = true, + [FieldDescriptor.TYPE_BYTES] = true +} + +local _VALUE_CHECKERS = { + [FieldDescriptor.CPPTYPE_INT32] = type_checkers.Int32ValueChecker(), + [FieldDescriptor.CPPTYPE_INT64] = type_checkers.Int32ValueChecker(), + [FieldDescriptor.CPPTYPE_UINT32] = type_checkers.Uint32ValueChecker(), + [FieldDescriptor.CPPTYPE_UINT64] = type_checkers.Uint32ValueChecker(), + [FieldDescriptor.CPPTYPE_DOUBLE] = type_checkers.TypeChecker({number = true}), + [FieldDescriptor.CPPTYPE_FLOAT] = type_checkers.TypeChecker({number = true}), + [FieldDescriptor.CPPTYPE_BOOL] = type_checkers.TypeChecker({boolean = true, bool = true, int=true}), + [FieldDescriptor.CPPTYPE_ENUM] = type_checkers.Int32ValueChecker(), + [FieldDescriptor.CPPTYPE_STRING] = type_checkers.TypeChecker({string = true}) +} + + +local TYPE_TO_BYTE_SIZE_FN = { + [FieldDescriptor.TYPE_DOUBLE] = wire_format.DoubleByteSize, + [FieldDescriptor.TYPE_FLOAT] = wire_format.FloatByteSize, + [FieldDescriptor.TYPE_INT64] = wire_format.Int64ByteSize, + [FieldDescriptor.TYPE_UINT64] = wire_format.UInt64ByteSize, + [FieldDescriptor.TYPE_INT32] = wire_format.Int32ByteSize, + [FieldDescriptor.TYPE_FIXED64] = wire_format.Fixed64ByteSize, + [FieldDescriptor.TYPE_FIXED32] = wire_format.Fixed32ByteSize, + [FieldDescriptor.TYPE_BOOL] = wire_format.BoolByteSize, + [FieldDescriptor.TYPE_STRING] = wire_format.StringByteSize, + [FieldDescriptor.TYPE_GROUP] = wire_format.GroupByteSize, + [FieldDescriptor.TYPE_MESSAGE] = wire_format.MessageByteSize, + [FieldDescriptor.TYPE_BYTES] = wire_format.BytesByteSize, + [FieldDescriptor.TYPE_UINT32] = wire_format.UInt32ByteSize, + [FieldDescriptor.TYPE_ENUM] = wire_format.EnumByteSize, + [FieldDescriptor.TYPE_SFIXED32] = wire_format.SFixed32ByteSize, + [FieldDescriptor.TYPE_SFIXED64] = wire_format.SFixed64ByteSize, + [FieldDescriptor.TYPE_SINT32] = wire_format.SInt32ByteSize, + [FieldDescriptor.TYPE_SINT64] = wire_format.SInt64ByteSize +} + +local TYPE_TO_ENCODER = { + [FieldDescriptor.TYPE_DOUBLE] = encoder.DoubleEncoder, + [FieldDescriptor.TYPE_FLOAT] = encoder.FloatEncoder, + [FieldDescriptor.TYPE_INT64] = encoder.Int64Encoder, + [FieldDescriptor.TYPE_UINT64] = encoder.UInt64Encoder, + [FieldDescriptor.TYPE_INT32] = encoder.Int32Encoder, + [FieldDescriptor.TYPE_FIXED64] = encoder.Fixed64Encoder, + [FieldDescriptor.TYPE_FIXED32] = encoder.Fixed32Encoder, + [FieldDescriptor.TYPE_BOOL] = encoder.BoolEncoder, + [FieldDescriptor.TYPE_STRING] = encoder.StringEncoder, + [FieldDescriptor.TYPE_GROUP] = encoder.GroupEncoder, + [FieldDescriptor.TYPE_MESSAGE] = encoder.MessageEncoder, + [FieldDescriptor.TYPE_BYTES] = encoder.BytesEncoder, + [FieldDescriptor.TYPE_UINT32] = encoder.UInt32Encoder, + [FieldDescriptor.TYPE_ENUM] = encoder.EnumEncoder, + [FieldDescriptor.TYPE_SFIXED32] = encoder.SFixed32Encoder, + [FieldDescriptor.TYPE_SFIXED64] = encoder.SFixed64Encoder, + [FieldDescriptor.TYPE_SINT32] = encoder.SInt32Encoder, + [FieldDescriptor.TYPE_SINT64] = encoder.SInt64Encoder +} + +local TYPE_TO_SIZER = { + [FieldDescriptor.TYPE_DOUBLE] = encoder.DoubleSizer, + [FieldDescriptor.TYPE_FLOAT] = encoder.FloatSizer, + [FieldDescriptor.TYPE_INT64] = encoder.Int64Sizer, + [FieldDescriptor.TYPE_UINT64] = encoder.UInt64Sizer, + [FieldDescriptor.TYPE_INT32] = encoder.Int32Sizer, + [FieldDescriptor.TYPE_FIXED64] = encoder.Fixed64Sizer, + [FieldDescriptor.TYPE_FIXED32] = encoder.Fixed32Sizer, + [FieldDescriptor.TYPE_BOOL] = encoder.BoolSizer, + [FieldDescriptor.TYPE_STRING] = encoder.StringSizer, + [FieldDescriptor.TYPE_GROUP] = encoder.GroupSizer, + [FieldDescriptor.TYPE_MESSAGE] = encoder.MessageSizer, + [FieldDescriptor.TYPE_BYTES] = encoder.BytesSizer, + [FieldDescriptor.TYPE_UINT32] = encoder.UInt32Sizer, + [FieldDescriptor.TYPE_ENUM] = encoder.EnumSizer, + [FieldDescriptor.TYPE_SFIXED32] = encoder.SFixed32Sizer, + [FieldDescriptor.TYPE_SFIXED64] = encoder.SFixed64Sizer, + [FieldDescriptor.TYPE_SINT32] = encoder.SInt32Sizer, + [FieldDescriptor.TYPE_SINT64] = encoder.SInt64Sizer +} + +local TYPE_TO_DECODER = { + [FieldDescriptor.TYPE_DOUBLE] = decoder.DoubleDecoder, + [FieldDescriptor.TYPE_FLOAT] = decoder.FloatDecoder, + [FieldDescriptor.TYPE_INT64] = decoder.Int64Decoder, + [FieldDescriptor.TYPE_UINT64] = decoder.UInt64Decoder, + [FieldDescriptor.TYPE_INT32] = decoder.Int32Decoder, + [FieldDescriptor.TYPE_FIXED64] = decoder.Fixed64Decoder, + [FieldDescriptor.TYPE_FIXED32] = decoder.Fixed32Decoder, + [FieldDescriptor.TYPE_BOOL] = decoder.BoolDecoder, + [FieldDescriptor.TYPE_STRING] = decoder.StringDecoder, + [FieldDescriptor.TYPE_GROUP] = decoder.GroupDecoder, + [FieldDescriptor.TYPE_MESSAGE] = decoder.MessageDecoder, + [FieldDescriptor.TYPE_BYTES] = decoder.BytesDecoder, + [FieldDescriptor.TYPE_UINT32] = decoder.UInt32Decoder, + [FieldDescriptor.TYPE_ENUM] = decoder.EnumDecoder, + [FieldDescriptor.TYPE_SFIXED32] = decoder.SFixed32Decoder, + [FieldDescriptor.TYPE_SFIXED64] = decoder.SFixed64Decoder, + [FieldDescriptor.TYPE_SINT32] = decoder.SInt32Decoder, + [FieldDescriptor.TYPE_SINT64] = decoder.SInt64Decoder +} + +local FIELD_TYPE_TO_WIRE_TYPE = { + [FieldDescriptor.TYPE_DOUBLE] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_FLOAT] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_INT64] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_UINT64] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_INT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_FIXED64] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_FIXED32] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_BOOL] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_STRING] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_GROUP] = wire_format.WIRETYPE_START_GROUP, + [FieldDescriptor.TYPE_MESSAGE] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_BYTES] = wire_format.WIRETYPE_LENGTH_DELIMITED, + [FieldDescriptor.TYPE_UINT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_ENUM] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_SFIXED32] = wire_format.WIRETYPE_FIXED32, + [FieldDescriptor.TYPE_SFIXED64] = wire_format.WIRETYPE_FIXED64, + [FieldDescriptor.TYPE_SINT32] = wire_format.WIRETYPE_VARINT, + [FieldDescriptor.TYPE_SINT64] = wire_format.WIRETYPE_VARINT +} + +local function IsTypePackable(field_type) + return NON_PACKABLE_TYPES[field_type] == nil +end + +local function GetTypeChecker(cpp_type, field_type) + if (cpp_type == FieldDescriptor.CPPTYPE_STRING and field_type == FieldDescriptor.TYPE_STRING) then + return type_checkers.UnicodeValueChecker() + end + return _VALUE_CHECKERS[cpp_type] +end + +local function _DefaultValueConstructorForField(field) + if field.label == FieldDescriptor.LABEL_REPEATED then + if type(field.default_value) ~= "table" or #(field.default_value) ~= 0 then + error('Repeated field default value not empty list:' .. tostring(field.default_value)) + end + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + local message_type = field.message_type + return function (message) + return containers.RepeatedCompositeFieldContainer(message._listener_for_children, message_type) + end + else + local type_checker = GetTypeChecker(field.cpp_type, field.type) + return function (message) + return containers.RepeatedScalarFieldContainer(message._listener_for_children, type_checker) + end + end + end + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + local message_type = field.message_type + return function (message) + result = message_type._concrete_class() + result._SetListener(message._listener_for_children) + return result + end + end + return function (message) + return field.default_value + end +end + +local function _AttachFieldHelpers(message_meta, field_descriptor) + local is_repeated = (field_descriptor.label == FieldDescriptor.LABEL_REPEATED) + local is_packed = (field_descriptor.has_options and field_descriptor.GetOptions().packed) + + rawset(field_descriptor, "_encoder", TYPE_TO_ENCODER[field_descriptor.type](field_descriptor.number, is_repeated, is_packed)) + rawset(field_descriptor, "_sizer", TYPE_TO_SIZER[field_descriptor.type](field_descriptor.number, is_repeated, is_packed)) + rawset(field_descriptor, "_default_constructor", _DefaultValueConstructorForField(field_descriptor)) + + local AddDecoder = function(wiretype, is_packed) + local tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) + message_meta._decoders_by_tag[tag_bytes] = TYPE_TO_DECODER[field_descriptor.type](field_descriptor.number, is_repeated, is_packed, field_descriptor, field_descriptor._default_constructor) + end + + AddDecoder(FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type], False) + if is_repeated and IsTypePackable(field_descriptor.type) then + AddDecoder(wire_format.WIRETYPE_LENGTH_DELIMITED, True) + end +end + +local function _AddEnumValues(descriptor, message_meta) + for _, enum_type in ipairs(descriptor.enum_types) do + for _, enum_value in ipairs(enum_type.values) do + message_meta._member[enum_value.name] = enum_value.number + end + end +end + +local function _InitMethod(message_meta) + return function() + local self = {} + self._cached_byte_size = 0 + self._cached_byte_size_dirty = false + self._fields = {} + self._is_present_in_parent = false + self._listener = listener_mod.NullMessageListener() + self._listener_for_children = listener_mod.Listener(self) + return setmetatable(self, message_meta) + end +end + +local function _AddPropertiesForRepeatedField(field, message_meta) + local property_name = field.name + + message_meta._getter[property_name] = function(self) + local field_value = self._fields[field] + if field_value == nil then + field_value = field._default_constructor(self) + + self._fields[field] = field_value + end + return field_value + end + + message_meta._setter[property_name] = function(self) + error('Assignment not allowed to repeated field "' .. property_name .. '" in protocol message object.') + end +end + +local function _AddPropertiesForNonRepeatedCompositeField(field, message_meta) + local property_name = field.name + local message_type = field.message_type + + message_meta._getter[property_name] = function(self) + local field_value = self._fields[field] + if field_value == nil then + field_value = message_type._concrete_class() + field_value:_SetListener(self._listener_for_children) + + self._fields[field] = field_value + end + return field_value + end + message_meta._setter[property_name] = function(self, new_value) + error('Assignment not allowed to composite field' .. property_name .. 'in protocol message object.' ) + end +end + +local function _AddPropertiesForNonRepeatedScalarField(field, message) + local property_name = field.name + local type_checker = GetTypeChecker(field.cpp_type, field.type) + local default_value = field.default_value + + message._getter[property_name] = function(self) + local value = self._fields[field] + if value ~= nil then + return self._fields[field] + else + return default_value + end + end + + message._setter[property_name] = function(self, new_value) + type_checker(new_value) + self._fields[field] = new_value + if not self._cached_byte_size_dirty then + message._member._Modified(self) + end + end +end + +local function _AddPropertiesForField(field, message_meta) + constant_name = field.name:upper() .. "_FIELD_NUMBER" + message_meta._member[constant_name] = field.number + + if field.label == FieldDescriptor.LABEL_REPEATED then + _AddPropertiesForRepeatedField(field, message_meta) + elseif field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + _AddPropertiesForNonRepeatedCompositeField(field, message_meta) + else + _AddPropertiesForNonRepeatedScalarField(field, message_meta) + end +end + +local _ED_meta = { + __index = function(self, extension_handle) + local _extended_message = rawget(self, "_extended_message") + local value = _extended_message._fields[extension_handle] + if value ~= nil then + return value + end + if extension_handle.label == FieldDescriptor.LABEL_REPEATED then + value = extension_handle._default_constructor(self._extended_message) + elseif extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + value = extension_handle.message_type._concrete_class() + value:_SetListener(_extended_message._listener_for_children) + else + return extension_handle.default_value + end + _extended_message._fields[extension_handle] = value + return value + end, + __newindex = function(self, extension_handle, value) + local _extended_message = rawget(self, "_extended_message") + if (extension_handle.label == FieldDescriptor.LABEL_REPEATED or + extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE) then + error('Cannot assign to extension "'.. extension_handle.full_name .. '" because it is a repeated or composite type.') + end + + local type_checker = GetTypeChecker(extension_handle.cpp_type, extension_handle.type) + type_checker.CheckValue(value) + _extended_message._fields[extension_handle] = value + _extended_message._Modified() + end +} + +local function _ExtensionDict(message) + local o = {} + o._extended_message = message + return setmetatable(o, _ED_meta) +end + +local function _AddPropertiesForFields(descriptor, message_meta) + for _, field in ipairs(descriptor.fields) do + _AddPropertiesForField(field, message_meta) + end + if descriptor.is_extendable then + message_meta._getter.Extensions = function(self) return _ExtensionDict(self) end + end +end + +local function _AddPropertiesForExtensions(descriptor, message_meta) + local extension_dict = descriptor._extensions_by_name + for extension_name, extension_field in pairs(extension_dict) do + local constant_name = string.upper(extension_name) .. "_FIELD_NUMBER" + message_meta._member[constant_name] = extension_field.number + end +end + +local function _AddStaticMethods(message_meta) + message_meta._member.RegisterExtension = function(extension_handle) + extension_handle.containing_type = message_meta._descriptor + _AttachFieldHelpers(message_meta, extension_handle) + + if message_meta._extensions_by_number[extension_handle.number] == nil then + message_meta._extensions_by_number[extension_handle.number] = extension_handle + else + error( + string.format('Extensions "%s" and "%s" both try to extend message type "%s" with field number %d.', + extension_handle.full_name, actual_handle.full_name, + message_meta._descriptor.full_name, extension_handle.number)) + end + message_meta._extensions_by_name[extension_handle.full_name] = extension_handle + end + + message_meta._member.FromString = function(s) + local message = message_meta._member.__call() + message.MergeFromString(s) + return message + end +end + +local function _IsPresent(descriptor, value) + if descriptor.label == FieldDescriptor.LABEL_REPEATED then + return value + elseif descriptor.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + return value._is_present_in_parent + else + return true + end +end + +local function _AddListFieldsMethod(message_descriptor, message_meta) + message_meta._member.ListFields = function (self) + local list_field = function(fields) + local f, s, v = pairs(self._fields) + local iter = function(a, i) + while true do + local descriptor, value = f(a, i) + if descriptor == nil then + return + elseif _IsPresent(descriptor, value) then + return descriptor, value + end + end + end + return iter, s, v + end + return list_field(self._fields) + end +end + +local function _AddHasFieldMethod(message_descriptor, message_meta) + local singular_fields = {} + for _, field in ipairs(message_descriptor.fields) do + if field.label ~= FieldDescriptor.LABEL_REPEATED then + singular_fields[field.name] = field + end + end + message_meta._member.HasField = function (self, field_name) + field = singular_fields[field_name] + if field == nil then + error('Protocol message has no singular "'.. field_name.. '" field.') + end + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + value = self._fields[field] + return value ~= nil and value._is_present_in_parent + else + return self._fields[field] + end + end +end + +local function _AddClearFieldMethod(message_descriptor, message_meta) + message_meta._member.ClearField = function(self, field_name) + if message_descriptor.fields_by_name[field_name] == nil then + error('Protocol message has no "' .. field_name .. '" field.') + end + + if self._fields[field] then + self._fields[field] = nil + end + message_meta._member._Modified(self) + end +end + +local function _AddClearExtensionMethod(message_meta) + message_meta._member.ClearExtension = function(self, extension_handle) + if self._fields[extension_handle] == nil then + self._fields[extension_handle] = nil + end + message_meta._member._Modified(self) + end +end + +local function _AddClearMethod(message_descriptor, message_meta) + message_meta._member.Clear = function(self) + self._fields = {} + message_meta._member._Modified(self) + end +end + +local function _AddStrMethod(message_meta) + local format = text_format.msg_format + message_meta.__tostring = function(self) + return format(self) + end +end + +local function _AddHasExtensionMethod(message_meta) + message_meta._member.HasExtension = function(self, extension_handle) + if extension_handle.label == FieldDescriptor.LABEL_REPEATED then + error(extension_handle.full_name .. ' is repeated.') + end + if extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + value = self._fields[extension_handle] + return value ~= nil and value._is_present_in_parent + else + return self._fields[extension_handle] + end + end +end + +local function _AddSetListenerMethod(message_meta) + message_meta._member._SetListener = function(self, listener) + if listener ~= nil then + self._listener = listener_mod.NullMessageListener() + else + self._listener = listener + end + end +end + +local function _AddByteSizeMethod(message_descriptor, message_meta) + message_meta._member.ByteSize = function(self) + if not self._cached_byte_size_dirty then + return self._cached_byte_size + end + local size = 0 + for field_descriptor, field_value in message_meta._member.ListFields(self) do + size = field_descriptor._sizer(field_value) + size + end + self._cached_byte_size = size + self._cached_byte_size_dirty = false + self._listener_for_children.dirty = false + return size + end +end + +local function _AddSerializeToStringMethod(message_descriptor, message_meta) + message_meta._member.SerializeToString = function(self) + if not message_meta._member.IsInitialized(self) then + error('Message is missing required fields: ' .. + table.concat(message_meta._member.FindInitializationErrors(self), ',')) + end + return message_meta._member.SerializePartialToString(self) + end + message_meta._member.SerializeToIOString = function(self, iostring) + if not message_meta._member.IsInitialized(self) then + error('Message is missing required fields: ' .. + table.concat(message_meta._member.FindInitializationErrors(self), ',')) + end + return message_meta._member.SerializePartialToIOString(self, iostring) + end +end + +local function _AddSerializePartialToStringMethod(message_descriptor, message_meta) + local concat = table.concat + local _internal_serialize = function(self, write_bytes) + for field_descriptor, field_value in message_meta._member.ListFields(self) do + field_descriptor._encoder(write_bytes, field_value) + end + end + + local _serialize_partial_to_iostring = function(self, iostring) + local w = iostring.write + local write = function(value) + w(iostring, value) + end + _internal_serialize(self, write) + return + end + + local _serialize_partial_to_string = function(self) + local out = {} + local write = function(value) + out[#out + 1] = value + end + _internal_serialize(self, write) + return concat(out) + end + + message_meta._member._InternalSerialize = _internal_serialize + message_meta._member.SerializePartialToIOString = _serialize_partial_to_iostring + message_meta._member.SerializePartialToString = _serialize_partial_to_string +end + +local function _AddMergeFromStringMethod(message_descriptor, message_meta) + local ReadTag = decoder.ReadTag + local SkipField = decoder.SkipField + local decoders_by_tag = message_meta._decoders_by_tag + + local _internal_parse = function(self, buffer, pos, pend) + message_meta._member._Modified(self) + local field_dict = self._fields + local tag_bytes, new_pos + local field_decoder + while pos ~= pend do + tag_bytes, new_pos = ReadTag(buffer, pos) + field_decoder = decoders_by_tag[tag_bytes] + if field_decoder == nil then + new_pos = SkipField(buffer, new_pos, pend, tag_bytes) + if new_pos == -1 then + return pos + end + pos = new_pos + else + pos = field_decoder(buffer, new_pos, pend, self, field_dict) + end + end + return pos + end + message_meta._member._InternalParse = _internal_parse + + local merge_from_string = function(self, serialized) + local length = #serialized + if _internal_parse(self, serialized, 0, length) ~= length then + error('Unexpected end-group tag.') + end + return length + end + message_meta._member.MergeFromString = merge_from_string + + message_meta._member.ParseFromString = function(self, serialized) + message_meta._member.Clear(self) + merge_from_string(self, serialized) + end +end + +local function _AddIsInitializedMethod(message_descriptor, message_meta) + local required_fields = {} + for _, field in ipairs(message_descriptor.fields) do + if field.label == FieldDescriptor.LABEL_REQUIRED then + required_fields[#required_fields + 1] = field + end + end + + message_meta._member.IsInitialized = function(self, errors) + for _, field in ipairs(required_fields) do + if self._fields[field] == nil or + (field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE and not self._fields[field]._is_present_in_parent) then + if errors ~= nil then + errors[#errors + 1] = message_meta._member.FindInitializationErrors(self) + end + return false + end + end + + for field, value in pairs(self._fields) do + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + if field.label == FieldDescriptor.LABEL_REPEATED then + for _, element in ipairs(value) do + if not element:IsInitialized() then + if errors ~= nil then + errors[#errors + 1] = message_meta._member.FindInitializationErrors(self) + end + return false + end + end + elseif value._is_present_in_parent and not value:IsInitialized() then + if errors ~= nil then + errors[#errors + 1] = message_meta._member.FindInitializationErrors(self) + end + return false + end + end + end + return true + end + + message_meta._member.FindInitializationErrors = function(self) + local errors = {} + + for _,field in ipairs(required_fields) do + if not message_meta._member.HasField(self, field.name) then + errors.append(field.name) + end + end + + for field, value in message_meta._member.ListFields(self) do + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE then + if field.is_extension then + name = io:format("(%s)", field.full_name) + else + name = field.name + end + if field.label == FieldDescriptor.LABEL_REPEATED then + for i, element in ipairs(value) do + prefix = io:format("%s[%d].", name, i) + sub_errors = element:FindInitializationErrors() + for _, e in ipairs(sub_errors) do + errors[#errors + 1] = prefix .. e + end + end + else + prefix = name .. "." + sub_errors = value:FindInitializationErrors() + for _, e in ipairs(sub_errors) do + errors[#errors + 1] = prefix .. e + end + end + end + end + return errors + end +end + +local function _AddMergeFromMethod(message_meta) + local LABEL_REPEATED = FieldDescriptor.LABEL_REPEATED + local CPPTYPE_MESSAGE = FieldDescriptor.CPPTYPE_MESSAGE + + message_meta._member.MergeFrom = function (self, msg) + assert(msg ~= self) + message_meta._member._Modified(self) + + local fields = self._fields + + for field, value in pairs(msg._fields) do + if field.label == LABEL_REPEATED or field.cpp_type == CPPTYPE_MESSAGE then + field_value = fields[field] + if field_value == nil then + field_value = field._default_constructor(self) + fields[field] = field_value + end + field_value:MergeFrom(value) + else + self._fields[field] = value + end + end + end +end + +local function _AddMessageMethods(message_descriptor, message_meta) + _AddListFieldsMethod(message_descriptor, message_meta) + _AddHasFieldMethod(message_descriptor, message_meta) + _AddClearFieldMethod(message_descriptor, message_meta) + if message_descriptor.is_extendable then + _AddClearExtensionMethod(message_meta) + _AddHasExtensionMethod(message_meta) + end + _AddClearMethod(message_descriptor, message_meta) +-- _AddEqualsMethod(message_descriptor, message_meta) + _AddStrMethod(message_meta) + _AddSetListenerMethod(message_meta) + _AddByteSizeMethod(message_descriptor, message_meta) + _AddSerializeToStringMethod(message_descriptor, message_meta) + _AddSerializePartialToStringMethod(message_descriptor, message_meta) + _AddMergeFromStringMethod(message_descriptor, message_meta) + _AddIsInitializedMethod(message_descriptor, message_meta) + _AddMergeFromMethod(message_meta) +end + +local function _AddPrivateHelperMethods(message_meta) + local Modified = function (self) + if not self._cached_byte_size_dirty then + self._cached_byte_size_dirty = true + self._listener_for_children.dirty = true + self._is_present_in_parent = true + self._listener:Modified() + end + end + message_meta._member._Modified = Modified + message_meta._member.SetInParent = Modified +end + +local function property_getter(message_meta) + local getter = message_meta._getter + local member = message_meta._member + + return function (self, property) + local g = getter[property] + if g then + return g(self) + else + return member[property] + end + end +end + +local function property_setter(message_meta) + local setter = message_meta._setter + + return function (self, property, value) + local s = setter[property] + if s then + s(self, value) + else + error(property .. " not found") + end + end +end + +function _AddClassAttributesForNestedExtensions(descriptor, message_meta) + local extension_dict = descriptor._extensions_by_name + for extension_name, extension_field in pairs(extension_dict) do + message_meta._member[extension_name] = extension_field + end +end + +local function Message(descriptor) + local message_meta = {} + message_meta._decoders_by_tag = {} + rawset(descriptor, "_extensions_by_name", {}) + for _, k in ipairs(descriptor.extensions) do + descriptor._extensions_by_name[k.name] = k + end + rawset(descriptor, "_extensions_by_number", {}) + for _, k in ipairs(descriptor.extensions) do + descriptor._extensions_by_number[k.number] = k + end + message_meta._descriptor = descriptor + message_meta._extensions_by_name = {} + message_meta._extensions_by_number = {} + + message_meta._getter = {} + message_meta._setter = {} + message_meta._member = {} +-- message_meta._name = descriptor.full_name + + local ns = setmetatable({}, message_meta._member) + message_meta._member.__call = _InitMethod(message_meta) + message_meta._member.__index = message_meta._member + message_meta._member.type = ns + + if rawget(descriptor, "_concrete_class") == nil then + rawset(descriptor, "_concrete_class", ns) + for k, field in ipairs(descriptor.fields) do + _AttachFieldHelpers(message_meta, field) + end + end + _AddEnumValues(descriptor, message_meta) + _AddClassAttributesForNestedExtensions(descriptor, message_meta) + _AddPropertiesForFields(descriptor, message_meta) + _AddPropertiesForExtensions(descriptor, message_meta) + _AddStaticMethods(message_meta) + _AddMessageMethods(descriptor, message_meta) + _AddPrivateHelperMethods(message_meta) + + message_meta.__index = property_getter(message_meta) + message_meta.__newindex = property_setter(message_meta) + + return ns +end + +_M.Message = Message + diff --git a/source/tools/monitor/unity/protobuf/text_format.lua b/source/tools/monitor/unity/protobuf/text_format.lua new file mode 100644 index 0000000000000000000000000000000000000000..b3e6181a0430f00e422e94d36177ccbc4b5c7380 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/text_format.lua @@ -0,0 +1,79 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: text_format.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- COMPANY: NetEase +-- CREATED: 2010年08月05日 15时14分13秒 CST +-------------------------------------------------------------------------------- +-- +local string = string +local math = math +local print = print +local getmetatable = getmetatable +local table = table +local ipairs = ipairs +local tostring = tostring + +local descriptor = require "descriptor" + +module "text_format" + +function format(buffer) + local len = string.len( buffer ) + for i = 1, len, 16 do + local text = "" + for j = i, math.min( i + 16 - 1, len ) do + text = string.format( "%s %02x", text, string.byte( buffer, j ) ) + end + print( text ) + end +end + +local FieldDescriptor = descriptor.FieldDescriptor + +msg_format_indent = function(write, msg, indent) + for field, value in msg:ListFields() do + local print_field = function(field_value) + local name = field.name + write(string.rep(" ", indent)) + if field.type == FieldDescriptor.TYPE_MESSAGE then + local extensions = getmetatable(msg)._extensions_by_name + if extensions[field.full_name] then + write("[" .. name .. "] {\n") + else + write(name .. " {\n") + end + msg_format_indent(write, field_value, indent + 4) + write(string.rep(" ", indent)) + write("}\n") + else + write(string.format("%s: %s\n", name, tostring(field_value))) + end + end + if field.label == FieldDescriptor.LABEL_REPEATED then + for _, k in ipairs(value) do + print_field(k) + end + else + print_field(value) + end + end +end + +function msg_format(msg) + local out = {} + local write = function(value) + out[#out + 1] = value + end + msg_format_indent(write, msg, 0) + return table.concat(out) +end + diff --git a/source/tools/monitor/unity/protobuf/type_checkers.lua b/source/tools/monitor/unity/protobuf/type_checkers.lua new file mode 100644 index 0000000000000000000000000000000000000000..6fe22fde2fe162997428d7f8267be58f5a079acc --- /dev/null +++ b/source/tools/monitor/unity/protobuf/type_checkers.lua @@ -0,0 +1,71 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: type_checkers.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- +-- COMPANY: NetEase +-- CREATED: 2010年07月29日 19时30分37秒 CST +-------------------------------------------------------------------------------- +-- + +local type = type +local error = error +local string = string + +module "type_checkers" +function TypeChecker(acceptable_types) + local acceptable_types = acceptable_types + + return function(proposed_value) + local t = type(proposed_value) + if acceptable_types[type(proposed_value)] == nil then + error(string.format('%s has type %s, but expected one of: %s', + proposed_value, type(proposed_value), acceptable_types)) + end + end +end + +function Int32ValueChecker() + local _MIN = -2147483648 + local _MAX = 2147483647 + return function(proposed_value) + if type(proposed_value) ~= 'number' then + error(string.format('%s has type %s, but expected one of: number', + proposed_value, type(proposed_value))) + end + if _MIN > proposed_value or proposed_value > _MAX then + error('Value out of range: ' .. proposed_value) + end + end +end + +function Uint32ValueChecker(IntValueChecker) + local _MIN = 0 + local _MAX = 0xffffffff + + return function(proposed_value) + if type(proposed_value) ~= 'number' then + error(string.format('%s has type %s, but expected one of: number', + proposed_value, type(proposed_value))) + end + if _MIN > proposed_value or proposed_value > _MAX then + error('Value out of range: ' .. proposed_value) + end + end +end + +function UnicodeValueChecker() + return function (proposed_value) + if type(proposed_value) ~= 'string' then + error(string.format('%s has type %s, but expected one of: string', proposed_value, type(proposed_value))) + end + end +end diff --git a/source/tools/monitor/unity/protobuf/wire_format.lua b/source/tools/monitor/unity/protobuf/wire_format.lua new file mode 100644 index 0000000000000000000000000000000000000000..c58f918f97071f8498d81895451a3c3beb794e36 --- /dev/null +++ b/source/tools/monitor/unity/protobuf/wire_format.lua @@ -0,0 +1,137 @@ +-- +-------------------------------------------------------------------------------- +-- FILE: wire_format.lua +-- DESCRIPTION: protoc-gen-lua +-- Google's Protocol Buffers project, ported to lua. +-- https://code.google.com/p/protoc-gen-lua/ +-- +-- Copyright (c) 2010 , 林卓毅 (Zhuoyi Lin) netsnail@gmail.com +-- All rights reserved. +-- +-- Use, modification and distribution are subject to the "New BSD License" +-- as listed at . +-- COMPANY: NetEase +-- CREATED: 2010年07月30日 15时59分53秒 CST +-------------------------------------------------------------------------------- +-- + +local pb = require "npb" +module "wire_format" + +WIRETYPE_VARINT = 0 +WIRETYPE_FIXED64 = 1 +WIRETYPE_LENGTH_DELIMITED = 2 +WIRETYPE_START_GROUP = 3 +WIRETYPE_END_GROUP = 4 +WIRETYPE_FIXED32 = 5 +_WIRETYPE_MAX = 5 + + +-- yeah, we don't need uint64 +local function _VarUInt64ByteSizeNoTag(uint64) + if uint64 <= 0x7f then return 1 end + if uint64 <= 0x3fff then return 2 end + if uint64 <= 0x1fffff then return 3 end + if uint64 <= 0xfffffff then return 4 end + return 5 +end + +function PackTag(field_number, wire_type) + return field_number * 8 + wire_type +end + +function UnpackTag(tag) + local wire_type = tag % 8 + return (tag - wire_type) / 8, wire_type +end + +ZigZagEncode32 = pb.zig_zag_encode32 +ZigZagDecode32 = pb.zig_zag_decode32 +ZigZagEncode64 = pb.zig_zag_encode64 +ZigZagDecode64 = pb.zig_zag_decode64 + +function Int32ByteSize(field_number, int32) + return Int64ByteSize(field_number, int32) +end + +function Int32ByteSizeNoTag(int32) + return _VarUInt64ByteSizeNoTag(int32) +end + +function Int64ByteSize(field_number, int64) + return UInt64ByteSize(field_number, int64) +end + +function UInt32ByteSize(field_number, uint32) + return UInt64ByteSize(field_number, uint32) +end + +function UInt64ByteSize(field_number, uint64) + return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64) +end + +function SInt32ByteSize(field_number, int32) + return UInt32ByteSize(field_number, ZigZagEncode(int32)) +end + +function SInt64ByteSize(field_number, int64) + return UInt64ByteSize(field_number, ZigZagEncode(int64)) +end + +function Fixed32ByteSize(field_number, fixed32) + return TagByteSize(field_number) + 4 +end + +function Fixed64ByteSize(field_number, fixed64) + return TagByteSize(field_number) + 8 +end + +function SFixed32ByteSize(field_number, sfixed32) + return TagByteSize(field_number) + 4 +end + +function SFixed64ByteSize(field_number, sfixed64) + return TagByteSize(field_number) + 8 +end + +function FloatByteSize(field_number, flt) + return TagByteSize(field_number) + 4 +end + +function DoubleByteSize(field_number, double) + return TagByteSize(field_number) + 8 +end + +function BoolByteSize(field_number, b) + return TagByteSize(field_number) + 1 +end + +function EnumByteSize(field_number, enum) + return UInt32ByteSize(field_number, enum) +end + +function StringByteSize(field_number, string) + return BytesByteSize(field_number, string) +end + +function BytesByteSize(field_number, b) + return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(#b) + #b +end + +function MessageByteSize(field_number, message) + return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(message.ByteSize()) + message.ByteSize() +end + +function MessageSetItemByteSize(field_number, msg) + local total_size = 2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3) + total_size = total_size + _VarUInt64ByteSizeNoTag(field_number) + local message_size = msg.ByteSize() + total_size = total_size + _VarUInt64ByteSizeNoTag(message_size) + total_size = total_size + message_size + return total_size +end + +function TagByteSize(field_number) + return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0)) +end + diff --git a/source/tools/monitor/unity/test/curl/beaver/beavers.lua b/source/tools/monitor/unity/test/curl/beaver/beavers.lua new file mode 100644 index 0000000000000000000000000000000000000000..619c4334c48adbd18edb9f9b05638474233f5a4e --- /dev/null +++ b/source/tools/monitor/unity/test/curl/beaver/beavers.lua @@ -0,0 +1,108 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/13 3:31 PM +--- + + +local socket = require("socket") +local system = require("common.system") +require("common.class") + +local Cbeavers = class("beavers") + +function Cbeavers:_init_(port, tmo, ip, backlog) + port = port or 8398 + self._tmo = tmo or 10 + + self._server = self:_install_server(port, ip, backlog) + + self._ss = {} + self._cos = {} + -- self._tmo = {} + table.insert(self._ss, self._server) +end + +function Cbeavers:_del_() + for _, s in ipairs(self._ss) do + s:close() + end + print("release server.") +end + +function Cbeavers:_install_server(port, ip, backlog) + ip = ip or "localhost" + backlog = backlog or 128 + + local server = socket.tcp() + local res, err = server:bind(ip, port) + + if res ~= 1 then + server:close() + error("bind return: " .. err) + end + + res, err = server:listen(backlog) + if res ~= 1 then + server:close() + error("listen return: " .. err) + end + + return server +end + +function Cbeavers:co_exit(s) + table.remove(self._cts, s:getfd()) + local index = system:valueIndex(self._ss, s) + if index > 0 then + table.remove(self._cts, index) + end +end + +function Cbeavers:_proc(s) + print("thread start.") + _, s = coroutine.yield() + print("thread next.") + s:settimeout(0, '') + while true do + local line, err = s:receive("*") + print("thread read.") + print(line, err, s:getfd()) + if line then + s:send("echo: " .. line) + coroutine.yield() + else + print("co exit.." .. err) + s:close() + self:co_exit(s) + break + end + end +end + +function Cbeavers:poll() + while true do + local reads, _, err = socket.select(self._ss, nil) + if err then + print("socket select return " .. err) + end + for _, read in pairs(reads) do + if type(read) == "number" then + break + elseif read == self._server then + local s = read:accept() + print("accept " .. s:getfd()) + table.insert(self._ss, s) + local co = coroutine.create(function(o, s) self._proc(o, s) end) + self._cos[s:getfd()] = co + coroutine.resume(co, self, s) + else + print("read " .. read:getfd()) + local co = self._cos[read:getfd()] + coroutine.resume(co, self, read) + end + end + end +end + +return Cbeavers \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/beaver/pyClient.py b/source/tools/monitor/unity/test/curl/beaver/pyClient.py new file mode 100644 index 0000000000000000000000000000000000000000..5f1a98ab86d311cf310e5d5d7a256000f0a5f019 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/beaver/pyClient.py @@ -0,0 +1,11 @@ + +import socket +server_address = ('localhost', 8398) + +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(server_address) +sock.send("hello") +print(sock.recv(100)) +sock.send("world.") +print(sock.recv(100)) +sock.close() diff --git a/source/tools/monitor/unity/test/curl/beaver/test.lua b/source/tools/monitor/unity/test/curl/beaver/test.lua new file mode 100644 index 0000000000000000000000000000000000000000..501968de8502f0990ce97b0ff90d75b76038905b --- /dev/null +++ b/source/tools/monitor/unity/test/curl/beaver/test.lua @@ -0,0 +1,12 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/13 3:34 PM +--- + +package.path = package.path .. ";../../../?.lua;" + +local Cbeavers = require("beavers") + +local bservers = Cbeavers.new() +bservers:poll() diff --git a/source/tools/monitor/unity/test/curl/beaverApi.py b/source/tools/monitor/unity/test/curl/beaverApi.py index 9051077c01396fc9cba4f8d6c9d55505b45928b0..e2db37c3739151f7408c5cf682adfe550cad5f7f 100644 --- a/source/tools/monitor/unity/test/curl/beaverApi.py +++ b/source/tools/monitor/unity/test/curl/beaverApi.py @@ -8,12 +8,12 @@ def randomNum(): return random.randint(0, 1000) / 100.0 -def post_test(): +def post_test(session): url = "http://127.0.0.1:8400/api/" mod = random.choice(("sum", "sub")) url += mod d = {"num1": randomNum(), "num2": randomNum()} - res = requests.post(url, json=d) + res = session.post(url, json=d) ret = res.content.decode() if mod == "sum": vLocal = d["num1"] + d["num2"] @@ -25,6 +25,7 @@ def post_test(): if __name__ == "__main__": + session = requests.session() while True: - post_test() + post_test(session) time.sleep(random.randint(1, 100)/100.0) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/instance.lua b/source/tools/monitor/unity/test/curl/instance.lua new file mode 100644 index 0000000000000000000000000000000000000000..0c9a940ed5780f9042e585860097d7516fa8c19f --- /dev/null +++ b/source/tools/monitor/unity/test/curl/instance.lua @@ -0,0 +1,13 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/10 12:39 AM +--- + +package.path = package.path .. ";../../common/?.lua;" +package.path = package.path .. ";../../httplib/?.lua;" +local ChttpCli = require("httpCli") + +local cli = ChttpCli.new() +local res = cli:get("http://100.100.100.200/latest/meta-data/instance-id") +print(res.body) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/localBeaver/localBeaver.lua b/source/tools/monitor/unity/test/curl/localBeaver/localBeaver.lua new file mode 100644 index 0000000000000000000000000000000000000000..5f086c52acb0e214d90093665e5f8ea5bb0b387e --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/localBeaver.lua @@ -0,0 +1,226 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/14 3:14 PM +--- + +local unistd = require("posix.unistd") +local socket = require("posix.sys.socket") +local system = require("common.system") +require("common.class") + +local CLocalBeaver = class("poBeaver") + +function CLocalBeaver:_init_(port, ip, backlog) + port = port or 8398 + + self._bfd = self:_install_fd(port, ip, backlog) + self._efd = self:_installFFI() + + self._cos = {} + self._last = os.time() + self._tmos = {} + + self._once = true +end + +function CLocalBeaver:_del_() + if self._efd then + self._cffi.deinit(self._efd) + end + if self._bfd then + unistd.close(self._bfd) + end +end + +local function posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + +function CLocalBeaver:_installTmo(fd) + self._tmos[fd] = os.time() +end + +function CLocalBeaver:_checkTmo() + local now = os.time() + if now - self._last >= 10 then + -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety + local tmos = system:dictCopy(self._tmos) + for fd, t in pairs(tmos) do + if now - t >= 10 then + local e = self._ffi.new("native_event_t") + e.ev_close = 1 + e.fd = fd + local co = self._cos[fd] + print("close " .. fd) + coroutine.resume(co, e) + end + end + self._last = now + end +end + +function CLocalBeaver:_installFFI() + local ffi = require("native.beavercffi") + + self._ffi = ffi.ffi + self._cffi = ffi.cffi + + local efd = self._cffi.init(self._bfd) + assert(efd > 0) + return efd +end + +function CLocalBeaver:_install_fd(port, ip, backlog) + ip = ip or "0.0.0.0" + backlog = backlog or 100 + + local fd, res, err, errno + fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + if fd then -- for socket + res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) + if res then -- for bind + res, err, errno = socket.listen(fd, backlog) + if res then -- for listen + return fd + else + posixError("socket listen failed", err, errno) + end + else -- for bind failed + unistd.close(fd) + posixError("socket bind failed", err, errno) + end + else -- socket failed + posixError("create socket failed", err, errno) + end +end + +function CLocalBeaver:read(fd, maxLen) + maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max + local function readFd() + local e = coroutine.yield() + if e.ev_close > 0 then + return nil + elseif e.ev_in > 0 then + local s, err, errno + s, err, errno = socket.recv(fd, maxLen) + if s then + maxLen = maxLen - #s + return s + else + posixError("socket recv error", err, errno) + end + else + print(system:dump(e)) + end + return nil + end + return readFd +end + +function CLocalBeaver:_readStream(fd) + local sockRead = self:read(fd) + local res = "" + while true do + local s = sockRead() + if s then + if #s > 0 then + res = res .. s + if string.find(res, "\r\n") then + return res + end + else + return nil + end + else + return nil + end + end +end + +function CLocalBeaver:_proc(fd) + while true do + local s = self:_readStream(fd) + if s then + socket.send(fd, string.format("echo: %s", s)) + else + self:co_exit(fd) + break + end + end +end + +function CLocalBeaver:co_add(fd) + local res = self._cffi.add_fd(self._efd, fd) + assert(res >= 0) + + local co = coroutine.create(function(o, fd) self._proc(o, fd) end) + self._cos[fd] = co + local res, msg = coroutine.resume(co, self, fd) + assert(res, msg) +end + +function CLocalBeaver:co_exit(fd) + local res = self._cffi.del_fd(self._efd, fd) + assert(res >= 0) + + self._cos[fd] = nil + self._tmos[fd] = nil +end + +function CLocalBeaver:accept(fd, e) + if e.ev_close > 0 then + error("should close bind fd.") + else + local nfd, err, errno = socket.accept(fd) + if nfd then + self:co_add(nfd) + self:_installTmo(nfd) + else + posixError("accept new socket failed", err, errno) + end + end +end + +function CLocalBeaver:_poll(bfd, nes) + for i = 0, nes.num - 1 do + local e = nes.evs[i]; + local fd = e.fd + if fd == bfd then + self:accept(fd, e) + else + local co = self._cos[fd] + assert(co, string.format("fd: %d not setup.", fd)) + self:_installTmo(fd) + local res, msg = coroutine.resume(co, e) + assert(res, msg) + end + end + self:_checkTmo() +end + +function CLocalBeaver:poll() + assert(self._once, "poll loop only run once time.") + self._once = false + + local bfd = self._bfd + local efd = self._efd + while true do + local nes = self._ffi.new("native_events_t") + local res = self._cffi.poll_fds(efd, 10, nes) + + if res < 0 then + break + end + + self:_poll(bfd, nes) + end + + for fd in pairs(self._cos) do + local res = self._cffi.del_fd(self._efd, fd) + assert(res >= 0) + end +end + +return CLocalBeaver \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/localBeaver/ltest.lua b/source/tools/monitor/unity/test/curl/localBeaver/ltest.lua new file mode 100644 index 0000000000000000000000000000000000000000..4fb17cf2aad54aacd09809b74663ee711bc81df8 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/ltest.lua @@ -0,0 +1,12 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/14 3:27 PM +--- +--- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./native/ +package.path = package.path .. ";../../../?.lua;" + +local CLocalBeaver = require("localBeaver") + +local bserver = CLocalBeaver.new() +bserver:poll() \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/localBeaver/native/Makefile b/source/tools/monitor/unity/test/curl/localBeaver/native/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..142def1079b774e237496fceef86f000c724bbc2 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/native/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := local_beaver.o +SO := liblbeaver.so + +all: $(SO) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/localBeaver/native/beavercffi.lua b/source/tools/monitor/unity/test/curl/localBeaver/native/beavercffi.lua new file mode 100644 index 0000000000000000000000000000000000000000..b8b7fa043c977d8aea210b4f5c099a484e23d82a --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/native/beavercffi.lua @@ -0,0 +1,28 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/14 2:59 PM +--- + +local ffi = require("ffi") +local cffi = ffi.load('lbeaver') +ffi.cdef [[ +typedef struct native_event { + int fd; + short int ev_in; + short int ev_close; +}native_event_t; + +typedef struct native_events { + int num; + native_event_t evs[64]; +}native_events_t; + +int init(int listen_fd); +int add_fd(int efd, int fd); +int del_fd(int efd, int fd); +int poll_fds(int efd, int tmo, native_events_t* nes); +void deinit(int efd); +]] + +return {ffi = ffi, cffi=cffi} diff --git a/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.c b/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.c new file mode 100644 index 0000000000000000000000000000000000000000..4d8f56ed31bee80721314239a7096e2d25879ada --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.c @@ -0,0 +1,132 @@ +// +// Created by 廖肇燕 on 2023/2/14. +// + +#include "local_beaver.h" +#include +#include +#include +#include +#include +#include +#include + +static int socket_non_blocking(int sfd) +{ + int flags, res; + + flags = fcntl(sfd, F_GETFL); + if (flags < 0) { + perror("error : cannot get socket flags!\n"); + return flags; + } + + flags |= O_NONBLOCK; + res = fcntl(sfd, F_SETFL, flags); + if (res < 0) { + perror("error : cannot set socket flags!\n"); + return res; + } + + return 0; +} + +static int epoll_add(int efd, int fd) { + struct epoll_event event; + int res; + + event.events = EPOLLIN; + event.data.fd = fd; + + res = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &event); + if (res < 0) { + perror("error : can not add event to epoll!\n"); + } + return res; +} + +static int epoll_del(int efd, int fd) { + int res; + + res = epoll_ctl(efd, EPOLL_CTL_DEL, fd, NULL); + if (res < 0) { + perror("error : can not del event to epoll!\n"); + } + return res; +} + +int init(int listen_fd) { + int efd; + int res; + + efd = epoll_create(NATIVE_EVENT_MAX); + if (efd < 0) { + perror("error : cannot create epoll!\n"); + exit(1); + } + + res = epoll_add(efd, listen_fd); + if (res < 0) { + goto end_epoll_add; + } + return efd; + + end_epoll_add: + return res; +} + +int add_fd(int efd, int fd) { + int res; + + res = socket_non_blocking(fd); + if (res < 0) { + goto end_socket_non_blocking; + } + + res = epoll_add(efd, fd); + if (res < 0) { + goto end_epoll_add; + } + return res; + + end_socket_non_blocking: + end_epoll_add: + return res; +} + +int del_fd(int efd, int fd) { + int res; + res = epoll_del(efd, fd); + + close(fd); + return res; +} + +int poll_fds(int efd, int tmo, native_events_t* nes) { + struct epoll_event events[NATIVE_EVENT_MAX]; + int i, res; + + res = epoll_wait(efd, events, NATIVE_EVENT_MAX, tmo * 1000); + if (res < 0) { + perror("error : epoll failed!\n"); + return res; + } + nes->num = res; + for (i = 0; i < res; i ++) { + nes->evs[i].fd = events[i].data.fd; + + if ( (events[i].events & EPOLLERR) || + (events[i].events & EPOLLHUP) || + !(events[i].events & EPOLLIN) ) { + nes->evs[i].ev_close = 1; + } + if (events[i].events & EPOLLIN) { + nes->evs[i].ev_in = 1; + } + } + return 0; +} + +void deinit(int efd) { + close(efd); +} diff --git a/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.h b/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.h new file mode 100644 index 0000000000000000000000000000000000000000..39410f8c5ce858117d306ba312d7f1b3571c0546 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/localBeaver/native/local_beaver.h @@ -0,0 +1,21 @@ +// +// Created by 廖肇燕 on 2023/2/14. +// + +#ifndef UNITY_LOCAL_BEAVER_H +#define UNITY_LOCAL_BEAVER_H + +#define NATIVE_EVENT_MAX 64 + +typedef struct native_event { + int fd; + short int ev_in; + short int ev_close; +}native_event_t; + +typedef struct native_events { + int num; + native_event_t evs[NATIVE_EVENT_MAX]; +}native_events_t; + +#endif //UNITY_LOCAL_BEAVER_H diff --git a/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua b/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua new file mode 100644 index 0000000000000000000000000000000000000000..e44e05358d81a7e698a8f20531bab507a7b374a4 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua @@ -0,0 +1,215 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/13 7:26 PM +--- + +local bit = require("bit") +local fcntl = require("posix.fcntl") +local poll = require("posix.poll") +local unistd = require("posix.unistd") +local socket = require("posix.sys.socket") +local system = require("common.system") +require("common.class") + +local CpoBeaver = class("poBeaver") + +function CpoBeaver:_init_(port, ip, backlog) + port = port or 8398 + self._server = self:_install_fd(port, ip, backlog) + + self._fds = {} + self._cos = {} + + self:_install_ev(self._server) +end + +function CpoBeaver:_del_() + for fd in pairs(self._fds) do + unistd.close(fd) + end +end + +local function posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + +function CpoBeaver:_install_ev(fd) + self._fds[fd] = {events={IN=true, HUP=true, ERR=true, NVAL=true}} +end + +function CpoBeaver:_install_fd(port, ip, backlog) + ip = ip or "0.0.0.0" + backlog = backlog or 100 + + local fd, res, err, errno + fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + if fd then -- for socket + res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) + if res then -- for bind + res, err, errno = socket.listen(fd, backlog) + if res then -- for listen + return fd + else + posixError("socket listen failed", err, errno) + end + else -- for bind failed + unistd.close(fd) + posixError("socket bind failed", err, errno) + end + else -- socket failed + posixError("create socket failed", err, errno) + end +end + +local function fdNonBlocking(fd) + local res, err, errno + local flag, err, errno = fcntl.fcntl(fd, fcntl.F_GETFL) + if flag then + res, err, errno = fcntl.fcntl(fd, fcntl.F_SETFL, bit.bor(flag, fcntl.O_NONBLOCK)) + if res then + return 0 + else + posixError("fcntl set failed", err, errno) + end + else + posixError("fcntl get failed", err, errno) + end +end + + +function CpoBeaver:read(fd, maxLen) + maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max + local function readFd() + local e = coroutine.yield() + if e.HUP or e.ERR or e.NVAL then + return nil + elseif e.IN then + local s, err, errno + s, err, errno = socket.recv(fd, maxLen) + if s then + maxLen = maxLen - #s + return s + else + posixError("socket recv error", err, errno) + end + else + print(system:dump(e)) + end + return nil + end + return readFd +end + +function CpoBeaver:co_exit(fd) + unistd.close(fd) + self._cos[fd] = nil + self._fds[fd] = nil +end + +function CpoBeaver:co_add(fd) + self:_install_ev(fd) + local co = coroutine.create(function(o, fd) self._proc(o, fd) end) + self._cos[fd] = co + coroutine.resume(co, self, fd) +end + +function CpoBeaver:_readStream(fd) + local sockRead = self:read(fd) + local res = "" + while true do + local s = sockRead() + if s then + if #s > 0 then + res = res .. s + if string.find(res, "\r\n") then + return res + end + else + return nil + end + else + return nil + end + end +end + +function CpoBeaver:_proc(fd) + --print("open " .. fd) + while true do + local s = self:_readStream(fd) + if s then + socket.send(fd, "echo: " .. s) + else + --print("exit" .. fd) + self:co_exit(fd) + break + end + end +end + +function CpoBeaver:checks(num) + local fds = {} + local cnt = 0 + local newEv = nil + + for fd, es in pairs(self._fds) do + local e = es.revents + if e.IN or e.HUP or e.ERR or e.NVAL then + if fd == self._server then + newEv = e + else + fds[fd] = e + end + cnt = cnt + 1 + if cnt >= num then + break + end + end + end + return newEv, fds +end + +function CpoBeaver:poll() + local res, err, errno + + while true do + res, err, errno = poll.poll(self._fds, 5 * 1000 * 1000) + if res > 0 then + local newEv, fds = self:checks(res) + if newEv then -- accept + res = self:accept(self._server, newEv) + if not res then -- accept corrupt. + return nil + end + end + + for fd, e in pairs(fds) do -- fd read. + local co = self._cos[fd] + if co then + coroutine.resume(co, e) + end + end + elseif not res then -- for poll failed. + posixError("poll failed.", err, errno) + end + end +end + +function CpoBeaver:accept(fd, e) + if e.HUP or e.ERR then + return nil + elseif e.IN then + local nfd, err, errno = socket.accept(fd) + if nfd then + fdNonBlocking(nfd) + self:co_add(nfd) + else + posixError("accept new socket failed", err, errno) + end + end + return 0 +end + +return CpoBeaver diff --git a/source/tools/monitor/unity/test/curl/poBeaver/poThreads.py b/source/tools/monitor/unity/test/curl/poBeaver/poThreads.py new file mode 100644 index 0000000000000000000000000000000000000000..fe3675791d0ffbf7ae68d5bf60b97a6a33e6df5c --- /dev/null +++ b/source/tools/monitor/unity/test/curl/poBeaver/poThreads.py @@ -0,0 +1,94 @@ +import os +import socket +import random +import time +import signal +from threading import Thread + + +class CcoThread(Thread): + def __init__(self, addr, tid): + super(CcoThread, self).__init__() + self.setDaemon(True) + self._addr = addr + self._tid = tid + self.start() + + def is_socket_closed(self, sock): + try: + # this will try to read bytes without blocking and also without + # removing them from buffer (peek only) + data = sock.recv(16, socket.MSG_DONTWAIT | socket.MSG_PEEK) + if len(data) == 0: + return True + except BlockingIOError: + return False # socket is open and reading from it would block + except ConnectionResetError: + return True # socket was closed for some other reason + except Exception as e: + print("unexpected exception when checking if a socket is closed", str(e)) + return False + return False + + def _work(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(self._addr) + print("close test.") + sock.send("a" * 64) + time.sleep(21) + if not self.is_socket_closed(sock): + print("failed.") + os.kill(os.getpid(), 9) + print("sock is closed: ", self.is_socket_closed(sock)) + sock.close() + + def run(self): + print("thread working.", self._tid) + while True: + self._work() + time.sleep(1) + + +class CpoThread(Thread): + def __init__(self, addr, tid): + super(CpoThread, self).__init__() + self.setDaemon(True) + self._addr = addr + self._tid = tid + self.start() + + def _work(self, loop): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(self._addr) + for _ in range(loop): + sock.send("a" * 8192) + send = "head%d" % self._tid + sock.send(send) + sock.send("\r\n") + s = "" + while len(s) < 8192 + 6: + s += sock.recv(16384) + assert s.endswith(send + "\r\n"), s + sock.send("world.") + time.sleep(0.01) + sock.send("\r\n") + s =sock.recv(100) + assert(s.endswith("world.\r\n")) + time.sleep(0.01) + sock.close() + + def run(self): + print("thread working.", self._tid) + while True: + var = random.randint(10, 50) + self._work(var) + time.sleep(0.1) + + +ts = [] +addr = ('localhost', 8398) +ts.append(CcoThread(addr, 100)) +for i in range(30): + ts.append(CpoThread(addr, i)) + time.sleep(1) +signal.pause() diff --git a/source/tools/monitor/unity/test/curl/poBeaver/ptest.lua b/source/tools/monitor/unity/test/curl/poBeaver/ptest.lua new file mode 100644 index 0000000000000000000000000000000000000000..baa54115a4998d9455e000aa29d4d957c9c5a908 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/poBeaver/ptest.lua @@ -0,0 +1,12 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/13 10:26 PM +--- + +package.path = package.path .. ";../../../?.lua;" + +local CpoBeaver = require("poBeaver") + +local bserver = CpoBeaver.new() +bserver:poll() diff --git a/source/tools/monitor/unity/test/curl/sls/pylz4.py b/source/tools/monitor/unity/test/curl/sls/pylz4.py new file mode 100644 index 0000000000000000000000000000000000000000..c463b98f143774adef2fb14199d7bea43db7494f --- /dev/null +++ b/source/tools/monitor/unity/test/curl/sls/pylz4.py @@ -0,0 +1,6 @@ +from lz4 import block +data = "hello lz4, hello lz4, hello lz4, hello lz4, hello lz4, hello lz4".encode() +compressed = block.compress(data, store_size=False) +print() +print(f'out/in: {len(compressed)}/{len(data)} Bytes') +print(f'Compression ratio: {len(compressed)/len(data):.2%}') \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/sls/slsTest.lua b/source/tools/monitor/unity/test/curl/sls/slsTest.lua index 146019865218e2443f5d0a02853a7b6628c43396..a9e2e5752bc2ddbab80cb2046637f32eb83a26ec 100644 --- a/source/tools/monitor/unity/test/curl/sls/slsTest.lua +++ b/source/tools/monitor/unity/test/curl/sls/slsTest.lua @@ -7,10 +7,10 @@ package.path = package.path .. ";../../../common/?.lua;" package.path = package.path .. ";../../../httplib/?.lua;" +package.path = package.path .. ";../../../protobuf/?.lua;" + local CslsCli = require("slsCli") -local cli = CslsCli.new("cn-heyuan.log.aliyuncs.com", - "netinfo-heyuan", "moon", - "key", "pswd") +local cli = CslsCli.new("xxx") cli:putLog("abc", "hello.") diff --git a/source/tools/monitor/unity/test/curl/tlz4.lua b/source/tools/monitor/unity/test/curl/tlz4.lua index 136a7d808fc4495ec3c631a8ae4f04f7b36bc48b..ddf2e9ebe9abe777f933c65a6423745e96a6a5a1 100644 --- a/source/tools/monitor/unity/test/curl/tlz4.lua +++ b/source/tools/monitor/unity/test/curl/tlz4.lua @@ -9,9 +9,12 @@ package.path = package.path .. ";../../common/?.lua;" local lz4 = require("lz4") +local s = "hello lz4, hello lz4, hello lz4, hello lz4, hello lz4, hello lz4" +local cmp = lz4.compress(s) +print(#s, #cmp) local data = "hello lz4" local errmsg, compressed_data, decompressed_data compressed_data, errmsg = lz4.compress(data) decompressed_data, errmsg = lz4.decompress(compressed_data) assert(decompressed_data == data) -print("lz4 test ok.") \ No newline at end of file +print("lz4 test ok.") diff --git a/source/tools/monitor/unity/test/host/hostIp.lua b/source/tools/monitor/unity/test/host/hostIp.lua new file mode 100644 index 0000000000000000000000000000000000000000..1b0266a630244aefccb2d605c291163465482e31 --- /dev/null +++ b/source/tools/monitor/unity/test/host/hostIp.lua @@ -0,0 +1,20 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/10 12:16 AM +--- + +local socket = require("socket") + +local function getAdd(hostName) + local _, resolved = socket.dns.toip(hostName) + local listTab = {} + for _, v in pairs(resolved.ip) do + table.insert(listTab, v) + end + return listTab +end + +print(unpack(getAdd("localhost"))) +print(socket.dns.gethostname()) +print(unpack(getAdd(socket.dns.gethostname()))) diff --git a/source/tools/monitor/unity/test/lab/foxTSDB.lua b/source/tools/monitor/unity/test/lab/foxTSDB.lua index 7d37d9e53a03a51164bc16bfb4baa5bc920989c7..cd2d65f436fb620795d968181d7aaffcc0bc0f0f 100644 --- a/source/tools/monitor/unity/test/lab/foxTSDB.lua +++ b/source/tools/monitor/unity/test/lab/foxTSDB.lua @@ -138,7 +138,6 @@ function CfoxTSDB:write(buff) local now = self:get_us() local date = self:getDateFrom_us(now) local stream = snappy.compress(buff) - print("write for time: ", now) assert(self.cffi.fox_write(self._man, date, now, self.ffi.string(stream, #stream), #stream) == 0) --assert(self.cffi.fox_write(self._man, date, now, self.ffi.string(buff), #buff) == 0) end diff --git a/source/tools/monitor/unity/test/string/tLineParse.lua b/source/tools/monitor/unity/test/string/tLineParse.lua new file mode 100644 index 0000000000000000000000000000000000000000..78c5961c8c3095d05b8723e2f2d80ef3569e3861 --- /dev/null +++ b/source/tools/monitor/unity/test/string/tLineParse.lua @@ -0,0 +1,26 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/9 11:26 PM +--- + +package.path = package.path .. ";../../?.lua;" +local lineParse = require("common.lineParse") + +local line = "hello v=3" +local title, ls, vs = lineParse.parse(line) +assert(title == "hello") +assert(vs.v == 3) +local cLine = lineParse.pack(title, ls, vs) +assert(cLine == line) + +line = "hello,i=1,c=abc log=\"hello,\\\"world.\",v=3,b=4" +title, ls, vs = lineParse.parse(line) +assert(title == "hello") +assert(ls.i == "1") +assert(ls.c == "abc") +assert(vs.v== 3) +assert(vs.log == "hello,\"world.", vs.log) +assert(vs.b == 4) +local cLine = lineParse.pack(title, ls, vs) +assert(cLine == line) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/unix/client.lua b/source/tools/monitor/unity/test/unix/client.lua new file mode 100644 index 0000000000000000000000000000000000000000..ae4889da0f04a636b4cf1512ae6cacaa50b9e75d --- /dev/null +++ b/source/tools/monitor/unity/test/unix/client.lua @@ -0,0 +1,34 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/7 11:40 PM +--- + +package.path = package.path .. ";../../?.lua;" +local unistd = require("posix.unistd") +local socket = require("socket") +local system = require("common.system") + +local pipe = "/tmp/udp" +if not unistd.access(pipe) then + print("host not listen.") +end + +socket.unix = require("socket.unix") +local s = socket.unix.udp() +s:connect(pipe) + +s:send("hello.") +s:close() + +pipe = "/tmp/udp2" +if not unistd.access(pipe) then + print("hosts not listen.") +end + +socket.unix = require("socket.unix") +s = socket.unix.udp() +s:connect(pipe) + +s:send("hello pipe2.") +s:close() diff --git a/source/tools/monitor/unity/test/unix/server.lua b/source/tools/monitor/unity/test/unix/server.lua new file mode 100644 index 0000000000000000000000000000000000000000..a7d492d4b4d0a09f4d1906eca94644a4a064e65c --- /dev/null +++ b/source/tools/monitor/unity/test/unix/server.lua @@ -0,0 +1,47 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/7 1:48 PM +--- +package.path = package.path .. ";../../?.lua;" +local unistd = require("posix.unistd") +local socket = require("socket") +local system = require("common.system") + +local pipe = "/tmp/udp" +if unistd.access(pipe) then + print("unlink file.") + unistd.unlink(pipe) +end + +socket.unix = require("socket.unix") +local s = socket.unix.udp() +s:bind(pipe) + +local pipe2 = "/tmp/udp2" +if unistd.access(pipe2) then + print("unlink file.") + unistd.unlink(pipe2) +end + +local s2 = socket.unix.udp() +s2:bind(pipe2) + +local servers = {s, s2} + +while true do + local canread, _, err = socket.select(servers, nil, 10) + system:dump(canread) + for _, sread in pairs(canread) do + if type(sread) == "number" then + break + end + local line, err = sread:receive() + print(line) + end + +end + +s:close() +unistd.unlink(pipe) + diff --git a/source/tools/monitor/unity/third/luarocks-3.9.1.tar.gz b/source/tools/monitor/unity/third/luarocks-3.9.1.tar.gz deleted file mode 100644 index 73a791594828f521de923230900dee155b331791..0000000000000000000000000000000000000000 Binary files a/source/tools/monitor/unity/third/luarocks-3.9.1.tar.gz and /dev/null differ diff --git a/source/tools/monitor/unity/third/v2.0.5.zip b/source/tools/monitor/unity/third/v2.0.5.zip deleted file mode 100644 index b37a21ff26d41c922e2576f8caeaad3b243a005c..0000000000000000000000000000000000000000 Binary files a/source/tools/monitor/unity/third/v2.0.5.zip and /dev/null differ diff --git a/source/tools/monitor/unity/third/yaml-0.2.5.tar.gz b/source/tools/monitor/unity/third/yaml-0.2.5.tar.gz deleted file mode 100644 index a78e0d998508e242fc7743d06f803cc0e45b2cf3..0000000000000000000000000000000000000000 Binary files a/source/tools/monitor/unity/third/yaml-0.2.5.tar.gz and /dev/null differ diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index 4c05d29057e0899afece75d37bd87f339ecd5c1e..b808489bce6fa04e39e34a25f76575037489dd5a 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -4,13 +4,13 @@ --- DateTime: 2022/12/17 11:04 AM --- -require("class") +require("common.class") -local system = require("system") +local system = require("common.system") local snappy = require("snappy") -local pystring = require("pystring") -local CprotoData = require("protoData") -local foxFFI = require("foxffi") +local pystring = require("common.pystring") +local CprotoData = require("common.protoData") +local foxFFI = require("tsdb.native.foxffi") local CfoxTSDB = class("CfoxTSDB") @@ -167,7 +167,6 @@ function CfoxTSDB:write(buff) local now = self:get_us() local date = self:getDateFrom_us(now) local stream = snappy.compress(buff) - print("write for time: ", now) assert(self.cffi.fox_write(self._man, date, now, self.ffi.string(stream, #stream), #stream) == 0) if self._man.new_day > 0 then self:rotateDb()