diff --git a/source/mk/pyinstaller.mk b/source/mk/pyinstaller.mk new file mode 100644 index 0000000000000000000000000000000000000000..c499222fbab6b376e065adf1b1976c1ae12fec02 --- /dev/null +++ b/source/mk/pyinstaller.mk @@ -0,0 +1,14 @@ +ifeq ($(KERNEL_DEPEND), Y) +TARGET_PATH := $(OBJ_TOOLS_PATH) +else +TARGET_PATH := $(OBJ_TOOLS_ROOT) +endif + +all: $(target) target_rule + +$(target): $@ + rm -Rf build dist + pyinstaller $@.spec + cp dist/$@ $(TARGET_PATH)/ + +include $(SRC)/mk/target.inc diff --git a/source/tools/monitor/ioMonitor/Makefile b/source/tools/monitor/ioMonitor/Makefile index 1345670e82b6c09a9ae9dccd7af1ae9d1c7dfaea..846a1e71597783e6f1167b3b6412680954e1c74a 100755 --- a/source/tools/monitor/ioMonitor/Makefile +++ b/source/tools/monitor/ioMonitor/Makefile @@ -1,4 +1,3 @@ -mods = ioMon target := ioMonitor -include $(SRC)/mk/sh.mk +include $(SRC)/mk/pyinstaller.mk diff --git a/source/tools/monitor/ioMonitor/README.md b/source/tools/monitor/ioMonitor/README.md index 4856ff901df85de226f1ff34314d68dfd311d54f..551166571ea88c9b15663e7d9715a384ac6f278f 100644 --- a/source/tools/monitor/ioMonitor/README.md +++ b/source/tools/monitor/ioMonitor/README.md @@ -1,2 +1,8 @@ # 功能说明 监控服务主程序，收集系统监控指标，支持查看历史监控数据 + +# 编译依赖 +ioMonitor默认会编译成二进制发布，因此编译环境上需要安装pyinstaller，参考步骤如下： +1. 通过yum install python2-pip先安装pip2 +2. 通过pip2 install pyinstaller==3.5命令安装pyinstaller + diff --git a/source/tools/monitor/ioMonitor/ioMon/__init__.py b/source/tools/monitor/ioMonitor/__init__.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/__init__.py rename to source/tools/monitor/ioMonitor/__init__.py diff --git a/source/tools/monitor/ioMonitor/ioMon/displayClass.py b/source/tools/monitor/ioMonitor/displayClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/displayClass.py rename to source/tools/monitor/ioMonitor/displayClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py b/source/tools/monitor/ioMonitor/exceptCheckClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py rename to source/tools/monitor/ioMonitor/exceptCheckClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py b/source/tools/monitor/ioMonitor/exceptDiagnoseClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py rename to source/tools/monitor/ioMonitor/exceptDiagnoseClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py b/source/tools/monitor/ioMonitor/ioMonCfgClass.py similarity index 97% rename from source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py rename to source/tools/monitor/ioMonitor/ioMonCfgClass.py index ffc13f293073751d9e0f185c42cd01939275a995..f1282279600a1ef8c0e8f58e349b65499265efc0 100755 --- a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py +++ b/source/tools/monitor/ioMonitor/ioMonCfgClass.py @@ -36,9 +36,9 @@ class ioMonCfgClass(object): cfg['iops'] = int(cfg['iops']) if cfg['iops'] else 150 cfg['bps'] = int(cfg['bps']) if cfg['bps'] else 31457280 cfg['cycle'] = int(cfg['cycle']) if cfg['cycle'] else 1000 - cfg['diagIowait'] = cfg['diagIowait'] if cfg['diagIowait'] else 'on' - cfg['diagIoburst'] = cfg['diagIoburst'] if cfg['diagIoburst'] else 'on' - cfg['diagIolat'] = cfg['diagIolat'] if cfg['diagIolat'] else 'on' + cfg['diagIowait'] = cfg['diagIowait'] if cfg['diagIowait'] else 'off' + cfg['diagIoburst'] = cfg['diagIoburst'] if cfg['diagIoburst'] else 'off' + cfg['diagIolat'] = cfg['diagIolat'] if cfg['diagIolat'] else 'off' cfg['diagIohang'] = cfg['diagIohang'] if cfg['diagIohang'] else 'off' self._updateCfg(cfg) return @@ -72,7 +72,7 @@ class ioMonCfgClass(object): sys.exit(0) cfgDicts[c[0]] = c[1] except Exception: - print "bad cfg: %s." %cfg + print("bad cfg: %s." %cfg) sys.exit(0) return cfgDicts @@ -126,7 +126,7 @@ class ioMonCfgClass(object): cmdline = f.read().strip() except Exception: sys.exit(0) - if 'ioMonitorMain' in cmdline: + if 'ioMonitor' in cmdline: os.system('kill -USR2 '+str(pid)) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py b/source/tools/monitor/ioMonitor/ioMonitor.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py rename to source/tools/monitor/ioMonitor/ioMonitor.py diff --git a/source/tools/monitor/ioMonitor/ioMonitor.sh b/source/tools/monitor/ioMonitor/ioMonitor.sh deleted file mode 100755 index dfd3d4689472b7715c3cbce077915fa054ee7f88..0000000000000000000000000000000000000000 --- a/source/tools/monitor/ioMonitor/ioMonitor.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -#****************************************************************# -# ScriptName: ioMonitor.sh -# Author: $SHTERM_REAL_USER@alibaba-inc.com -# Create Date: 2021-06-06 16:53 -# Modify Author: $SHTERM_REAL_USER@alibaba-inc.com -# Modify Date: 2021-06-06 16:53 -# Function: -#***************************************************************# -TOOLS_ROOT="$SYSAK_WORK_PATH/tools" -python $TOOLS_ROOT/ioMon/ioMonitorMain.py $* diff --git a/source/tools/monitor/ioMonitor/ioMonitor.spec b/source/tools/monitor/ioMonitor/ioMonitor.spec new file mode 100644 index 0000000000000000000000000000000000000000..02963eab8ab715bdcf9b925847b33bf0cf522b9b --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMonitor.spec @@ -0,0 +1,33 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['ioMonitor.py'], + pathex=["./"], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='ioMonitor', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True ) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMonitorClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py rename to source/tools/monitor/ioMonitor/ioMonitorClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/nfPut.py b/source/tools/monitor/ioMonitor/nfPut.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/nfPut.py rename to source/tools/monitor/ioMonitor/nfPut.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py b/source/tools/monitor/ioMonitor/tools/__init__.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/__init__.py rename to source/tools/monitor/ioMonitor/tools/__init__.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py b/source/tools/monitor/ioMonitor/tools/iofstool/__init__.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py rename to source/tools/monitor/ioMonitor/tools/iofstool/__init__.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py b/source/tools/monitor/ioMonitor/tools/iofstool/common.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py rename to source/tools/monitor/ioMonitor/tools/iofstool/common.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py b/source/tools/monitor/ioMonitor/tools/iofstool/diskstatClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py rename to source/tools/monitor/ioMonitor/tools/iofstool/diskstatClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py b/source/tools/monitor/ioMonitor/tools/iofstool/fsstatClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py rename to source/tools/monitor/ioMonitor/tools/iofstool/fsstatClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py b/source/tools/monitor/ioMonitor/tools/iofstool/iofsstat.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py rename to source/tools/monitor/ioMonitor/tools/iofstool/iofsstat.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py b/source/tools/monitor/ioMonitor/tools/iofstool/iostatClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py rename to source/tools/monitor/ioMonitor/tools/iofstool/iostatClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py b/source/tools/monitor/ioMonitor/tools/iofstool/promiscClass.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py rename to source/tools/monitor/ioMonitor/tools/iofstool/promiscClass.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py b/source/tools/monitor/ioMonitor/tools/iowaitstat/__init__.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py rename to source/tools/monitor/ioMonitor/tools/iowaitstat/__init__.py diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py b/source/tools/monitor/ioMonitor/tools/iowaitstat/iowaitstat.py similarity index 100% rename from source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py rename to source/tools/monitor/ioMonitor/tools/iowaitstat/iowaitstat.py diff --git a/source/tools/monitor/unity/Dockerfile b/source/tools/monitor/unity/Dockerfile index c60554cb71280dc462401e4c556b063a08a4f257..49e867c282f16434ceb911992f9899df582e47f0 100644 --- a/source/tools/monitor/unity/Dockerfile +++ b/source/tools/monitor/unity/Dockerfile @@ -34,6 +34,7 @@ RUN source /opt/rh/devtoolset-9/enable && \ luarocks install md5 && \ luarocks install luaposix 35.1-1 && \ luarocks install http && \ + luarocks install inotify && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ && \ cd ../beeQ/ && \ make \ No newline at end of file diff --git a/source/tools/monitor/unity/Makefile b/source/tools/monitor/unity/Makefile index 6149e8e655e4c0fc3f431d536194c0a34c92fd6c..33cfc768cbb739e4d406c19026d4ff2ad71d9686 100644 --- a/source/tools/monitor/unity/Makefile +++ b/source/tools/monitor/unity/Makefile @@ -9,4 +9,6 @@ all: cp $(OBJ_LIB_PATH)/libcoolbpf.so collector/native/ cp $(OBJ_LIB_PATH)/libcoolbpf.so.0 collector/native/ cp $(OBJ_LIB_PATH)/libcoolbpf.so.0.1.0 collector/native/ + + echo "build for beeQ" make dist -C beeQ diff --git a/source/tools/monitor/unity/beaver/beaver.c b/source/tools/monitor/unity/beaver/beaver.c index 9885f0567b3893d087c3fe89c49caf169e20fa16..8cb6b856e682926f61f4156266d85a9f8300d59a 100644 --- a/source/tools/monitor/unity/beaver/beaver.c +++ b/source/tools/monitor/unity/beaver/beaver.c @@ -11,18 +11,21 @@ #include #include #include +#include +#include extern int lua_reg_errFunc(lua_State *L); extern int lua_check_ret(int ret); int lua_load_do_file(lua_State *L, const char* path); -static int call_init(lua_State *L, int err_func, char *fYaml) { +static int call_init(lua_State *L, int err_func, struct beeQ* q, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); + lua_pushlightuserdata(L, q); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 1, 1, err_func); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { lua_check_ret(ret); goto endCall; @@ -62,7 +65,8 @@ void LuaAddPath(lua_State *L, char *name, char *value) { lua_pop(L, 2); } -static lua_State * echos_init(char *fYaml) { +extern int collector_qout(lua_State *L); +static lua_State * echos_init(struct beeQ* q, char *fYaml) { int ret; int err_func; @@ -78,12 +82,13 @@ static lua_State * echos_init(char *fYaml) { LuaAddPath(L, "path", "../beaver/?.lua"); err_func = lua_reg_errFunc(L); + lua_register(L, "collector_qout", collector_qout); ret = lua_load_do_file(L, "../beaver/beaver.lua"); if (ret) { goto endLoad; } - ret = call_init(L, err_func, fYaml); + ret = call_init(L, err_func, q, fYaml); if (ret < 0) { goto endCall; } @@ -129,11 +134,11 @@ static int echos(lua_State *L) { return ret; } -int beaver_init(char *fYaml) { +int beaver_init(struct beeQ* q, char *fYaml) { int ret = 0; while (ret == 0) { - lua_State *L = echos_init(fYaml); + lua_State *L = echos_init(q, fYaml); if (L == NULL) { break; } diff --git a/source/tools/monitor/unity/beaver/beaver.h b/source/tools/monitor/unity/beaver/beaver.h index f1a6fd3769312b1f0e90c252dc3d80da96547bf5..058fbb946edea7d79afcaa79e10e415de943a04d 100644 --- a/source/tools/monitor/unity/beaver/beaver.h +++ b/source/tools/monitor/unity/beaver/beaver.h @@ -5,6 +5,7 @@ #ifndef UNITY_BEAVER_H #define UNITY_BEAVER_H -int beaver_init(char *fYaml); +#include "../beeQ/beeQ.h" +int beaver_init(struct beeQ* q, char *fYaml); #endif //UNITY_BEAVER_H diff --git a/source/tools/monitor/unity/beaver/beaver.lua b/source/tools/monitor/unity/beaver/beaver.lua index 2431d99ce7bcc5069490f8d0d3510e55df6a4da3..15518b92dd2fdbce7f0e5bd01c823976edf10792 100644 --- a/source/tools/monitor/unity/beaver/beaver.lua +++ b/source/tools/monitor/unity/beaver/beaver.lua @@ -17,15 +17,14 @@ local CurlExportRaw = require("beaver.url_export_raw") local CLocalBeaver = require("beaver.localBeaver") local CbaseQuery = require("beaver.query.baseQuery") -local lb = nil +g_lb = nil -function init(fYaml) +function init(que, fYaml) fYaml = fYaml or "../collector/plugin.yaml" - print(fYaml) local web = Cframe.new() CurlIndex.new(web) - CurlApi.new(web, fYaml) + CurlApi.new(web, que, fYaml) CurlRpc.new(web) CurlGuide.new(web) CbaseQuery.new(web, fYaml) @@ -36,10 +35,10 @@ function init(fYaml) CurlExportHtml.new(web, export) CurlExportRaw.new(web, export) - lb = CLocalBeaver.new(web, fYaml) + g_lb = CLocalBeaver.new(web, fYaml) return 0 end function echo() - return lb:poll() + return g_lb:poll() end diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 2f64132c91958a7e68f971deb97ae60f9ccec9fe..ea90a6eeacc7532bd1299f31bbccbf7439997c7c 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -119,7 +119,7 @@ function Cframe:echo404() ["Content-Type"] = "text/plain", } local body = "Oops! The page may have flown to Mars!!!\n" - local headers = self:packHeaders(tHead, #body) + local headers = self:packServerHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end diff --git a/source/tools/monitor/unity/beaver/guide/common.md b/source/tools/monitor/unity/beaver/guide/common.md index 648b4ceefb9235137c1015e901d3c4d7dddb62bb..c80009ef34aa2d5def6c2b052083f48f9c9a2bf0 100644 --- a/source/tools/monitor/unity/beaver/guide/common.md +++ b/source/tools/monitor/unity/beaver/guide/common.md @@ -126,8 +126,8 @@ python 最擅长在于字符串处理。可以参考这里的[官方库说明](h 使用 pystring 库是要注意以下事项： -1. 当前unity\-mon 采用的是lua 自动的正则匹配方法，未集成regexp 标准正则库； -2. pystring 中提供的绝大部分函数都与python的处理方法保持一致，但是不能保证百分百，故使用时建议根据实际用例测试一遍，可以参考[测试用例函数](https://gitee.com/anolis/sysak/blob/opensource_branch/source/tools/monitor/unity/test/string/py.lua) +1. 当前unity\-mon 采用的是lua 自带的正则匹配方法，未集成regexp 标准正则库； +2. pystring 中提供的绝大部分函数都与python的处理方法保持一致，但是不保证百分百兼容，故使用时建议根据实际用例测试一遍，可以参考[测试用例函数](https://gitee.com/anolis/sysak/blob/opensource_branch/source/tools/monitor/unity/test/string/py.lua) ### pystring:shift(s, n) @@ -139,48 +139,69 @@ python 最擅长在于字符串处理。可以参考这里的[官方库说明](h ### pystring:islower(s) * 函数说明：判断字符串是否全部为小写字母 -* 传参 1：s 要判断的字符串 +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:isupper(s) * 函数说明：同python 实现 +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:ishex(s) * 函数说明：判断目标字符串是否为hex 字符串，含0-9,a-f,A-F +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:isalnum(s) * 函数说明：同python 实现 +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:istilte(s) * 函数说明：判断单词是否符合首字母大写，其余字母小写的方式 +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:isfloat(s) * 函数说明：判断字符串是否为浮点数呈现形式 +* 参数1： s 目标字符串 +* 返回值：true/false ### pystring:lower(s) * 函数说明：同python 实现，转小写 +* 参数1： s 目标字符串 +* 返回值：处理后的字符串 ### pystring:upper(s) * 函数说明：同python 实现，转大写 +* 参数1： s 目标字符串 +* 返回值：处理后的字符串 ### pystring:swapcase(s) * 函数说明：同python 实现，交换大小写 +* 参数1： s 目标字符串 +* 返回值：处理后的字符串 ### pystring:capitalize(s) * 函数说明：单词首字母大写 +* 参数1： s 目标字符串 +* 返回值：处理后的字符串 ### pystring:title(s) * 函数说明：字符串中所有单词首字母大写 +* 参数1： s 目标字符串 +* 返回值：处理后的字符串 ### pystring:ljust(s, len, ch) @@ -207,3 +228,138 @@ python 最擅长在于字符串处理。可以参考这里的[官方库说明](h * 参数3：ch 填充字符，默认为空格，必须是单字符，否则会抛异常 * 返回值：对齐后的字符串 +### pystring:zfill(s, len) + +* 函数说明：将字符串s 按照 len 长度向左填0 +* 参数1： s 目标字符串 +* 参数2：len 对齐长度 +* 返回值：填齐后的字符串 + +### pystring:split(s, delimiter, n) + +* 函数说明：将字符串s 按照 delimiter 分隔符进行分割 +* 参数1： s 目标字符串 +* 参数2：delimiter 分隔符，如果为空，则按照 多空格模式进行分割 +* 参数3：n 分割次数，默认为最多次数 +* 返回值：分割后的字符串数组 + +### pystring:rsplit(s, delimiter, n) + +* 函数说明：将字符串s 按照 delimiter 分隔符从右边进行分割 +* 参数1： s 目标字符串 +* 参数2：delimiter 分隔符，如果为空，则按照 多空格模式进行分割 +* 参数3：n 分割次数，默认为最多次数 +* 返回值：分割后的字符串数组 + + +### pystring:partition(s, del) + +* 函数说明：将字符串s 按照 del 分隔符进行一次分割，返回 [head, del, tail] list +* 参数1： s 目标字符串 +* 参数2：del 分隔符，如果为空，则按照 多空格模式进行分割 +* 返回值：分割后的字符串数组 + +### pystring:rpartition(s, del) + +* 函数说明：将字符串s 按照 del 分隔符从右侧进行一次分割，返回 [head, del, tail] list +* 参数1： s 目标字符串 +* 参数2：del 分隔符，如果为空，则按照 多空格模式进行分割 +* 返回值：分割后的字符串数组 + +### pystring:splitlines(s) + +* 函数说明：将字符串s 按照换行符进行分割 +* 参数1： s 目标字符串 +* 返回值：分割后的字符串数组 + +### pystring:lstrip(s, chars) + +* 函数说明：strip 左边的字符 +* 参数1： s 目标字符串 +* 参数2：chars 要strip 字符串 +* 返回值：strip 后面的字符串 + +### pystring:rstrip(s, chars) + +* 函数说明：strip 右边的字符 +* 参数1： s 目标字符串 +* 参数2：chars 要strip 字符串 +* 返回值：strip 后面的字符串 + +### pystring:strip(s, chars) + +* 函数说明：strip 两边的字符 +* 参数1： s 目标字符串 +* 参数2：chars 要strip 字符串 +* 返回值：strip 后面的字符串 + +### pystring:join(delim, strings) + +* 函数说明：将字符串list拼接成单一字符串 +* 参数1： delim 字符串间的分隔符号 +* 参数2：strings 要拼接的字符串list +* 返回值：拼接后的字符串 + +### pystring:startswith(s1, s2) +* 函数说明：判断字符串s1是否以s2 开头 +* 参数1： s1 目标字符串 +* 参数2：s2 +* 返回值：true/false + +### pystring:endswith(s1, s2) +* 函数说明：判断字符串s1是否以s2 结尾 +* 参数1： s1 目标字符串 +* 参数2：s2 +* 返回值：true/false + +### pystring:find(s1, s2, start, stop) +* 函数说明：判断s1 是否包含s2 子串 +* 参数1： s1 目标字符串 +* 参数2：s2 目标子串 +* 参数3： 字符串起点 默认为1 +* 参数4：字符串截止点 默认为-1 +* 返回值：如果包含s2，返回字符下标，不包含返回-1 + +### pystring:rfind(s1, s2, start, stop) +* 函数说明：从右边开始,判断s1 是否包含s2 子串 +* 参数1： s1 目标字符串 +* 参数2：s2 目标子串 +* 参数3： 字符串起点 默认为1 +* 参数4：字符串截止点 默认为-1 +* 返回值：如果包含s2，返回字符下标，不包含返回-1 + +### pystring:index(s1, s2, start, stop) +* 函数说明：判断s1 是否包含s2 子串 +* 参数1： s1 目标字符串 +* 参数2：s2 目标子串 +* 参数3： 字符串起点 默认为1 +* 参数4：字符串截止点 默认为-1 +* 返回值：如果包含s2，返回字符下标，不包含则抛出异常 + +### pystring:rindex(s1, s2, start, stop) +* 函数说明：从右边开始,判断s1 是否包含s2 子串 +* 参数1： s1 目标字符串 +* 参数2：s2 目标子串 +* 参数3： 字符串起点 默认为1 +* 参数4：字符串截止点 默认为-1 +* 返回值：如果包含s2，返回字符下标，不包含则抛出异常 + +### pystring:count(s, find) +* 函数说明：获取目标字符串中 子串数量 +* 参数1： s 目标字符串 +* 参数2：find 要查找的子串 +* 返回值：查找到的子串数量 + +### pystring:replace(s, find, repl, n) +* 函数说明：替换字符串中的子串数据 +* 参数1： s 目标字符串 +* 参数2：find 要查找的子串 +* 参数3：repl 要替换的结果 +* 参数4：替换数量，默认为不限 +* 返回值：替换结果 + +### pystring:expandtabs(s, tabs) +* 函数说明：将字符串中tabs 替换成空格 +* 参数1： s 目标字符串 +* 参数2：tabs 占空格数量，默认为4 +* 返回值：替换结果 \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 2a2feff70097a6fefd319e8e07390cf958be3657..5431a3cc6cb9312d21b7bcaf17db7ccc8732424e 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -448,7 +448,7 @@ return CkvProc 1. unity\_set\_table 中 table 参数长度应该小于32（不含） 2. unity\_set\_index 中 name、index和unity\_set\_value 中 name 参数长度应该要小于16（不含） 3. unity\_set\_index 下标从0开始，并小于 4，即最多4个索引。而且下标数值应该连续，否则数据会从留白处截断 -4. unity\_set\_index 下标从0开始，并小于 32，即最多32个数值。而且下标数值应该连续，否则数据会从留白处截断； +4. unity\_set\_value 下标从0开始，并小于 32，即最多32个数值。而且下标数值应该连续，否则数据会从留白处截断； 5. unity\_set\_log 中的log 指针需要开发者进行释放； 6. get\_unity\_proc参考2.3节中 proc_path 中的内容； diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index f2d83799bae2068e9409cb238d571e39b0fae134..a377b616f0ff63163f31bed87582f5d936bf362d 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -4,4 +4,5 @@ 3. [metric 指标数据说明](/guide/metrics.md) 4. [在lua 中使用pystring](/guide/pystring.md) 5. [bpf\ map 开发](/guide/bpf.md) -6. [bpf perf 开发](/guide/bpf_perf.md) \ No newline at end of file +6. [bpf perf 开发](/guide/bpf_perf.md) +7. [外部行协议写入](/guide/outLine.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index b8d4190b2fd18e6123d2b733c1ccebfc4927ac2a..1752edea98d610127b6168b1cbc542ba96a7d381 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -26,6 +26,74 @@ | machine | - | uname -r | | collector/proc\_uptime.lua | | sysname | - | uname -r | | collector/proc\_uptime.lua | +### cgroups 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| type | - | subsys类型 | | collector/proc\_cgroups.lua | +| blkio | 个 | blkio cgroup 数量 | | collector/proc\_cgroups.lua | +| freezer | 个 | freezer cgroup数量 | | collector/proc\_cgroups.lua | +| devices | 个 | devices cgroup数量 | | collector/proc\_cgroups.lua | +| hugetlb | 个 | hugetlb cgroup数量 | | collector/proc\_cgroups.lua | +| pids | 个 | blkio cgroup 数量 | | collector/proc\_cgroups.lua | +| rdma | 个 | rdma cgroup数量 | | collector/proc\_cgroups.lua | +| net\_prio | 个 | net_prio cgroup数量 | | collector/proc\_cgroups.lua | +| net\_cls | 个 | net_cls cgroup数量 | | collector/proc\_cgroups.lua | +| cpu | 个 | cpu cgroup 数量 | | collector/proc\_cgroups.lua | +| cpuacct | 个 | cpuacct cgroup数量 | | collector/proc\_cgroups.lua | +| perf\_event | 个 | perf_event cgroup数量 | | collector/proc\_cgroups.lua | +| memory | 个 | memory cgroup数量 | | collector/proc\_cgroups.lua | + +### interrupts 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| cpu | - | CPU ID | | collector/proc\_interrupts.lua | +| 中断名称 | 次 | 中断触发次数 | | collector/proc\_interrupts.lua | + +### mounts 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| fs | - | sysfs | | collector/proc\_mounts.lua | +| mount | - | 挂载目录 | | collector/proc\_mounts.lua | +| f\_bsize | - | Filesystem block size | | collector/proc\_mounts.lua | +| f\_blocks | - | Size of fs in f_frsize units | | collector/proc\_mounts.lua | +| f\_bfree | - | Number of free blocks | | collector/proc\_mounts.lua | +| f\_bavail | - | Number of free blocks for unprivileged users | | collector/proc\_mounts.lua | +| f\_files | - | Number of inodes | | collector/proc\_mounts.lua | +| f\_ffree | - | Number of free inodes | | collector/proc\_mounts.lua | +| f\_favail | - | Number of free inodes for unprivileged users | | collector/proc\_mounts.lua | + +### softirqs 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| cpu | - | CPU ID | | collector/proc\_softirqs.lua | +| HI | 次 | HI软中断触发次数 | | collector/proc\_softirqs.lua | +| TIMER | 次 | TIMER软中断触发次数 | | collector/proc\_softirqs.lua | +| NET\_TX | 次 | NET\_TX软中断触发次数 | | collector/proc\_softirqs.lua | +| NET\_RX | 次 | NET\_RX软中断触发次数 | | collector/proc\_softirqs.lua | +| BLOCK | 次 | BLOCK软中断触发次数 | | collector/proc\_softirqs.lua | +| IRQ_POLL | 次 | IRQ\_POLL软中断触发次数 | | collector/proc\_softirqs.lua | +| TASKLET | 次 | TASKLET软中断触发次数 | | collector/proc\_softirqs.lua | +| SCHED | 次 | SCHED软中断触发次数 | | collector/proc\_softirqs.lua | +| HRTIMER | 次 | HRTIMER软中断触发次数 | | collector/proc\_softirqs.lua | +| RCU | 次 | RCU软中断触发次数 | | collector/proc\_softirqs.lua | + +### self_statm 表 +统计监控进程的statm信息 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| size | Page | total program size | | collector/proc\_statm.lua | +| resident | Page | resident set size | | collector/proc\_statm.lua | +| shared | Page | number of resident shared pages | | collector/proc\_statm.lua | +| text | Page | text (code) | | collector/proc\_statm.lua | +| lib | Page | library | | collector/proc\_statm.lua | +| data | Page | data + stack | | collector/proc\_statm.lua | +| dt | Page | dirty pages | | collector/proc\_statm.lua | + ## 网络指标 ----------- @@ -77,20 +145,20 @@ 统计所有包状态。[参考连接](https://developer.aliyun.com/article/484451) - 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | -| :--- | ---: | :---- | :---- | :--- | -| frag\_inuse | 个 | | 使用的IP段数量 | collector/proc\_sockstat.lua | -| frag\_memory | 页 | | IP段使用内存数量 | collector/proc\_sockstat.lua | +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +|:---------------| ---: | :---- | :---- | :--- | +| frag\_inuse | 个 | | 使用的IP段数量 | collector/proc\_sockstat.lua | +| frag\_memory | 页 | | IP段使用内存数量 | collector/proc\_sockstat.lua | | udplite\_inuse | 个 | | udplite 使用量 | collector/proc\_sockstat.lua | -| udp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | -| udp\_inuse | 个 | | udp 使用量 | collector/proc\_sockstat.lua | -| tcp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | -| tcp\_alloc | 个 | | TCP socket 申请总数 | collector/proc\_sockstat.lua | -| tcp\_tw | 个 | | TCP time wait socket 总数 | collector/proc\_sockstat.lua | -| tcp\_orphan | 个 | | TCP ophan socket 总数 | collector/proc\_sockstat.lua | -| tcp\_inuse | 个 | | TCP 常规 socket 总数 | collector/proc\_sockstat.lua | -| raw\_inuse | 个 | | raw socket 使用量 | collector/proc\_sockstat.lua | -| sockets\_used | 个 | | 总socket 使用量 | collector/proc\_sockstat.lua | +| udp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | +| udp\_inuse | 个 | | udp 使用量 | collector/proc\_sockstat.lua | +| tcp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | +| tcp\_alloc | 个 | | TCP socket 申请总数 | collector/proc\_sockstat.lua | +| tcp\_tw | 个 | | TCP time wait socket 总数 | collector/proc\_sockstat.lua | +| tcp\_orphan | 个 | | TCP ophan socket 总数 | collector/proc\_sockstat.lua | +| tcp\_inuse | 个 | | TCP 常规 socket 总数 | collector/proc\_sockstat.lua | +| raw\_inuse | 个 | | raw socket 使用量 | collector/proc\_sockstat.lua | +| sockets\_used | 个 | | 总socket 使用量 | collector/proc\_sockstat.lua | ### softnets @@ -173,3 +241,103 @@ This parser parses the stats from network devices. These stats includes events p | lib | - | library | | collector/proc\_statm.lua | | data | - | data + stack | | collector/proc\_statm.lua | | dt | - | dirty pages | | collector/proc\_statm.lua | + +## IO指标 + +------------- + +### IOMonIndForDisksIO 表 +统计磁盘级IO信息 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +|:---------------| ---: | :---- | :---- | :--- | +| ioburstCnt | 次 | 每分钟io burst（IO压力突发增大）次数 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| iodelayCnt | 次 | 每分钟io延迟高次数 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| iohangCnt | 次 | 每分钟io hang次数 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| bps | kB | 磁盘bps | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| iops | 个 | 磁盘iops | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| qusize | 个 | 未完成io数 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| util | 占比 | IO繁忙度 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| await | ms | 平均每个IO的延迟 | - | ../ioMonitor/ioMon/ioMonitorClass.py | + +### IOMonIndForSystemIO 表 +统计系统IO异常 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +|:---------------| ---: | :---- | :---- | :--- | +| iowait | 占比 | 系统等待IO的占比 | - | ../ioMonitor/ioMon/ioMonitorClass.py | +| iowaithighCnt | 次 | 每分钟iowait高次数 | - | ../ioMonitor/ioMon/ioMonitorClass.py | + +## 混部指标 + +----------- + +### cg_cpu_stat 表 +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| nr_throttled | - | total throttled number | | collector/container/cg\_cpu\_stat.lua | +| throttled_time | ms | total throttled time | | collector/container/cg\_cpu\_stat.lua | + +### cg_proc_stat 表 +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| user | % | usr cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| nice | % | nice cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| system | % | system cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| idle | % | idl cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| iowait | % | iowait cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| irq | % | irq cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| softirq | % | softirq cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| steal | % | steal cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| guest | % | guest cpu util | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| load1min | - | load of 1min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| load5min | - | load of 5min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| load15min | - | load of 15min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| r_load1min | - | running load of 1min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| r_load5min | - | running load of 5min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| r_load15min | - | running load of 15min | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| nr_running | - | number of runable tasks | | collector/container/cg\_cpuacct\_proc\_stat.lua | +| nr_uninterruptible | - | number of deep sleep tasks | | collector/container/cg\_cpuacct\_proc\_stat.lua | + +### cg_memfail_cnt 表 +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| fail_cnt | - | number of mem fail counts | | collector/container/cg\_memory\_fail\_cnt.lua | + +### cg_memdrcm_latency 表 +This table show the hist of the latency of direct memory reclamation +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| memDrcm_lat_1to5ms | - | times 1to5ms | | collector/container/cg\_memory\_drcm\_latency.lua | +| memDrcm_lat_5to10ms | - | times 5to10ms | | collector/container/cg\_memory\_drcm\_latency.lua | +| memDrcm_lat_10to100ms | - | times 10to100ms | | collector/container/cg\_memory\_drcm\_latency.lua | +| memDrcm_lat_100to500ms | - | times 100to500ms | | collector/container/cg\_memory\_drcm\_latency.lua | +| memDrcm_lat_500to1000ms | - | times 500msto1s | | collector/container/cg\_memory\_drcm\_latency.lua | +| memDrcm_lat_1000ms | - | times more than 1s | | collector/container/cg\_memory\_drcm\_latency.lua | + +### cg_memmcmp_latency 表 +This table show the hist of the latency of direct memory compaction +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| memDcmp_lat_1to5ms | - | times 1to5ms | | collector/container/cg\_memory\_dcmp\_latency.lua | +| memDcmp_lat_5to10ms | - | times 5to10ms | | collector/container/cg\_memory\_dcmp\_latency.lua | +| memDcmp_lat_10to100ms | - | times 10to100ms | | collector/container/cg\_memory\_dcmp\_latency.lua | +| memDcmp_lat_100to500ms | - | times 100to500ms | | collector/container/cg\_memory\_dcmp\_latency.lua | +| memDcmp_lat_500to1000ms | - | times 500msto1s | | collector/container/cg\_memory\_dcmp\_latency.lua | +| memDcmp_lat_1000ms | - | times more than 1s | | collector/container/cg\_memory\_dcmp\_latency.lua | + +### pmu_events 表 +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| cpu_cycles | - | cycles | | collector/plugin/pmu_events/pmu\_events.c | +| instructions | - | instructions | | collector/plugin/pmu_events/pmu\_events.c | +| ipc | - | instructions per cycles | | collector/plugin/pmu_events/pmu\_events.c | +| cpi | - | cycles per instructions | | collector/plugin/pmu_events/pmu\_events.c | +| llc_store_ref | - | llc stroe hits counts | | collector/plugin/pmu_events/pmu\_events.c | +| llc_store_miss | - | llc stroe miss counts | | collector/plugin/pmu_events/pmu\_events.c | +| llc_load_ref | - | llc load hits counts | | collector/plugin/pmu_events/pmu\_events.c | +| llc_load_miss | - | llc load miss counts | | collector/plugin/pmu_events/pmu\_events.c | +| llc_rmiss_rate | - | llc load miss rate | | collector/plugin/pmu_events/pmu\_events.c | +| llc_wmiss_rate | - | llc store miss rate | | collector/plugin/pmu_events/pmu\_events.c | +| llc_miss_rate | - | llc miss rate | | collector/plugin/pmu_events/pmu\_events.c | +| llc_cache_mpi | - | llc miss per instructions | | collector/plugin/pmu_events/pmu\_events.c | diff --git a/source/tools/monitor/unity/beaver/guide/outLine.md b/source/tools/monitor/unity/beaver/guide/outLine.md new file mode 100644 index 0000000000000000000000000000000000000000..22a77dcd9ed46f9fdd6cebba7f18815a4dd367f4 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/outLine.md @@ -0,0 +1,81 @@ +# 外部数据写入支持 +unity-mon可以作为一个独立的TSDB 数据库进行使用，支持[行协议](https://jasper-zhang1.gitbooks.io/influxdb/content/Write_protocols/line_protocol.html)写入数据，并按需完成对外数据吐出。 + +## 行协议格式支持情况 +unity-mon 当前除了不支持时间戳，支持行协议其它所有的数据类型，包含数值和日志。写行数据时，有以下注意事项： + +* 不要将同一表名和同一索引，但数值不同的数据放在同一批次写入操作中，会发生时序数据覆盖，如； + +``` +talbe_a,index=table_a value1=1,value2=2 +talbe_a,index=table_a value1=3,value2=4 +``` + +* 同一张表 写同一批数据应该要保持一致，如 + +``` +talbe_a,index=table_a value1=1,value2=2 +talbe_a,index=table_b value1=3,value2=4 +``` + +不要出现同一张表，但是写入的索引和数值不的情况，如： + +``` +talbe_a,index=table_a value1=1 +talbe_a,index=table_b value2=3 +``` + +这样操作会导致数据库中的数值需要用空值来表示，为后面的数据解析带来极大地不便，可以将上面两行数据合并成一行数据，或者拆分成两个table 来分开写入 + +## 外部数据写入示例 +unity-mon 同时支持管道和http post 两种方式进行写入，两者差别如下： + +| 模式 | pipe | http | +| --- | --- | --- | +| 适用范围 | 内部 | 内部 + 外部 | +| 写入效率 | 高 | 低 | +| 最大写入数据长度 | 64K | 2M | + +使用者可以结合自己的实际情况进行推送 + +### pipe api + +``` +import os +import socket + +PIPE_PATH = "/var/sysom/outline" # 这里可以参考yaml 中的配置 +MAX_BUFF = 64 * 1024 + + +class CnfPut(object): + def __init__(self, path=PIPE_PATH): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = path + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check Netinfo is running.") + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut() + nf.puts('runtime,mode=java log="hello runtime."') +``` + +### http post + +``` +import requests + +url = "http://127.0.0.1:8400/api/line" +line = "lineTable,index=abc value=2" +res = requests.post(url, data=line) +print(res) # 成功返回 200 +``` + +## 数据反查入口 +写入到unity-mon 数据后，一般会缓存7天左右，有一个[查询入口](http://127.0.01:8400/query/base)可以查询到最近写入数据，确定数据是否已经写入本地监控。 \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/testjs.html b/source/tools/monitor/unity/beaver/guide/testjs.html new file mode 100644 index 0000000000000000000000000000000000000000..ac199064ac3885984c64d8869a14793340fc7620 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/testjs.html @@ -0,0 +1,20 @@ + + + + + 菜鸟教程(runoob.com) + + + + +

我的第一个 JavaScript 程序

+

这是一个段落

+ + + + + \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 220b60661e47af0f4c108f0294a8480876e5ac51..483dde1fd1a6dde20eedbe6776679257d7f32655 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -7,6 +7,7 @@ require("common.class") local system = require("common.system") local socket = require("socket") +local stdlib = require("posix.stdlib") local Cidentity = class("identity") @@ -18,7 +19,8 @@ function Cidentity:_init_(fYaml) curl = function() return self:curl() end, hostname = function() return self:hostname() end, file = function() return self:file() end, - specify = function() return self:specify() end + specify = function() return self:specify() end, + env = function() return self:env() end } end @@ -77,6 +79,15 @@ function Cidentity:specify() end end +function Cidentity:env() + if self._opts.name then + local inst_name = stdlib.getenv(self._opts.name) + return inst_name + else + return "None" + end +end + function Cidentity:id() return self._funcs[self._opts.mode]() end diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 748a20e14b7d6db873968c0caf6bfde113b8cc0c..22d679e88f44942c0f7071341771cc4ae2d6f4c7 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -18,7 +18,7 @@ local function setupServer(fYaml) local ip = config["bind_addr"] or "0.0.0.0" local backlog = config["backlog"] or 32 local unix_socket = config["unix_socket"] - return port, ip, backlog,unix_socket + return port, ip, backlog, unix_socket end function CLocalBeaver:_init_(frame, fYaml) @@ -59,6 +59,7 @@ function CLocalBeaver:_installTmo(fd) end function CLocalBeaver:_checkTmo() + local res, msg local now = os.time() if now - self._last >= 30 then -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety @@ -70,7 +71,8 @@ function CLocalBeaver:_checkTmo() e.fd = fd local co = self._cos[fd] print("close " .. fd) - coroutine.resume(co, e) + res, msg = coroutine.resume(co, e) + assert(res, msg) end end self._last = now @@ -98,7 +100,7 @@ local function localBind(fd, tPort) system:posixError("set sock opt failed."); end - while try < 120 do + while try < 5 do res, err, errno = socket.bind(fd, tPort) if res then return 0 @@ -162,7 +164,7 @@ function CLocalBeaver:_install_fd(port, ip, backlog) end function CLocalBeaver:read(fd, maxLen) - maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max + maxLen = maxLen or 2 * 1024 * 1024 -- signal conversation accept 2M stream max local function readFd() local e = coroutine.yield() if e.ev_close > 0 then @@ -193,7 +195,7 @@ function CLocalBeaver:write(fd, stream) local res sent, err, errno = socket.send(fd, stream) - if sent then + if sent ~= nil then if sent < #stream then -- send buffer may full res = self._cffi.mod_fd(self._efd, fd, 1) -- epoll write ev assert(res == 0) @@ -204,6 +206,9 @@ function CLocalBeaver:write(fd, stream) return nil elseif e.ev_out then stream = string.sub(stream, sent + 1) + if stream == nil then + return 1 + end sent, err, errno = socket.send(fd, stream) if sent == nil then if errno == 11 then -- EAGAIN ? @@ -247,31 +252,39 @@ function CLocalBeaver:_proc(fd) end end -function CLocalBeaver:co_add(fd) - local res = self._cffi.add_fd(self._efd, fd) +function CLocalBeaver:co_add(fd, cb) + local res = self._cffi.add_fd(self._efd, fd) -- add to epoll fd assert(res >= 0) - local co = coroutine.create(function(o, fd) self._proc(o, fd) end) + local co = coroutine.create(function(o, fd) cb(o, fd) end) self._cos[fd] = co - local res, msg = coroutine.resume(co, self, fd) - assert(res, msg) + return co end function CLocalBeaver:co_exit(fd) - local res = self._cffi.del_fd(self._efd, fd) + local res = self._cffi.del_fd(self._efd, fd) -- remove from epoll fd assert(res >= 0) self._cos[fd] = nil self._tmos[fd] = nil end +function CLocalBeaver:mod_fd(fd, wr) + local res = self._cffi.mod_fd(self._efd, fd, wr) + assert(res == 0) +end + function CLocalBeaver:accept(fd, e) if e.ev_close > 0 then error("should close bind fd.") else local nfd, err, errno = socket.accept(fd) if nfd then - self:co_add(nfd) + local co = self:co_add(nfd, self._proc) + + local res, msg = coroutine.resume(co, self, nfd) + assert(res, msg) + self:_installTmo(nfd) else system:posixError("accept new socket failed", err, errno) diff --git a/source/tools/monitor/unity/beaver/native/local_beaver.c b/source/tools/monitor/unity/beaver/native/local_beaver.c index 9795580a1ce8a0f25a1156613459985bd32a1dd7..641e8837fffa6c406c80d432fb63b43c1bc4530a 100644 --- a/source/tools/monitor/unity/beaver/native/local_beaver.c +++ b/source/tools/monitor/unity/beaver/native/local_beaver.c @@ -18,14 +18,14 @@ static int socket_non_blocking(int sfd) flags = fcntl(sfd, F_GETFL); if (flags < 0) { perror("error : cannot get socket flags!\n"); - return flags; + return -errno; } flags |= O_NONBLOCK; res = fcntl(sfd, F_SETFL, flags); if (res < 0) { perror("error : cannot set socket flags!\n"); - return res; + return -errno; } return 0; @@ -41,6 +41,7 @@ static int epoll_add(int efd, int fd) { res = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &event); if (res < 0) { perror("error : can not add event to epoll!\n"); + return -errno; } return res; } @@ -51,6 +52,7 @@ static int epoll_del(int efd, int fd) { res = epoll_ctl(efd, EPOLL_CTL_DEL, fd, NULL); if (res < 0) { perror("error : can not del event to epoll!\n"); + return -errno; } return res; } @@ -72,7 +74,7 @@ int init(int listen_fd) { return efd; end_epoll_add: - return res; + return -errno; } int add_fd(int efd, int fd) { @@ -104,6 +106,7 @@ int mod_fd(int efd, int fd, int wr) { res = epoll_ctl(efd, EPOLL_CTL_MOD, fd, &event); if (res < 0) { perror("error : can not add event to epoll!\n"); + return -errno; } return res; } @@ -123,7 +126,7 @@ int poll_fds(int efd, int tmo, native_events_t* nes) { res = epoll_wait(efd, events, NATIVE_EVENT_MAX, tmo * 1000); if (res < 0) { perror("error : epoll failed!\n"); - return res; + return -errno; } nes->num = res; for (i = 0; i < res; i ++) { diff --git a/source/tools/monitor/unity/beaver/pushLine.lua b/source/tools/monitor/unity/beaver/pushLine.lua new file mode 100644 index 0000000000000000000000000000000000000000..846568ec6f0fadb5c64e01b1a56b939e382e91f4 --- /dev/null +++ b/source/tools/monitor/unity/beaver/pushLine.lua @@ -0,0 +1,58 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/24 01:34 +--- + +require("common.class") +local system = require("common.system") +local CprotoData = require("common.protoData") +local lineParse = require("common.lineParse") +local pystring = require("common.pystring") + +local CpushLine = class("CpushLine") + +function CpushLine:_init_(que) + self._proto = CprotoData.new(que) +end + +local function trans(title, ls, vs, log) + local labels = {} + local values = {} + local logs = {} + + local c = 0 + for k, v in pairs(ls) do + c = c + 1 + labels[c] = {name=k, index=v} + end + c = 0 + for k, v in pairs(vs) do + c = c + 1 + values[c] = {name=k, value=v} + end + c = 0 + for k, v in pairs(log) do + c = c + 1 + logs[c] = {name=k, log=v} + end + return {line = title, ls = labels, vs = values, log = logs} +end + +function CpushLine:procLine(line) + return trans(lineParse.parse(line)) +end + +function CpushLine:procLines(stream) + local ss = pystring:split(stream, "\n") + local lines = self._proto:protoTable() + + for _, line in ipairs(ss) do + table.insert(lines.lines, self:procLine(line)) + end + local bytes = self._proto:encode(lines) + self._proto:que(bytes) + return 0 +end + +return CpushLine \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/url_api.lua b/source/tools/monitor/unity/beaver/url_api.lua index e78df15419e3a9ef16001da7425185a5c45724d0..e54fa5c3d1b8a03b69e7a6b0237edbae0e4c5f9d 100644 --- a/source/tools/monitor/unity/beaver/url_api.lua +++ b/source/tools/monitor/unity/beaver/url_api.lua @@ -8,15 +8,130 @@ require("common.class") local system = require("common.system") local ChttpApp = require("httplib.httpApp") local CfoxTSDB = require("tsdb.foxTSDB") +local postQue = require("beeQ.postQue.postQue") +local CpushLine = require("beaver.pushLine") +local CasyncDns = require("httplib.asyncDns") +local CasyncHttp = require("httplib.asyncHttp") +local CasyncOSS = require("httplib.asyncOSS") local CurlApi = class("urlApi", ChttpApp) -function CurlApi:_init_(frame, fYaml) +function CurlApi:_init_(frame, que, fYaml) ChttpApp._init_(self) + self._pushLine = CpushLine.new(que) self._urlCb["/api/sum"] = function(tReq) return self:sum(tReq) end self._urlCb["/api/sub"] = function(tReq) return self:sub(tReq) end self._urlCb["/api/query"] = function(tReq) return self:query(tReq) end + self._urlCb["/api/trig"] = function(tReq) return self:trig(tReq) end + self._urlCb["/api/line"] = function(tReq) return self:line(tReq) end + self._urlCb["/api/dns"] = function(tReq) return self:dns(tReq) end + self._urlCb["/api/proxy"] = function(tReq) return self:proxy(tReq) end + self:_ossIntall(fYaml) + self:_install(frame) self:_setupQs(fYaml) + self._proxy = CasyncHttp.new() +end + +function CurlApi:_ossIntall(fYaml) + local res = system:parseYaml(fYaml) + if res.oss then + self._oss = CasyncOSS.new(res.oss) + self._urlCb["/api/oss"] = function(tReq) return self:oss(tReq) end + end +end + +local function reqOSS(oss, uuid, stream) + return oss:put(uuid, stream) +end + +function CurlApi:oss(tReq) + local stat, tJson = pcall(self.getJson, self, tReq) + if stat and tJson then + local uuid = tJson.uuid + local stream = tJson.stream + if uuid and stream then + local stat, body = pcall(reqOSS, self._oss, uuid, stream) + if stat then + return body + else + return "bad req dns " .. body, 400 + end + else + return "need uuid and stream arg.", 400 + end + else + return "bad dns " .. tReq.data, 400 + end +end + +local function reqProxy(proxy, host, uri) + return proxy:get(host, uri) +end + +function CurlApi:proxy(tReq) + local stat, tJson = pcall(self.getJson, self, tReq) + if stat and tJson then + local host = tJson.host + local uri = tJson.uri + if host and uri then + local stat, body = pcall(reqProxy, self._proxy, host, uri) + if stat then + return {body = body} + else + return "bad req dns " .. body, 400 + end + else + return "need domain arg.", 400 + end + else + return "bad dns " .. tReq.data, 400 + end +end + +local function reqDns(domain) + local dns = CasyncDns.new() + return dns:dns_lookup(domain) +end + +function CurlApi:dns(tReq) + local stat, tJson = pcall(self.getJson, self, tReq) + if stat and tJson then + local domain = tJson.domain + if domain then + local stat, ip = pcall(reqDns, domain) + if stat then + return {domain = tReq.data, ip = ip} + else + return "bad req dns " .. tReq.data .. ip, 400 + end + else + return "need domain arg.", 400 + end + else + return "bad dns " .. tReq.data, 400 + end +end + +function CurlApi:line(tReq) + local stat, _ = pcall(self._pushLine.procLines, self._pushLine, tReq.data) + if stat then + return "ok" + else + return "bad line " .. tReq.data, 400 + end +end + +function CurlApi:trig(tReq) + local stat, tJson = pcall(self.getJson, self, tReq) + if stat and tJson then + local s = self:jencode(tJson) + if #s > 5 then + postQue.post(s) + end + return tJson + else + return {} + end end function CurlApi:sum(tReq) diff --git a/source/tools/monitor/unity/beeQ/Makefile b/source/tools/monitor/unity/beeQ/Makefile index c2360532a31a0fb5b93076d109a53a3a8fc00dd2..048c58a6d8464dc83b6ce42e036f69b90dbda83b 100644 --- a/source/tools/monitor/unity/beeQ/Makefile +++ b/source/tools/monitor/unity/beeQ/Makefile @@ -2,11 +2,11 @@ LIB= -lpthread -ldl CC=gcc CFLAG := -g -I../beaver -I../collector/outline -LDFLAG := -g -lm -ldl -lrt -lpthread -lluajit-5.1 -L./lib/ -lbeeQ -L../beaver -lbeaver -lcollectorApi -L../collector/outline/ -loutline -L../collector/plugin/ -lproto_sender +LDFLAG := -g -lm -ldl -lrt -lpthread -lluajit-5.1 -L./lib/ -lbeeQ -L../beaver -lbeaver -lcollectorApi -Lclock/ -llocalclock -LpostQue -lpostQue -L../collector/outline/ -loutline -L../collector/plugin/ -lproto_sender PRG=unity-mon -OBJ=apps.o bees.o -DEPMOD=lib rbtree ../beaver ../collector/native ../collector/interface ../collector/outline ../collector/plugin ../tsdb/native +OBJ=apps.o bees.o daemon.o pushTo.o +DEPMOD=lib rbtree clock postQue ../beaver ../collector/native ../collector/interface ../collector/outline ../collector/plugin ../tsdb/native $(PRG): $(DEPMOD) $(OBJ) $(CC) $(LIB) -o $@ $(OBJ) $(LDFLAG) diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index e1ea2932aadf36ba2a736813e278fb6f230ed860..40e37b57efc2196544691a1694c855a2e889bb76 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -7,9 +7,14 @@ #include #include #include +#include #include "beeQ.h" #include "apps.h" +#include "pushTo.h" +#include "clock/ee_clock.h" +#include "daemon.h" #include +#include #define gettidv1() syscall(__NR_gettid) extern char *g_yaml_file; @@ -63,6 +68,19 @@ int lua_load_do_file(lua_State *L, const char* path) { return lua_check_ret(ret); } +static int lua_push_start(lua_State *L) { + int ret; + int fd = lua_tonumber(L, 1); + ret = pushTo_start(fd); + lua_pushnumber(L, ret); + return 1; +} + +static int lua_push_stop(lua_State *L) { + pushTo_stop(); + return 0; +} + static int call_init(lua_State *L, int err_func) { int ret; lua_Number lret; @@ -108,6 +126,9 @@ static lua_State * app_recv_init(void) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); + lua_register(L, "lua_push_start", lua_push_start); + lua_register(L, "lua_push_stop", lua_push_stop); + ret = lua_load_do_file(L, "../beeQ/bees.lua"); if (ret) { goto endLoad; @@ -126,9 +147,11 @@ static lua_State * app_recv_init(void) { return NULL; } +// 这是接收队列初始化动作 int app_recv_setup(struct beeQ* q) { lua_State *L; + prctl(PR_SET_NAME, (unsigned long)"app_recv"); L = app_recv_init(); if (L == NULL) { return -1; @@ -205,6 +228,25 @@ int app_recv_proc(void* msg, struct beeQ* q) { return ret; } +static int lua_local_clock(lua_State *L) { + clock_t t = get_local_clock(); + lua_pushnumber(L, t); + return 1; +} + +static int lua_setup_daemon(lua_State *L) { + int fd; + int period = lua_tonumber(L, 1); + fd = setup_daemon(period); + lua_pushnumber(L, fd); + return 1; +} + +static int prctl_death_kill(lua_State *L) { + prctl(PR_SET_PDEATHSIG, SIGKILL); + return 0; +} + int collector_qout(lua_State *L) { int ret; struct beeQ* q = (struct beeQ*) lua_topointer(L, 1); @@ -225,11 +267,14 @@ int collector_qout(lua_State *L) { return 1; // return a value. } +// 这是 collector 初始化操作 static int app_collector_work(void* q, void* proto_q) { int ret; int err_func; lua_Number lret; + prctl(PR_SET_NAME, (unsigned long)"app_collector"); + /* create a state and load standard library. */ lua_State *L = luaL_newstate(); if (L == NULL) { @@ -239,19 +284,23 @@ static int app_collector_work(void* q, void* proto_q) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); + lua_register(L, "collector_qout", collector_qout); + lua_register(L, "lua_local_clock", lua_local_clock); + lua_register(L, "lua_setup_daemon", lua_setup_daemon); + lua_register(L, "prctl_death_kill", prctl_death_kill); + ret = lua_load_do_file(L, "../beeQ/collectors.lua"); if (ret) { goto endLoad; } - lua_register(L, "collector_qout", collector_qout); - // call init. lua_getglobal(L, "work"); lua_pushlightuserdata(L, q); lua_pushlightuserdata(L, proto_q); lua_pushstring(L, g_yaml_file); - ret = lua_pcall(L, 3, 1, err_func); + lua_pushinteger(L, (int)gettidv1()); + ret = lua_pcall(L, 4, 1, err_func); if (ret < 0) { lua_check_ret(ret); goto endCall; @@ -281,6 +330,7 @@ static int app_collector_work(void* q, void* proto_q) { return -1; } + int app_collector_run(struct beeQ* q, void* arg) { int ret = 0; struct beeQ* proto_que = (struct beeQ* )arg; @@ -289,6 +339,7 @@ int app_collector_run(struct beeQ* q, void* arg) { ret = app_collector_work(q, proto_que); if (ret < 0) { perror("collect work run failed."); + exit(1); break; } } diff --git a/source/tools/monitor/unity/beeQ/bees.c b/source/tools/monitor/unity/beeQ/bees.c index f6174e49c4641bb0fb49da394a16ec38351d0f7e..950c4bc9b36d01b2b8afeaf0c4ae7d21c8b53281 100644 --- a/source/tools/monitor/unity/beeQ/bees.c +++ b/source/tools/monitor/unity/beeQ/bees.c @@ -9,6 +9,8 @@ #include #include #include +#include "clock/ee_clock.h" +#include "postQue/postQue.h" #define RUN_THREAD_MAX 8 #define RUN_QUEUE_SIZE 32 @@ -29,17 +31,30 @@ void sig_handler(int num) break; case SIGUSR1: // to stop break; + case SIGUSR2: + break; default: printf("signal %d exit.\n", num); exit(1); } } +static void sig_register(void) { + struct sigaction action; + + action.sa_handler = sig_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = 0; + sigaction(SIGUSR2, &action, NULL); +} + +char ** entry_argv; // for daemon process extern struct beeQ* proto_sender_init(struct beeQ* pushQ); int main(int argc, char *argv[]) { struct beeQ* q; //for proto-buf stream struct beeQ* proto_que; //for trans c to proto-buf stream + entry_argv = argv; if (argc > 1) { g_yaml_file = argv[1]; } @@ -47,6 +62,14 @@ int main(int argc, char *argv[]) { signal(SIGHUP, sig_handler); signal(SIGUSR1, sig_handler); signal(SIGINT, sig_handler); + sig_register(); + + if (calibrate_local_clock() < 0) { + printf("calibrate_local_clock failed.\n"); + exit(1); + } + + postQue_init(); q = beeQ_init(RUN_QUEUE_SIZE, app_recv_setup, @@ -68,7 +91,7 @@ int main(int argc, char *argv[]) { if (pid_outline == 0) { exit(1); } - beaver_init(g_yaml_file); + beaver_init(q, g_yaml_file); fprintf(stderr, "loop exit."); beeQ_stop(q); diff --git a/source/tools/monitor/unity/beeQ/clock/Makefile b/source/tools/monitor/unity/beeQ/clock/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f12bd8983dcf36b35b5c27adbf5aaeff391d7f0a --- /dev/null +++ b/source/tools/monitor/unity/beeQ/clock/Makefile @@ -0,0 +1,17 @@ +CC := gcc +AR := ar +CFLAG := -g +OBJS := ee_clock.o +LIB := liblocalclock.a + +all: $(LIB) + + +%.o: %.c, %.h + $(CC) -c $< -o $@ $(CFLAG) + +$(LIB): $(OBJS) + $(AR) cr $@ $(OBJS) + +clean: + rm -f $(LIB) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/beeQ/clock/ee_clock.c b/source/tools/monitor/unity/beeQ/clock/ee_clock.c new file mode 100644 index 0000000000000000000000000000000000000000..3ccebd204062d102d3aef1bb3bb337d505d52d19 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/clock/ee_clock.c @@ -0,0 +1,59 @@ +// +// 高效时钟获取方案 +// Created by 廖肇燕 on 2023/3/17. +// + +#include "ee_clock.h" +#include +#include +#include + +#define TIME_SECOND_UNIT 100000UL // 睡眠校准时间， + +static ee_clock_t clk_coef = 0; + +static ee_clock_t get_cycles() { + unsigned int hi, lo; + ee_clock_t res; + __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); + res = ((ee_clock_t)lo) | (((ee_clock_t)hi) << 32); + return res; +} + +// 校准时钟 +int calibrate_local_clock(){ + ee_clock_t coef1, coef2; + ee_clock_t t1, t2, t3; + ee_clock_t delta1, delta2; + ee_clock_t res; + + t1 = get_cycles(); + usleep(TIME_SECOND_UNIT); + t2 = get_cycles(); + usleep(TIME_SECOND_UNIT); + t3 = get_cycles(); + + delta1 = t2 - t1; + delta2 = t3 - t2; + + coef1 = delta1 / TIME_SECOND_UNIT; + coef2 = delta2 / TIME_SECOND_UNIT; + + if (coef1 <= 100 || coef2 <= 100) { + fprintf(stderr, "read clock too small.\n"); + return -EIO; + } + + res = 100 * coef1 / coef2; + if (res >= 110 || res <= 90) { + fprintf(stderr, "calibrate local clock failed.\n"); + return -EIO; + } + + clk_coef = (coef1 + coef2) / 2; + return 0; +} + +ee_clock_t get_local_clock() { + return get_cycles() / clk_coef; +} diff --git a/source/tools/monitor/unity/beeQ/clock/ee_clock.h b/source/tools/monitor/unity/beeQ/clock/ee_clock.h new file mode 100644 index 0000000000000000000000000000000000000000..dbe68e3a3f30b9c7a2ff510c116852a45d118551 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/clock/ee_clock.h @@ -0,0 +1,14 @@ +// +// 高效时钟获取方案 +// Created by 廖肇燕 on 2023/3/17. +// + +#ifndef UNITY_EE_CLOCK_H +#define UNITY_EE_CLOCK_H + +typedef unsigned long ee_clock_t; + +int calibrate_local_clock(); +ee_clock_t get_local_clock(); + +#endif //UNITY_EE_CLOCK_H diff --git a/source/tools/monitor/unity/beeQ/collectors.lua b/source/tools/monitor/unity/beeQ/collectors.lua index e92a06f58f49a9eae6fe207bdadcb433f183d16c..b083d7a67df59b4d32f5251b33b07fa0dd22bbac 100644 --- a/source/tools/monitor/unity/beeQ/collectors.lua +++ b/source/tools/monitor/unity/beeQ/collectors.lua @@ -3,6 +3,7 @@ --- Created by liaozhaoyan. --- DateTime: 2022/12/26 11:26 PM --- +package.cpath = package.cpath .. ";../lib/?.so" package.path = package.path .. ";../?.lua;" local dirent = require("posix.dirent") @@ -10,7 +11,7 @@ local unistd = require("posix.unistd") local stat = require("posix.sys.stat") local bit = require("bit") local system = require("common.system") -local ptime = require("posix.time") +local CrbEvent = require("beeQ.rbtree.rbEvent") local srcPath = "../collector/native/" local dstPath = "../collector/lib/" @@ -107,34 +108,32 @@ local function setupFreq(fYaml) end end -local function calcSleep(hope, now) - if hope.tv_nsec >= now.tv_nsec then - return {tv_sec = hope.tv_sec - now.tv_sec, - tv_nsec = hope.tv_nsec - now.tv_nsec} - else - return {tv_sec = hope.tv_sec - now.tv_sec - 1, - tv_nsec = 1e9 + hope.tv_nsec - now.tv_nsec} - end +local function setupMainCollector(que, proto_q, fYaml, tid) + local Cloop = require("collector.loop") + local w = Cloop.new(que, proto_q, fYaml, tid) + local unit = setupFreq(fYaml) + return w, unit end -function work(que, proto_q, yaml) +local function setupPostEngine(que, proto_q, fYaml, tid) + local Cengine = require("collector.postEngine.engine") + local w = Cengine.new(que, proto_q, fYaml, tid) + return w, 1 +end + +function work(que, proto_q, yaml, tid) local fYaml = yaml or "../collector/plugin.yaml" checkSos() - local Cloop = require("collector.loop") - local w = Cloop.new(que, proto_q, fYaml) - local unit = setupFreq(fYaml) - local tStart = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) - while true do - w:work(unit) - local now = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) - local hope = {tv_sec = tStart.tv_sec + unit, tv_nsec = tStart.tv_sec} - local diff = calcSleep(hope, now) - assert(diff.tv_sec >= 0) - local _, s, errno, _ = ptime.nanosleep(diff) - if errno then -- interrupt by signal - print(string.format("new sleep stop. %d, %s", errno, s)) - return 0 - end - tStart = hope - end + local e = CrbEvent.new() + + local main, engine, unit + main, unit = setupMainCollector(que, proto_q, fYaml, tid) + e:addEvent("mainCollector", main, unit) + + engine, unit = setupPostEngine(que, proto_q, fYaml, tid) + engine:setTask(main.postPlugin.tasks) + e:addEvent("postEngine", engine, unit) + engine:setMainloop(main) + + return e:proc() end diff --git a/source/tools/monitor/unity/beeQ/daemon.c b/source/tools/monitor/unity/beeQ/daemon.c new file mode 100644 index 0000000000000000000000000000000000000000..c8339ee0170daf0d73bbccc0b6f138ad1548d11f --- /dev/null +++ b/source/tools/monitor/unity/beeQ/daemon.c @@ -0,0 +1,171 @@ +// +// Created by 廖肇燕 on 2023/3/21. +// + +#include "daemon.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DAEMON_NAME "daemon" +#define DAEMON_NAME_SIZE sizeof(DAEMON_NAME) +#define DAEMON_PIPE_SIZE 4096 + +#define TASK_NAME "unity-mon" +#define TASK_NAME_SIZE sizeof(TASK_NAME) + +extern char** entry_argv; + +static int init_daemon(int fd) { + pid_t pid; + char *s; + + pid = fork(); + if (pid < 0) { + perror("fork error!"); + exit(1); + } + else if (pid > 0) { + exit(0); + } + + // mask signal. + signal(SIGTTOU, SIG_IGN); + signal(SIGTTIN, SIG_IGN); + signal(SIGTSTP, SIG_IGN); + signal(SIGHUP, SIG_IGN); + + // create new pid group, leave to old pid groups. + setsid(); + + // fork a new child process + pid = fork(); + if( pid > 0) { + exit(0); + } + else if (pid< 0) { + exit(1); + } + + // change process name. + s = entry_argv[0]; + strcpy(s, DAEMON_NAME); + s[DAEMON_NAME_SIZE] = '\0'; + + // close all no use fd. + int i; + for (i = 0; i < NOFILE; i ++){ + if (i != fd) { + close(i); + } + } + + // change work dir. ignore all child signal and file mask + chdir("/"); + umask(0); + signal(SIGCHLD, SIG_IGN); + return 0; +} + +static int kill_mon(pid_t mon) { + int fd; + char path[64]; + char buffer[64]; + snprintf(path, 64, "/proc/%d/cmdline", mon); + + if (access(path, F_OK) < 0) { + return errno; + } + + fd = open(path, O_RDONLY); + if (fd < 0) { + return errno; + } + read(fd, buffer, 64); + close(fd); + if (strstr(buffer, TASK_NAME) != NULL) { + return kill(mon, SIGKILL); + } + return 0; +} + +static int loop_daemon(pid_t mon, int period, int fd) { + int ret; + char buff[DAEMON_PIPE_SIZE]; + fd_set read_fds; //读文件操作符 + fd_set except_fds; + struct timeval tv; + + FD_ZERO(&read_fds); + FD_ZERO(&except_fds); + + init_daemon(fd); + while (1) { + FD_SET(fd, &read_fds); + FD_SET(fd, &except_fds); + tv.tv_sec = period * 2; + tv.tv_usec = 0; + + ret = select(fd + 1, &read_fds, NULL, &except_fds, &tv); + + if (ret == 0) { + syslog(LOG_NOTICE, "daemon: select time out\n"); + kill_mon(mon); + return 0; + } else if ( ret == -1) { + syslog(LOG_NOTICE, "daemon: select return %d\n", errno); + kill_mon(mon); + return -errno; + } else { + ret = read(fd, buff, DAEMON_PIPE_SIZE); + if (ret <= 0) { + kill_mon(mon); + return 0; + } else if (strncmp(buff, "stop", 4) == 0) { + exit(0); + } + } + } +} + +static int fork_daemon(int period, int fd0, int fd1) { + pid_t pid; + pid_t mon = getpid(); + + pid = fork(); + if (pid < 0) { + perror("fork failed"); + exit(1); + } + if (pid == 0) { + close(fd1); + loop_daemon(mon, period, fd0); + exit(0); + } else { + wait(NULL); + } + close(fd0); + return fd1; +} + +int setup_daemon(int period) { + int pipefd[2]; + + if(pipe(pipefd) < 0) { + perror("create pipe failed."); + exit(1); + } + + return fork_daemon(period, pipefd[0], pipefd[1]); +} diff --git a/source/tools/monitor/unity/beeQ/daemon.h b/source/tools/monitor/unity/beeQ/daemon.h new file mode 100644 index 0000000000000000000000000000000000000000..d5255a55dfa53d5f4a0d733eaa33cac48aee605f --- /dev/null +++ b/source/tools/monitor/unity/beeQ/daemon.h @@ -0,0 +1,10 @@ +// +// Created by 廖肇燕 on 2023/3/21. +// + +#ifndef UNITY_DAEMON_H +#define UNITY_DAEMON_H + +int setup_daemon(int period); + +#endif //UNITY_DAEMON_H diff --git a/source/tools/monitor/unity/beeQ/foxRecv.lua b/source/tools/monitor/unity/beeQ/foxRecv.lua index 5f1e88bf75b1259d2810f96aee1364d6f6605a7a..1e3c2dd62b45d75c7d5cfca275483187259279d7 100644 --- a/source/tools/monitor/unity/beeQ/foxRecv.lua +++ b/source/tools/monitor/unity/beeQ/foxRecv.lua @@ -7,16 +7,59 @@ require("common.class") local CfoxTSDB = require("tsdb.foxTSDB") - +local system = require("common.system") local CfoxRecv = class("CfoxRecv") +local unistd = require("posix.unistd") +local fcntl = require("posix.fcntl") +local struct = require("struct") + +local function setupCo(fYaml) + local res = system:parseYaml(fYaml) + if res.pushTo then + local fdIn, fdOut = unistd.pipe() + if not fdIn then + print("setup pipe failed.") + os.exit(1) + end + + fcntl.fcntl(fdIn, 1031, 1024 * 1024) + fcntl.fcntl(fdOut, 1031, 1024 * 1024) + lua_push_start(fdIn) + return fdIn, fdOut + end + return nil +end function CfoxRecv:_init_(fYaml) self._fox = CfoxTSDB.new(fYaml) self._fox:setupWrite() + self.fdIn, self.fdOut = setupCo(fYaml) + + if self.fdIn then + self._outFunc = function(stream) self:outToFd(stream) end + else + self._outFunc = function(stream) self._fox:write(stream) end + end end -function CfoxRecv:write(stream) +function CfoxRecv:_del_() + if self.fdIn then + lua_push_stop(); + unistd.close(self.fdIn) + unistd.close(self.fdOut) + end +end + +function CfoxRecv:outToFd(stream) + local len = #stream + local s = struct.pack(" +#include + +#define UNITY_POSTQUE_NUM 4 +#define UNITY_POSTQUE_MSG_SIZE 128 + +// post que for out post +struct unity_postQue { + int num; + pthread_mutex_t mtx; + char msgs[UNITY_POSTQUE_NUM][UNITY_POSTQUE_MSG_SIZE]; +}; + +static struct unity_postQue que; + +int postQue_pull(char *msg) { + int ret; + int i; + + pthread_mutex_lock(&que.mtx); + ret = que.num; + for (i = 0; i < ret; i ++) { + strcat(msg, que.msgs[i]); + strcat(msg, "\n"); + } + que.num = 0; + pthread_mutex_unlock(&que.mtx); + if (ret) { // strip last \n + int len = strlen(msg); + msg[len - 1] = '\0'; + } + return ret; +} + +// post a message +int postQue_post(const char *msg) { + int ret = 0; + int len = strlen(msg); + if (len >= UNITY_POSTQUE_MSG_SIZE) { + return -EINVAL; + } + + pthread_mutex_lock(&que.mtx); + if (que.num < UNITY_POSTQUE_NUM) { + strcpy(que.msgs[que.num], msg); + que.num ++; + } else { + ret = -EAGAIN; + } + pthread_mutex_unlock(&que.mtx); + return ret; +} + +int postQue_init() { + memset(&que, 0, sizeof (struct unity_postQue)); + pthread_mutex_init(&que.mtx, NULL); + return 0; +} diff --git a/source/tools/monitor/unity/beeQ/postQue/postQue.h b/source/tools/monitor/unity/beeQ/postQue/postQue.h new file mode 100644 index 0000000000000000000000000000000000000000..070325e54fec6711f41163a24b394b7aba85615d --- /dev/null +++ b/source/tools/monitor/unity/beeQ/postQue/postQue.h @@ -0,0 +1,16 @@ +// +// Created by 廖肇燕 on 2023/3/24. +// + +#ifndef UNITY_POSTQUE_H +#define UNITY_POSTQUE_H + +#include +#include +#include + +int postQue_pull(char *msg); +int postQue_post(const char *msg); +int postQue_init(); + +#endif //UNITY_POSTQUE_H diff --git a/source/tools/monitor/unity/beeQ/postQue/postQue.lua b/source/tools/monitor/unity/beeQ/postQue/postQue.lua new file mode 100644 index 0000000000000000000000000000000000000000..becbf24d0055dcd24762ea44e074141f919b3583 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/postQue/postQue.lua @@ -0,0 +1,30 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/24 11:13 PM +--- + +local mod = {} +local ffi = require("ffi") + +ffi.cdef [[ +int postQue_pull(char *msg); +int postQue_post(const char *msg); +]] + +local cffi = ffi.load("postQue") + +function mod.pull() + local s = ffi.new("char[?]", 1024) + local ret = cffi.postQue_pull(s) + if ret > 0 then + return ffi.string(s) + end + return nil +end + +function mod.post(msg) + return cffi.postQue_post(msg) +end + +return mod \ No newline at end of file diff --git a/source/tools/monitor/unity/beeQ/proto_queue.lua b/source/tools/monitor/unity/beeQ/proto_queue.lua index ecece0f1ab04fbcc11a73fb2cec5fb63f4cfdceb..91f296493e0e8cb4f78da573cc2d95b3a15c8efb 100644 --- a/source/tools/monitor/unity/beeQ/proto_queue.lua +++ b/source/tools/monitor/unity/beeQ/proto_queue.lua @@ -8,6 +8,8 @@ require("common.class") local CprotoData = require("common.protoData") local CprotoQueue = class("loop") +local cjson = require("cjson.safe") +local json = cjson.new() local system = require("common.system") function CprotoQueue:_init_(que) @@ -77,7 +79,6 @@ function CprotoQueue:_proc(unity_lines, lines) end function CprotoQueue:send(num, pline) - --print(string.format("proto que send a %d message.", num)) local unity_lines = self._ffi.new("struct unity_lines") local lines = self._proto:protoTable() unity_lines.num = num diff --git a/source/tools/monitor/unity/beeQ/pushTo.c b/source/tools/monitor/unity/beeQ/pushTo.c new file mode 100644 index 0000000000000000000000000000000000000000..f89508aebe92499a78a6132d72b5696fabcd7034 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/pushTo.c @@ -0,0 +1,115 @@ +// +// Created by 廖肇燕 on 2023/4/7. +// + +#include "pushTo.h" +#include +#include +#include +#include +#include +#include +#include + +extern char *g_yaml_file; +static int fdIn; +static pthread_t pid_push = 0; + +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); + +static int call_work(lua_State *L, int err_func, int fd, char *fYaml) { + int ret; + lua_Number lret; + + lua_getglobal(L, "work"); + lua_pushinteger(L, fd); + lua_pushstring(L, fYaml); + ret = lua_pcall(L, 2, 1, err_func); + if (ret) { + goto endCall; + } + if (!lua_isnumber(L, -1)) { // check + errno = -EINVAL; + perror("function pushTo.lua init must return a number."); + goto endReturn; + } + lret = lua_tonumber(L, -1); + lua_pop(L, 1); + if (lret < 0) { + errno = -EINVAL; + ret = -1; + perror("pushTo.lua init failed."); + goto endReturn; + } + + return ret; + endReturn: + endCall: + return ret; +} + +static lua_State * push_init(int fd, char* fYaml) { + int ret; + int err_func; + /* create a state and load standard library. */ + lua_State *L = luaL_newstate(); + if (L == NULL) { + perror("new lua failed."); + goto endNew; + } + /* opens all standard Lua libraries into the given state. */ + luaL_openlibs(L); + err_func = lua_reg_errFunc(L); + + ret = lua_load_do_file(L, "../beeQ/pushTo.lua"); + if (ret) { + goto endLoad; + } + + ret = call_work(L, err_func, fd, fYaml); + if (ret < 0) { + goto endCall; + } + + return L; + endCall: + endLoad: + lua_close(L); + endNew: + return NULL; +} + +static int push_work(int fd, char *fYaml) { + push_init(fd, fYaml); + return 0; +} + + +static void* pushTo_worker(void *arg) { + int fd = fdIn; + + push_work(fd, g_yaml_file); + return NULL; +} + +int pushTo_start(int fd) { + int ret; + if (fd > 0) { + fdIn = fd; + ret = pthread_create(&pid_push, NULL, pushTo_worker, NULL); + if (ret < 0) { + perror("create push thread failed"); + exit(1); + } + return 0; + } + return -EINVAL; +} + +void pushTo_stop(void) { + pthread_kill(pid_push, SIGUSR2); + pthread_join(pid_push, NULL); + pid_push = 0; +} diff --git a/source/tools/monitor/unity/beeQ/pushTo.h b/source/tools/monitor/unity/beeQ/pushTo.h new file mode 100644 index 0000000000000000000000000000000000000000..dac8ca3dc58f6a2d36401bc2b812fdf0c15469b0 --- /dev/null +++ b/source/tools/monitor/unity/beeQ/pushTo.h @@ -0,0 +1,11 @@ +// +// Created by 廖肇燕 on 2023/4/7. +// + +#ifndef UNITY_PUSHTO_H +#define UNITY_PUSHTO_H + +int pushTo_start(int fd); +void pushTo_stop(void); + +#endif //UNITY_PUSHTO_H diff --git a/source/tools/monitor/unity/beeQ/pushTo.lua b/source/tools/monitor/unity/beeQ/pushTo.lua new file mode 100644 index 0000000000000000000000000000000000000000..bf301ffedc8bbb7ed1e97d450ef577e55268960c --- /dev/null +++ b/source/tools/monitor/unity/beeQ/pushTo.lua @@ -0,0 +1,18 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/7 01:36 +--- + +package.path = package.path .. ";../?.lua;" + +local coCli = require("httplib.coCli") +local coInflux = require("httplib.coInflux") + +function work(fd, fYaml) + local frame = coCli.new(fd) + local c = coInflux.new(fYaml) + frame:poll(c) + print("end push.") + return 0 +end diff --git a/source/tools/monitor/unity/beeQ/rbtree/Makefile b/source/tools/monitor/unity/beeQ/rbtree/Makefile index 45d070999b45a269bf99aab7d46da95acba65897..7b5383ae15e88dc9744c9a33ee5de02ca5235ed0 100644 --- a/source/tools/monitor/unity/beeQ/rbtree/Makefile +++ b/source/tools/monitor/unity/beeQ/rbtree/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := rbtree.o lrbtree.o SO = lrbtree.so -all: $(SO) install +all: install %.o: %.c $(CC) -c $< -o $@ $(CFLAG) @@ -12,7 +12,7 @@ all: $(SO) install $(SO): $(OBJS) $(CC) -o $@ $(OBJS) $(LDFLAG) -install: +install: $(SO) cp $(SO) ../lib clean: diff --git a/source/tools/monitor/unity/beeQ/rbtree/lrbtree.c b/source/tools/monitor/unity/beeQ/rbtree/lrbtree.c index dd9597402bcff6652b7bce4188643f093b5f3b4c..b218453665959b7d78cdc6e70ba8a4dba62b560f 100644 --- a/source/tools/monitor/unity/beeQ/rbtree/lrbtree.c +++ b/source/tools/monitor/unity/beeQ/rbtree/lrbtree.c @@ -4,6 +4,7 @@ Author: turbobhh Mail: turbobhh@gmail.com CreatedTime: Tue 25 Apr 2017 03:09:31 PM CST ************************************************************************/ + #include #include #include diff --git a/source/tools/monitor/unity/common/rbEvent.lua b/source/tools/monitor/unity/beeQ/rbtree/rbEvent.lua similarity index 76% rename from source/tools/monitor/unity/common/rbEvent.lua rename to source/tools/monitor/unity/beeQ/rbtree/rbEvent.lua index ac823e6e90ad8ed58a6204c02ca1da0d29084a14..c9b45317037d83002ad251fdc545cf29d1021f8d 100644 --- a/source/tools/monitor/unity/common/rbEvent.lua +++ b/source/tools/monitor/unity/beeQ/rbtree/rbEvent.lua @@ -36,8 +36,7 @@ function CrbEvent:_init_() self._nsec = timeNsec() end -function CrbEvent:addEvent(e, period, start, loop) - start = start or false +function CrbEvent:addEvent(name, obj, period, delay, loop) loop = loop or -1 -- -1: 会永远增加下去，大于1 则会递减，减少0 不再使用 if loop == 0 then @@ -45,12 +44,14 @@ function CrbEvent:addEvent(e, period, start, loop) end local beg = os.time() - if not start then + if delay then beg = beg + period + elseif loop > 0 then loop = loop - 1 end local node = { - e = e, + name = name, + obj = obj, t = beg, period = period, loop = loop, @@ -58,9 +59,12 @@ function CrbEvent:addEvent(e, period, start, loop) self._tree:insert(node) end -function CrbEvent:_proc(node) - print(node.e) - if node.loop ~= 0 then -- add to tail. +function CrbEvent:_proc(now, node) + local ret = -1 + if node.obj.work then + ret = node.obj:work(node.period, self) + end + if node.loop ~= 0 and ret ~= -1 then -- add to tail. node.t = node.t + node.period self._tree:insert(node) if node.loop > 0 then @@ -86,16 +90,18 @@ function CrbEvent:proc() while node.t <= now do -- 到了预期时间 self._tree:delete(node) - self:_proc(node) + self:_proc(now, node) node = self._tree:first() end tHope = {tv_sec = tStart.tv_sec + node.t - now, tv_nsec = tStart.tv_sec} diff = calcSleep(tHope, tStart) - local _, s, errno, _ = ptime.nanosleep(diff) - if errno then -- interrupt by signal - print(string.format("new sleep stop. %d, %s", errno, s)) - return 0 + if diff.tv_sec >= 0 then + local _, s, errno, _ = ptime.nanosleep(diff) + if errno then -- interrupt by signal + print(string.format("new sleep stop. %d, %s", errno, s)) + return 0 + end end end end diff --git a/source/tools/monitor/unity/beeQ/rbtree/rbtree.c b/source/tools/monitor/unity/beeQ/rbtree/rbtree.c index 8ac94895c6b44492138d366de4e8ef916e7e167d..f536e2c34c64163945a141cd83c02d8ab52e6c78 100644 --- a/source/tools/monitor/unity/beeQ/rbtree/rbtree.c +++ b/source/tools/monitor/unity/beeQ/rbtree/rbtree.c @@ -1,21 +1,3 @@ -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - (C) 2002 David Woodhouse - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - linux/lib/rbtree.c -*/ - #include "rbtree.h" static void __rb_rotate_left(struct rb_node *node, struct rb_root *root) diff --git a/source/tools/monitor/unity/beeQ/rbtree/rbtree.h b/source/tools/monitor/unity/beeQ/rbtree/rbtree.h index 346ec6a7446e31244da9dce8b462abd34e9461c3..8e4e5a9c412a31db35eb7088536ef167015f0381 100644 --- a/source/tools/monitor/unity/beeQ/rbtree/rbtree.h +++ b/source/tools/monitor/unity/beeQ/rbtree/rbtree.h @@ -1,83 +1,5 @@ -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - linux/include/linux/rbtree.h - To use rbtrees you'll have to implement your own insert and search cores. - This will avoid us to use callbacks and to drop drammatically performances. - I know it's not the cleaner way, but in C (not in C++) to get - performances and genericity... - Some example of insert and search follows here. The search is a plain - normal search over an ordered tree. The insert instead must be implemented - in two steps: First, the code must insert the element in order as a red leaf - in the tree, and then the support library function rb_insert_color() must - be called. Such function will do the not trivial work to rebalance the - rbtree, if necessary. ------------------------------------------------------------------------ -static inline struct page * rb_search_page_cache(struct inode * inode, - unsigned long offset) -{ - struct rb_node * n = inode->i_rb_page_cache.rb_node; - struct page * page; - while (n) - { - page = rb_entry(n, struct page, rb_page_cache); - if (offset < page->offset) - n = n->rb_left; - else if (offset > page->offset) - n = n->rb_right; - else - return page; - } - return NULL; -} -static inline struct page * __rb_insert_page_cache(struct inode * inode, - unsigned long offset, - struct rb_node * node) -{ - struct rb_node ** p = &inode->i_rb_page_cache.rb_node; - struct rb_node * parent = NULL; - struct page * page; - while (*p) - { - parent = *p; - page = rb_entry(parent, struct page, rb_page_cache); - if (offset < page->offset) - p = &(*p)->rb_left; - else if (offset > page->offset) - p = &(*p)->rb_right; - else - return page; - } - rb_link_node(node, parent, p); - return NULL; -} -static inline struct page * rb_insert_page_cache(struct inode * inode, - unsigned long offset, - struct rb_node * node) -{ - struct page * ret; - if ((ret = __rb_insert_page_cache(inode, offset, node))) - goto out; - rb_insert_color(node, &inode->i_rb_page_cache); - out: - return ret; -} ------------------------------------------------------------------------ -*/ - -#ifndef _LINUX_RBTREE_H -#define _LINUX_RBTREE_H +#ifndef _RBTREE_H +#define _RBTREE_H #if defined(container_of) #undef container_of @@ -181,4 +103,4 @@ static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, *rb_link = node; } -#endif /* _LINUX_RBTREE_H */ +#endif /* _RBTREE_H */ diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 9c23dcdd510933a747a26ef07e111ea320f80d48..18b67ff55ed64d162647ef50abe82676987cb365 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -1,5 +1,10 @@ #!/bin/bash +DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd); +echo $DIR +cd $DIR +ulimit -c unlimited + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../collector/native/ @@ -7,7 +12,7 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../beaver/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../install/ export LUA_PATH="../../lua/?.lua;../../lua/?/init.lua;" -export LUA_CPATH="../../lib/?.so;../../lib/loadall.so;" +export LUA_CPATH="./lib/?.so;../../lib/?.so;../../lib/loadall.so;" yaml_path=$1 [ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" diff --git a/source/tools/monitor/unity/collector/btfLoader.lua b/source/tools/monitor/unity/collector/btfLoader.lua new file mode 100644 index 0000000000000000000000000000000000000000..500be2ad8f590274586da97c4082762a19f47b41 --- /dev/null +++ b/source/tools/monitor/unity/collector/btfLoader.lua @@ -0,0 +1,89 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/24 22:09 +--- +require("common.class") + +local unistd = require("posix.unistd") +local posix = require("posix") +local utsname = require("posix.sys.utsname") +local ChttpCli = require("httplib.httpCli") + +local CbtfLoader = class("btfLoader") + +local function regionId() + local url = "http://100.100.100.200/latest/meta-data/region-id" + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + return res.body +end + +local function checkBtf(path) + if unistd.access("/sys/kernel/btf/vmlinux") then + return false + end + if unistd.access(path) then + return false + end + return true +end + +local function checkKo(path) + if unistd.access(path) then + return false + end + return true +end + +local function downBtf(path, region, machine, release) + local url = "https://sysom-".. region ..".oss-".. region .."-internal.aliyuncs.com/home/hive/btf/".. machine .."/vmlinux-" .. release + if not unistd.access("/boot") then + local res, err, errno = posix.mkdir("/boot") + assert(res ~= 0, err .. errno) + end + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + + local file = io.open(path,"wb") + file:write(res.body) + file:close() +end + +local function downKo(path, name, region, machine, release) + local url = "https://sysom-".. region ..".oss-".. region .."-internal.aliyuncs.com/home/hive/sysak/modules/".. machine .."/sysak-" .. release .. ".ko" + if not unistd.access(path) then + local res, err, errno = posix.mkdir(path) + print(res) + assert(res ~= 0, err .. errno) + end + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + + print(path..name) + local file = io.open(path .. name,"wb") + file:write(res.body) + file:close() +end + +function CbtfLoader:_init_(root) + local distro = utsname.uname() + if distro then + local release, machine = distro.release, distro.machine + local path = '/boot/vmlinux-' .. release + local region = regionId() + if checkBtf(path) then + downBtf(path, region, machine, release) + end + local ko = root .. "/lib/" .. release .. "/sysak.ko" + if checkKo(ko) then + local path = root .. "/lib/" .. release .. "/" + ko = downKo(path, "sysak.ko", region, machine, release) + end + end +end + +return CbtfLoader diff --git a/source/tools/monitor/unity/collector/execEngine/forkRun.lua b/source/tools/monitor/unity/collector/execEngine/forkRun.lua new file mode 100644 index 0000000000000000000000000000000000000000..0e6c0f6a46f4afd75a72975061a7030c7ba0f9f5 --- /dev/null +++ b/source/tools/monitor/unity/collector/execEngine/forkRun.lua @@ -0,0 +1,48 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/22 01:34 +--- + +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/12 00:01 +--- + +require("common.class") +local exec = require("common.exec") +local pwait = require("posix.sys.wait") +local pystring = require("common.pystring") +local system = require("common.system") +local CvProc = require("collector.vproc") + +local CforkRun = class("execBase", CvProc) + +function CforkRun:_init_(opts, proto, pffi, mnt) + CvProc._init_(self, proto, pffi, mnt) + self._pid = exec.run(opts.cmd, opts.args) + self._opts = opts + + print("fork run: ", self._pid) +end + +function CforkRun:_del_() + if self._pid then + exec.kill(self._pid) + end + print("kill " .. self._pid) +end + +function CforkRun:proc(elapsed, lines) + local pid, stat, exit = pwait.wait(self._pid, pwait.WNOHANG) + if pid == nil then + error("wait failed " .. stat .. exit) + elseif exit then + print("mon thread exit " .. self._pid) + self._pid = nil + return -1 + end +end + +return CforkRun diff --git a/source/tools/monitor/unity/collector/guard/calcJiffies.lua b/source/tools/monitor/unity/collector/guard/calcJiffies.lua new file mode 100644 index 0000000000000000000000000000000000000000..922f345bcc7ab5c9d6ee6f6bef882741e0a2d40d --- /dev/null +++ b/source/tools/monitor/unity/collector/guard/calcJiffies.lua @@ -0,0 +1,65 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/18 11:34 PM +--- + +local mod = {} +local ptime = require("posix.time") +local unistd = require("posix.unistd") +local system = require("common.system") + +local function read_jiffies(path, procffi) + local s + for line in io.lines(path) do + s = line + break + end + local data = procffi.ffi.new("var_kvs_t") + assert(procffi.cffi.var_input_kvs(procffi.ffi.string(s), data) == 0) + assert(data.no == 10) + + local res = 0 + for i = 0, data.no - 1 do + res = res + tonumber(data.value[i]) + end + return res +end + +local function nproc() + local r, err, errno = unistd.sysconf(84) + if err then + system:posixError("sysconf failed", err, errno) + end + return r +end + +function mod.calc(mnt, procffi) + local t = {tv_sec=0, tv_nsec=2e8} -- 200ms + local path = mnt .. "proc/stat" + local r, err, errno + + local j1 = read_jiffies(path, procffi) + r, err, errno = ptime.nanosleep(t) + if err then + system:posixError("nano sleep failed", err, errno) + end + + local j2 = read_jiffies(path, procffi) + ptime.nanosleep(t) + if err then + system:posixError("nano sleep failed", err, errno) + end + + local j3 = read_jiffies(path, procffi) + local delta1, delta2 = j2 - j1, j3 -j2 + local comp = delta1 / delta2 + + if comp >= 1.1 or comp < 0.9 then + error("calculate jiffies failed.") + end + + return (delta1 + delta2) * 2.5 / nproc() +end + +return mod diff --git a/source/tools/monitor/unity/collector/guard/collector_stat.lua b/source/tools/monitor/unity/collector/guard/collector_stat.lua new file mode 100644 index 0000000000000000000000000000000000000000..0b94abdbf9d22e9981a0e39c1c44c0da10f74bf6 --- /dev/null +++ b/source/tools/monitor/unity/collector/guard/collector_stat.lua @@ -0,0 +1,28 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/19 3:36 PM +--- + +require("common.class") +local pystring = require("common.pystring") +-- refer to https://blog.csdn.net/longyuelang/article/details/114025476 + +local CcollectorStat = class("CcollectorStat") + +function CcollectorStat:_init_(tid) + self._path = string.format("/proc/self/task/%d/stat", tid) +end + +function CcollectorStat:jiffies() + local s + for line in io.lines(self._path) do + s = line + end + local ss = pystring:rsplit(s, ") ", 1) + local rest = ss[2] + local vs = pystring:split(rest, " ", 14) + return tonumber(vs[12]) + tonumber(vs[13]) +end + +return CcollectorStat diff --git a/source/tools/monitor/unity/collector/guard/guardDaemon.lua b/source/tools/monitor/unity/collector/guard/guardDaemon.lua new file mode 100644 index 0000000000000000000000000000000000000000..a637edae22121f1bafd2e1ab4211bb745891df61 --- /dev/null +++ b/source/tools/monitor/unity/collector/guard/guardDaemon.lua @@ -0,0 +1,42 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/22 11:05 PM +--- + +require("common.class") + +local unistd = require("posix.unistd") +local system = require("common.system") +local CguardDaemon = class("guardDaemon") + +local function feedDaemon(fd) + local ws, err, errno = unistd.write(fd, "feed.") + if ws == nil then + system:posixError("feed daemo failed.", err, errno) + end +end + +function CguardDaemon:_init_(resYaml) + local freq = resYaml.config.freq + if resYaml.config.daemon then + self.fd = lua_setup_daemon(freq) + self.feedOnce = function() feedDaemon(self.fd) end + else + self.fd = -1 + self.feedOnce = function() end + end +end + +function CguardDaemon:_del_() + if self.fd > 0 then + unistd.write(self.fd, "stop") + unistd.close(self.fd) + end +end + +function CguardDaemon:feed() + self.feedOnce() +end + +return CguardDaemon diff --git a/source/tools/monitor/unity/collector/guard/guardSched.lua b/source/tools/monitor/unity/collector/guard/guardSched.lua new file mode 100644 index 0000000000000000000000000000000000000000..cccbc15eeb781db1a1f2aa93c67faaa3526fb733 --- /dev/null +++ b/source/tools/monitor/unity/collector/guard/guardSched.lua @@ -0,0 +1,59 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/20 5:16 PM +--- + +require("common.class") +local CcollectorStat = require("collector.guard.collector_stat") +local system = require("common.system") + +local CguardSched = class("guardSched") + +function CguardSched:_init_(tid, procs, names, jperiod) + self._stat = CcollectorStat.new(tid) + self._jperiod = jperiod + self._procs = procs + self._names = names + self._limit = 1e5 +end + +function CguardSched:proc(t, lines) + local toRemove = {} + + local start = lua_local_clock() -- unit us + local stop = 0 + local j1 = self._stat:jiffies() + local ret + + for i, obj in ipairs(self._procs) do + ret = obj:proc(t, lines) + + if ret == -1 then + table.insert(toRemove, i) + goto continue + end + + stop = lua_local_clock() + if stop - start >= self._limit then -- + local j2 = self._stat:jiffies() + if j2 - j1 >= self._limit / 1e6 * self._jperiod * 3 / 4 then -- 3/4 time used bye plugin + table.insert(toRemove, i) + end + end + start = stop + + ::continue:: + end + + if #toRemove > 0 then + system:reverseTable(toRemove) -- list should reverse at first. + for _, i in ipairs(toRemove) do + print("remove " .. self._names[i]) + table.remove(self._procs, i) + table.remove(self._names, i) + end + end +end + +return CguardSched diff --git a/source/tools/monitor/unity/collector/guard/guardSelfStat.lua b/source/tools/monitor/unity/collector/guard/guardSelfStat.lua new file mode 100644 index 0000000000000000000000000000000000000000..bbdc5e3ddab8e7f3a96b2693358dd810590c4e5c --- /dev/null +++ b/source/tools/monitor/unity/collector/guard/guardSelfStat.lua @@ -0,0 +1,96 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/23 8:15 AM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +-- refer to https://blog.csdn.net/longyuelang/article/details/114025476 + +local CguardSelfStat = class("guardSelfStat", CvProc) + +local function readProc(path) + local s + for line in io.lines(path) do + s = line + end + local ss = pystring:rsplit(s, ") ", 1) + local rest = ss[2] + local vs = pystring:split(rest, " ") + local user, sys = tonumber(vs[12]), tonumber(vs[13]) + local vsize, rss = tonumber(vs[21]), tonumber(vs[22]) + return user, sys, vsize, rss +end + +function CguardSelfStat:_init_(proto, pffi, mnt, resYaml, jperiod) + CvProc._init_(self, proto, pffi, mnt, nil) + self._path = "/proc/self/stat" + + self._lastUser, self._lastSys, _, _ = readProc(self._path) + self._period = jperiod + self._cpuLimit = resYaml.config.limit.cpu * jperiod / 100 + self._memLimit = resYaml.config.limit.mem * 1024 * 1024 +end + +local function rssRssAnon() + local anon = 0 + + local f = io.open("/proc/self/status") + for line in f:lines() do + if pystring:startswith(line, "RssAnon:") then + local res = pystring:split(line) + anon = tonumber(res[2]) * 1024 + end + end + f:close() + + return anon +end + +function CguardSelfStat:proc(elapsed, lines) + CvProc.proc(self) + + local user, sys, vsize, rss = readProc(self._path) + local _user, _sys = user - self._lastUser, sys - self._lastSys + local cpus = _user, _sys + + self._lastUser, self._lastSys = user, sys + if cpus > self._cpuLimit * elapsed then + print("last cpu usage overflow." .. cpus) + os.exit(1) + end + + local anon = rssRssAnon() + if anon > self._memLimit then + print("last mem usage overflow." .. rss) + os.exit(1) + end + local vs = { + { + name = "user", + value = _user * 100 / (self._period * elapsed) + }, + { + name = "sys", + value = _sys * 100 / (self._period * elapsed) + }, + { + name = "vsize", + value = vsize + }, + { + name = "rss", + value = rss * 4096 + }, + { + name = "anon", + value = anon, + }, + } + self:appendLine(self:_packProto("self_stat", nil, vs)) + self:push(lines) +end + +return CguardSelfStat \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/interface/sig_stop.c b/source/tools/monitor/unity/collector/interface/sig_stop.c index 769e743f8cd3dfdd3dac774b1e230630771f508b..c96b6823a9a4afe243c2c46ad2bebffe9f835916 100644 --- a/source/tools/monitor/unity/collector/interface/sig_stop.c +++ b/source/tools/monitor/unity/collector/interface/sig_stop.c @@ -28,19 +28,6 @@ void plugin_thread_stop(pthread_t tid) { } } -static void stop_signal_handler(int no) { - ; -} - -static void sig_register(void) { - struct sigaction action; - - action.sa_handler = stop_signal_handler; - sigemptyset(&action.sa_mask); - action.sa_flags = 0; - sigaction(SIGUSR2, &action, NULL); -} - static void bump_memlock_rlimit1(void) { struct rlimit rlim_new = { @@ -57,7 +44,6 @@ static void bump_memlock_rlimit1(void) void plugin_init(void) { bump_memlock_rlimit1(); ksym_setup(1); - sig_register(); working = 1; } diff --git a/source/tools/monitor/unity/collector/io/diskFifo.lua b/source/tools/monitor/unity/collector/io/diskFifo.lua new file mode 100644 index 0000000000000000000000000000000000000000..ac57e6edcb9e70f3d9d8fcdd6b89f558b725f591 --- /dev/null +++ b/source/tools/monitor/unity/collector/io/diskFifo.lua @@ -0,0 +1,75 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/12 15:51 +--- + +require("common.class") +local system = require("common.system") +local Cfifo = require("common.fifo") + +local CdiskFifo = class("diskFifo", Cfifo) + +function CdiskFifo:_init_(maxLen) + Cfifo._init_(self, maxLen) + self._nr = 0 +end + +function CdiskFifo:push(v) + Cfifo.push(self, v) + self._nr = self._nr + 1 +end + +function CdiskFifo:iowait() + local sum = 0 + local value + local cells = {} + local len = self:len() + local c = 0 + + if len < self:capacity() then + return + end + + for _, v in pairs(self.list) do + c = c + 1 + value = v.iowait + cells[c], sum = value, sum + value + end + return {max = math.max(unpack(cells)), + min = math.min(unpack(cells)), + nr = self._nr, + avg = sum / len, + last = value} +end + +function CdiskFifo:values(disk, key) + local c = 0 + local sum = 0 + local value + local cells = {} + local len = self:len() + + if len < self:capacity() then + return + end + + local d + for _, v in pairs(self.list) do + d = v[disk] + if d then + value = d[key] + c = c + 1 + cells[c], sum = value, sum + value + else -- not full + return + end + end + return {max = math.max(unpack(cells)), + min = math.min(unpack(cells)), + nr = self._nr, + avg = sum / len, + last = value} +end + +return CdiskFifo diff --git a/source/tools/monitor/unity/collector/io/exceptCheck.lua b/source/tools/monitor/unity/collector/io/exceptCheck.lua new file mode 100644 index 0000000000000000000000000000000000000000..8947e0bd4bf5eb8ee928aed55f1fcab9bc7eff3b --- /dev/null +++ b/source/tools/monitor/unity/collector/io/exceptCheck.lua @@ -0,0 +1,252 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/12 16:26 +--- + +require("common.class") +local system = require("common.system") +local CdiskFifo = require("collector.io.diskFifo") +local CexecptCheck = class("CexecptCheck") + +-- [[ v format +-- iowait: 1 +-- disk_a: { +-- iops = 1, +-- bps = 2, +-- qusize = 3, +-- await = 4, +-- util = 5, +-- } +-- ]] + +local fifoSize = 60 +local keys = {"iops", "bps", "qusize", "await", "util"} + +local function addItem() + return { + baseThresh = { + nrSample = 0, + curWinMinVal = 1e9, + curWinMaxVal = 0, + moveAvg = 0, + last = 0, + thresh = 0 + }, + compensation = { + thresh = 0, + shouldUpdThreshComp = true, + decRangeThreshAvg = 0, + decRangeCnt = 0, + minStableThresh = 1e9, + maxStableThresh = 0, + stableThreshAvg = 0, + nrStableThreshSample = 0, + }, + dynTresh = 1e9, + usedWin = 0, + } +end + +local function addItems() + local ret = {} + for _, key in ipairs(keys) do + ret[key] = addItem() + end + return ret +end + +function CexecptCheck:_init_(diag) + self._diag = diag + + self._fifo = CdiskFifo.new(fifoSize) + self._waitItem = addItem() + self._diskItem = {} + + self._cpuStatIowait = {sum = 0, iowait = 0} + self._uploadInter = 0 + self._exceptionStat = {system = {['IOwait-High'] = {cur = 0, max = 0}}} + self._dataStat = {system = {iowait= 0}} + + self._diagSwitch = { + diagIowait = { sw = self._diag.cfg.diagIowait, + esi = 'IOwait-High'}, + diagIoburst = { sw = self._diag.cfg.diagIoburst, + esi ='IO-Delay'}, + diagIolat = {sw = self._diag.cfg.diagIolat, + esi = 'IO-Burst'}, + diagIohang = {sw = self._diag.cfg.diagIohang, + esi = 'IO-Hang'} + } +end + +local function calcBase(item, vs) + local bt = item.baseThresh + + local min, max, avg, nr = vs.min, vs.max, vs.avg, vs.nr + + bt.nrSample = nr + bt.curWinMinVal = math.min(min, bt.curWinMinVal) + bt.curWinMaxVal = math.max(max, bt.curWinMaxVal) + bt.last = vs.last + + local nrThreshSample = nr + 1 - fifoSize + local thresh = math.max(max - avg, avg - min) + local threshAvg = (bt.thresh * (nrThreshSample - 1) + thresh) / nrThreshSample + bt.thresh = threshAvg + bt.moveAvg = avg + + local usedWin = item.usedWin + usedWin = usedWin + 1 + if usedWin >= fifoSize then + bt.curWinMinVal = 1e9 + bt.curWinMaxVal = 0 + item.usedWin = 0 + else + item.usedWin = usedWin + end + return thresh +end + +local function calcStableThresh(ct, curBaseThresh, curThresh) + local avg = ct.decRangeThreshAvg + + if (curThresh - avg) < ((curBaseThresh - avg) / 10.0) then + local nrStableThreshSample = ct.nrStableThreshSample + + local tSum = ct.stableThreshAvg * ct.nrStableThreshSample + curThresh + nrStableThreshSample = nrStableThreshSample + 1 + + ct.nrStableThreshSample = nrStableThreshSample + ct.stableThreshAvg = tSum / nrStableThreshSample + ct.minStableThresh = math.min(ct.minStableThresh, curThresh) + ct.maxStableThresh = math.max(ct.maxStableThresh, curThresh) + + if nrStableThreshSample > fifoSize * 1.5 then + ct.thresh = math.max(ct.stableThreshAvg - ct.minStableThresh, + ct.maxStableThresh - ct.stableThreshAvg) + ct.shouldUpdThreshComp = false + ct.minStableThresh = 1e9 + ct.maxStableThresh = 0 + ct.stableThreshAvg, ct.decRangeThreshAvg = 0, 0 + ct.nrStableThreshSample, ct.decRangeCnt = 0, 0 + end + end +end + +local function calcCompThresh(item, lastBaseThresh, curThresh) + local curBaseThresh = item.baseThresh.thresh + local ct = item.compensation + + if ct.shouldUpdThreshComp and (ct.thresh < curThresh or item.usedWin == 0) then + ct.thresh = curThresh + end + + if curBaseThresh < lastBaseThresh then + local decRangeCnt = ct.decRangeCnt + local decRangeThreshAvg = ct.decRangeThreshAvg + local tSum = decRangeThreshAvg * decRangeCnt + curThresh + + decRangeCnt = decRangeCnt + 1 + ct.decRangeThreshAvg = tSum / decRangeCnt + ct.decRangeCnt = decRangeCnt + + if decRangeCnt >= fifoSize * 1.5 then + calcStableThresh(ct, curBaseThresh, curThresh) + else + ct.minStableThresh = 1e9 + ct.maxStableThresh = 0 + ct.stableThreshAvg, ct.decRangeThreshAvg = 0, 0 + ct.nrStableThreshSample, ct.decRangeCnt = 0, 0 + end + end +end + +local function updateDynThresh(item, vs) + local bt = item.baseThresh + local ct = item.compensation + local lastBaseThresh = bt.thresh + local curThresh = calcBase(item, vs) + + calcCompThresh(item, lastBaseThresh, curThresh) + item.dynTresh = bt.thresh + bt.moveAvg + ct.thresh + --print("thresh: ", item.dynTresh) +end + +function CexecptCheck:calcs() + local iowaits = self._fifo:iowait() + if iowaits then + updateDynThresh(self._waitItem, iowaits) + self:checkIOwaitException(self._waitItem) + end + + local vs + for disk, item in pairs(self._diskItem) do + for _, key in ipairs(keys) do + vs = self._fifo:values(disk, key) + if vs then + updateDynThresh(item[key], vs) + end + end + end +end + +function CexecptCheck:addValue(v) + for disk, _ in pairs(v) do -- new iowait names one disk + if disk ~= "iowait" then + if not system:keyIsIn(self._diskItem, disk) then + self._diskItem[disk] = addItems() + end + end + end + for disk, _ in pairs(self._diskItem) do -- del + if not system:keyIsIn(v, disk) then + self._diskItem[disk] = nil + end + end + self._fifo:push(v) + self:calcs() +end + +local function disableThreshComp(item) + local ct = item.compensation + local bt = item.baseThresh + if ct.shouldUpdThreshComp then + ct.shouldUpdThreshComp = false + item.dynTresh = bt.thresh + bt.moveAvg + ct.thresh = 0.000001 + end +end + +function CexecptCheck:checkIOwaitException(item) + local iowait = item.baseThresh.last + local dataStat = self._dataStat.system + local es = self._exceptionStat.system['IOwait-High'] + local uploadInter = self._uploadInter + + if iowait >= self._diag.cfg.iowait then + disableThreshComp(item) + end + + dataStat.iowait = (dataStat.iowait * (uploadInter - 1) + iowait) / uploadInter + + local minThresh = self._diag.cfg.iowait + local iowaitThresh = math.max(item.dynTresh, minThresh) + if minThresh >= iowaitThresh then + es.cur = es.cur + 1 + local diagSW = self._diagSwitch.diagIowait + local rDiagValid = nil + end +end + +function CexecptCheck:checks() + local uploadInter = self._uploadInter + + if uploadInter % fifoSize == 0 then + self._uploadInter = 1 + else + self._uploadInter = uploadInter + 1 + end +end + +return CexecptCheck diff --git a/source/tools/monitor/unity/collector/io/io_diagnose.lua b/source/tools/monitor/unity/collector/io/io_diagnose.lua new file mode 100644 index 0000000000000000000000000000000000000000..5725acd96944f7e49ed17d1ee73eddbfc70435bc --- /dev/null +++ b/source/tools/monitor/unity/collector/io/io_diagnose.lua @@ -0,0 +1,126 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/29 8:25 PM +--- + +require("common.class") +local CprotoData = require("common.protoData") +local CvProto = require("collector.vproto") +local CioDiagnose = class("ioDiagnose", CvProto) +local system = require("common.system") +local procffi = require("collector.native.procffi") +local pystring = require("common.pystring") +local CexceptCheck = require("collector.io.exceptCheck") +local unistd = require("posix.unistd") + +local function readIoWait(fStat) + local data = procffi["ffi"].new("var_kvs_t") + local stat = io.open(fStat, "r") + local s = stat:read() + assert(procffi["cffi"].var_input_kvs(procffi["ffi"].string(s), data) == 0) + stat:close() + local sum = 0 + for i = 0, data.no do + sum = sum + tonumber(data.value[i]) + end + + return sum, tonumber(data.value[4]) +end + +local function diskExist(dirSys, disk) + local path = dirSys .. disk + return unistd.access(path) +end + +local function distStat(fDisks) + local disks = {} + for line in io.lines(fDisks) do + local s = pystring:split(line, nil, 3)[4] + local data= procffi["ffi"].new("var_kvs_t") + assert(procffi["cffi"].var_input_kvs(procffi["ffi"].string(s), data) == 0) + local disk = procffi["ffi"].string(data.s) + disks[disk] = data + end + return disks +end + +local function pickValue(value) + return { + iops = tonumber(value[0] + value[4]), -- 1 5 + bps = tonumber(value[2] + value[6]) * 512, -- 3 7 + qusize = tonumber(value[10]) / 1000.0, -- 11 + await = tonumber(value[3] + value[7]), -- 4 8 + util = tonumber(value[9]) / 10.0 -- 10 + } +end + +function CioDiagnose:_init_(que, proto_q, fYaml, tid) + CvProto._init_(self, CprotoData.new(que)) + self._tid = tid + + local res = system:parseYaml(fYaml) + local proc_path = res.config.proc_path + self.fStat = proc_path .. "proc/stat" + self.fDiskStat = proc_path .. "proc/diskstats" + self.dirSys = proc_path .. "sys/block/" + + self._lastCpuTotal, self._lastCpuIO = readIoWait(self.fStat) + self._disks = {} + self._disksLast = {} + self:readProc() + self:storeProc() + + local diag = res.ioDiagnose + self._check = CexceptCheck.new(diag) +end + +function CioDiagnose:readProc() + local disks = distStat(self.fDiskStat) + for disk, value in pairs(disks) do + if diskExist(self.dirSys, disk) then + self._disks[disk] = pickValue(value.value) + else -- not exist any more + if self._disks[disk] then + self._disks[disk] = nil + end + end + end +end + +function CioDiagnose:storeProc() + self._disksLast = self._disks + self._disks = {} +end + +function CioDiagnose:diff(t, iowait) + local res = {iowait = iowait} + for disk, value in pairs(self._disks) do + local vLast = self._disksLast[disk] + if vLast then + local vs = {} + for k, v in pairs(value) do + vs[k] = (v - vLast[k]) / t + end + if vs.iops > 0 then + vs.await = vs.await / vs.iops + end + res[disk] = vs + end + end + return res +end + +function CioDiagnose:work(t) + local cpuTotal, cpuIO = readIoWait(self.fStat) + local iowait = (cpuIO - self._lastCpuIO) * 100 / (cpuTotal - self._lastCpuTotal) / t + self._lastCpuTotal, self._lastCpuIO = cpuTotal, cpuIO + + self:readProc() + self._check:checks() + local res = self:diff(t, iowait) + self._check:addValue(res) + self:storeProc() +end + +return CioDiagnose diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 28b23464f21b2a12d2dd6a6682dbac7510cc2f34..517dfb9a4f8d30061f04a69b777994a1b3b4c72d 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -7,39 +7,74 @@ require("common.class") local CprotoData = require("common.protoData") local procffi = require("collector.native.procffi") -local Cplugin = require("collector.plugin") local system = require("common.system") +local CbtfLoader = require("collector.btfLoader") +local CpluginManager = require("collector.pluginManager") +local calcJiffies = require("collector.guard.calcJiffies") +local CguardSched = require("collector.guard.guardSched") +local CguardDaemon = require("collector.guard.guardDaemon") +local CguardSelfStat = require("collector.guard.guardSelfStat") +local CpostPlugin = require("collector.postPlugin.postPlugin") +local CforkRun = require("collector.execEngine.forkRun") local Cloop = class("loop") -function Cloop:_init_(que, proto_q, fYaml) +function Cloop:_init_(que, proto_q, fYaml, tid) + CbtfLoader.new("../../../..") -- setup btf local res = system:parseYaml(fYaml) + self._daemon = CguardDaemon.new(res) self._proto = CprotoData.new(que) + self._tid = tid self:loadLuaPlugin(res, res.config.proc_path) - self._plugin = Cplugin.new(self._proto, procffi, que, proto_q, fYaml) + self:forkRun(res) + local jperiod = calcJiffies.calc(res.config.proc_path, procffi) -- + + self._guardSched = CguardSched.new(tid, self._procs, self._names, jperiod) + self.soPlugins = CpluginManager.new(procffi, proto_q, res, tid, jperiod) + self._guardStat = CguardSelfStat.new(self._proto, procffi, "/", res, jperiod) + self.postPlugin = CpostPlugin.new(self._proto, procffi, res) end function Cloop:loadLuaPlugin(res, proc_path) local luas = res.luaPlugins self._procs = {} + self._names = {} if res.luaPlugins then - for i, plugin in ipairs(luas) do + local c = 1 + for _, plugin in ipairs(luas) do local CProcs = require("collector." .. plugin) - self._procs[i] = CProcs.new(self._proto, procffi, proc_path) + self._procs[c] = CProcs.new(self._proto, procffi, proc_path) + self._names[c] = plugin + c = c + 1 + end + end + print("add " .. system:keyCount(self._procs) .. " lua plugin.") +end + +function Cloop:forkRun(res) + local runs = res.forkRun + local c = system:keyCount(self._procs) + if runs then + for _, run in ipairs(runs) do + c = c + 1 + self._procs[c] = CforkRun.new(run, self._proto, procffi) + self._names[c] = run.cmd end end - print("add " .. #self._procs .. " lua plugin.") end function Cloop:work(t) local lines = self._proto:protoTable() - for _, obj in pairs(self._procs) do - obj:proc(t, lines) - end - self._plugin:proc(t, lines) + + self._guardSched:proc(t, lines) + self.soPlugins:proc(t, lines) + self._guardStat:proc(t, lines) + self.postPlugin:proc(t, lines) + local bytes = self._proto:encode(lines) self._proto:que(bytes) + self._daemon:feed() end return Cloop diff --git a/source/tools/monitor/unity/collector/outline/Makefile b/source/tools/monitor/unity/collector/outline/Makefile index 118b288e3d6a5e9debda4ce86f7f5b4ef392527d..79d46ef84c1a374bbafa5fe2327cb3bd49f63449 100644 --- a/source/tools/monitor/unity/collector/outline/Makefile +++ b/source/tools/monitor/unity/collector/outline/Makefile @@ -4,7 +4,7 @@ CFLAG := -g OBJS := outline.o LIB := liboutline.a -all: $(DEPMOD) $(LIB) +all: $(LIB) %.o: %.c, %.h @@ -14,4 +14,4 @@ $(LIB): $(OBJS) $(AR) cr $@ $(OBJS) clean: - rm -f $(EXEC) $(OBJS) \ No newline at end of file + rm -f $(LIB) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 90ec914fe35f5f92996083b52bad70c30f670cb7..89d4a7723bd629542f27d7d384d9d721e0d2c224 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -4,6 +4,8 @@ #include "outline.h" #include +#include +#include extern int lua_reg_errFunc(lua_State *L); extern int lua_check_ret(int ret); @@ -125,6 +127,7 @@ static int outline_work(struct beeQ* q, char *fYaml) { static int outline_run(struct beeQ* q, void* arg) { int ret; char *fYaml = (char *)arg; + prctl(PR_SET_NAME, (unsigned long)"outline_run"); while (1) { ret = outline_work(q, fYaml); diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 57823c0fec1cbf98c51ad7be78fa62fefc495208..a37801f4fd6e1e9c730aefb03fdc6ee8c27ec2da 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -5,65 +5,51 @@ --- require("common.class") -local system = require("common.system") local Cplugin = class("plugin") - -function Cplugin:_init_(proto, procffi, que, proto_q, fYaml) - self._proto = proto - - local res = system:parseYaml(fYaml) - self:setProcSys(procffi, res.config) - - self._sig_cffi = procffi["cffi"] - self._sig_cffi.ffi_plugin_init() - - self._ffi = require("collector.native.plugincffi") - self:setup(res.plugins, proto_q) +local dockerinfo = require("common.dockerinfo") +local cjson = require("cjson.safe") +local json = cjson.new() + +function Cplugin:_init_(resYaml, ffi, proto_q, so, loop) + self._ffi = ffi + self._cffi = self._ffi.load(so) + self._cffi.init(proto_q) + self._so = so + self._loop = loop or -1 + self.proc_fs = resYaml.config["proc_path"] or "/" + self.fill_arg = {["podname"]="pid"} end function Cplugin:_del_() - self._sig_cffi.ffi_plugin_stop() - for _, plugin in ipairs(self._plugins) do - local cffi = plugin.cffi - cffi.deinit() - end - self._sig_cffi.ffi_plugin_deinit() + print("uninstall " .. self._so) + self._cffi.deinit() end -function Cplugin:setProcSys(procFFI, config) - local proc = config["proc_path"] or "/" - local sys = config["sys_path"] or "/" +function Cplugin:load_label(unity_line, line) - procFFI.cffi.ffi_set_unity_proc(procFFI.ffi.string(proc)) - procFFI.cffi.ffi_set_unity_sys(procFFI.ffi.string(sys)) -end + local c = #line.ls + local table_dict = {} -function Cplugin:setup(plugins, proto_q) - self._plugins = {} - for _, plugin in ipairs(plugins) do - local so = plugin.so - if so then - print(so) - local cffi = self._ffi.load(so) - local plugin = { - so = plugin.so, - cffi = cffi - } - cffi.init(proto_q); - table.insert(self._plugins, plugin) - end + for i=0, 4 - 1 do + local name = self._ffi.string(unity_line.indexs[i].name) + local index = self._ffi.string(unity_line.indexs[i].index) + table_dict[name] = index end -end -function Cplugin:load_label(unity_line, line) - local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then c = c + 1 - line.ls[c] = {name = name, index = index} + if index == "?" and self.fill_arg[name] and table_dict[self.fill_arg[name]] then + if name == "podname" then + local podname = dockerinfo:get_podname_pid(table_dict[self.fill_arg[name]], self.proc_fs) + line.ls[c] = {name = name, index = podname} + end + else + line.ls[c] = {name = name, index = index} + end else return end @@ -112,14 +98,20 @@ function Cplugin:_proc(unity_lines, lines) end function Cplugin:proc(t, lines) - for _, plugin in ipairs(self._plugins) do - local cffi = plugin.cffi - local unity_lines = self._ffi.new("struct unity_lines") - local res = cffi.call(t, unity_lines) - if res == 0 then - self:_proc(unity_lines, lines) + local unity_lines = self._ffi.new("struct unity_lines") + local res = self._cffi.call(t, unity_lines) + if res == 0 then + self:_proc(unity_lines, lines) + end + self._ffi.C.free(unity_lines.line) -- should free memory. + + local loop = self._loop + if loop > 0 then + loop = loop - 1 + if loop == 0 then + return -1 -- to remove end - self._ffi.C.free(unity_lines.line) -- should free memory. + self._loop = loop end end diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index e0d0d92ec1a64eab8a4f8afeb5bf92c679a72b0b..175441db6cf0b863f8b2c4fbebf233cd7da4c670 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -1,9 +1,10 @@ config: freq: 20 # unit second + daemon: true port: 8400 # bind port bind_addr: 0.0.0.0 # bind ip backlog: 32 # listen backlog - identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg) + identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg), env(need name arg) mode: curl url: "http://100.100.100.200/latest/meta-data/instance-id" # name: test_specify @@ -15,9 +16,51 @@ config: db: rotate: 7 # tsdb file retention time, unit day budget: 200 # max query buffer from tsdb. + limit: + cpu: 30 # unit % + mem: 40 # unit mb + tasks: 10 # monitor 10 pid max. + +forkRun: + - + cmd: "/usr/bin/python" + args: ["../test/curl/forkRun.py"] + +oss: + bucket: "netinfo-shenzhen" + endPoint: "oss-cn-shenzhen.aliyuncs.com" + ak: "ak" + sk: "sk" + +diagnose: + io_hang: + block: 60 + time: 15 + cmd: "../../../iosdiag" + report: + title: "iosdiag" + files: + - "/var/log/sysak/iosdiag/hangdetect/result.log.stat" + - "/var/log/sysak/iosdiag/hangdetect/result.log.seq" + net_edge: + block: 300 + time: 60 + so: + virtiostat: 15 + cmd: "../../../netCli" + jruntime: + block: 60 + time: 30 + cmd: "../../../java_collect" outline: - - /tmp/sysom + - /var/sysom/outline + +pushTo: + to: "Influx" + host: "ld-wz9d17b514mg6kjkx-proxy-tsdb.lindorm.rds.aliyuncs.com" + port: 8242 + url: "/api/v2/write?db=lua" container: mode: "pods" @@ -67,6 +110,9 @@ plugins: - so: gpuinfo description: "collect gpuinfo" + - + so: pmu_cpi + description: "collect cpi" metrics: - @@ -221,3 +267,13 @@ metrics: #head: value #help: "get pod alloc page used" #type: "gauge" + - title: sysak_pmu_events_percpu + from: pmu_events_percpu + head: value + help: "pmu events of percpu" + type: "gauge" + - title: sysak_pmu_cpi + from: pmu cpi + head: value + help: "pmu info of cycles/instructions." + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 9c538406f8e6c8e4b1cf7c31c221c52de158cfaa..7efdddc1c17ac8e3e92fad8bbce4b24337397fa3 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -5,7 +5,7 @@ OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq gpuinfo +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg unity_nosched unity_irqoff cpudist cpu_bled net_health net_retrans netlink cpufreq gpuinfo pmu_events virtout sum_retrans virtiostat all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/cpu_bled/Makefile b/source/tools/monitor/unity/collector/plugin/cpu_bled/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fe795e18fc90f11ee5eda9e026e7b120fcb4aeb4 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpu_bled/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := cpu_bled.o +SO := libcpu_bled.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.c b/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.c new file mode 100644 index 0000000000000000000000000000000000000000..f22fa9db6459e09407492749533f1bfaa17ac19a --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.c @@ -0,0 +1,28 @@ +// +// Created by 廖肇燕 on 2023/3/21. +// + +#include "cpu_bled.h" +#include +#include + +static int bled_wait = 0; + +int init(void * arg) { + printf("setup for cpu_bled.\n"); + return 0; +} + +int call(int t, struct unity_lines* lines) { + bled_wait ++; + if (bled_wait > 10) { + printf("inject cpu bled.\n"); + time_t now = time(NULL) + 3; + while (time(NULL) <= now); + } + return 0; +} + +void deinit(void) { + printf("cpu bled uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.h b/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.h new file mode 100644 index 0000000000000000000000000000000000000000..9272511ab116446fc5ee0bf47ec3e81b26aa5593 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpu_bled/cpu_bled.h @@ -0,0 +1,14 @@ +// +// Created by 廖肇燕 on 2023/3/21. +// + +#ifndef UNITY_CPU_BLED_H +#define UNITY_CPU_BLED_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + +#endif //UNITY_CPU_BLED_H diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c index 1dc01053bd26eb1e96b899b233c1b17012c8b2db..cd3da8aec56b644cfb563b9a37ed344286aba2c8 100644 --- a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c @@ -33,7 +33,7 @@ int sched_switch_hook(struct sched_switch_args *args){ } pv = bpf_map_lookup_elem(&start, &prev); if (pv && ts > *pv) { - hist10_push((struct bpf_map_def *)&cpudist, (ts - *pv) / 1000); + hist10_push((struct bpf_map_def *)&cpudist, (ts - *pv) / 1000 * 10); } return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c index 5ea28bacb5c54b54cf2852f898c8352843461953..0dfa928ae757eb98552c79a3d491a67a64d40949 100644 --- a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c @@ -17,7 +17,9 @@ int init(void *arg) int ret; printf("cpudist plugin install.\n"); ret = LOAD_SKEL_OBJECT(cpudist, perf); - dist_fd = coobpf_map_find(cpudist->obj, "cpudist"); + if (ret >= 0) { + dist_fd = coobpf_map_find(cpudist->obj, "cpudist"); + } return ret; } diff --git a/source/tools/monitor/unity/collector/plugin/kmsg/kmsg.c b/source/tools/monitor/unity/collector/plugin/kmsg/kmsg.c index 3f1dd1e7b49e53c58ddec4645a52665a4b63ec9d..591159f9db65bad32b41761ca5ecac121e8adfb1 100644 --- a/source/tools/monitor/unity/collector/plugin/kmsg/kmsg.c +++ b/source/tools/monitor/unity/collector/plugin/kmsg/kmsg.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #define KMSG_LINE 8192 @@ -34,16 +36,17 @@ static int kmsg_thread_func(struct beeQ* q, void * arg) { int fd; int ret; char buff[KMSG_LINE]; + prctl(PR_SET_NAME, (unsigned long)"kmsg collector"); fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); if (fd < 0) { goto endOpen; - } + }/* ret = lseek(fd, 0, SEEK_DATA); if (ret < 0) { perror("kmsg seek error"); goto endLseek; - } + }*/ ret = 1; // strip old message. while (ret > 0) { @@ -76,6 +79,7 @@ static int kmsg_thread_func(struct beeQ* q, void * arg) { perror("kmsg read2 failed."); goto endRead; } + buff[ret -1] = '\0'; unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c index 32aacfb70da732f31b8395890edd8b0e9ec266fb..5bd2eabc98f3647173ba44a8f760dd0cc0bbcdbd 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -4,7 +4,7 @@ #include #include -BPF_HASH(outCnt, int, u64, 2); +BPF_ARRAY(outCnt, u64, 2); BPF_ARRAY(netHist, u64, 20); SEC("kprobe/tcp_validate_incoming") @@ -15,8 +15,7 @@ int j_tcp_validate_incoming(struct pt_regs *ctx) { if (ms > 0) { add_hist((struct bpf_map_def *)&outCnt, 0, ms); add_hist((struct bpf_map_def *)&outCnt, 1, 1); - hist10_push((struct bpf_map_def *)&netHist, ms); + hist10_push((struct bpf_map_def *)&netHist, ms * 10); } return 0; } - diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c index de4a53093227d335c06e0c2d11b6306a57830f80..41b61a1cd197ce7c76ae20134a6af4a3e8847230 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -20,8 +20,10 @@ int init(void *arg) int ret; printf("net_health plugin install.\n"); ret = LOAD_SKEL_OBJECT(net_health, perf); - cnt_fd = coobpf_map_find(net_health->obj, "outCnt"); - dist_fd = coobpf_map_find(net_health->obj, "netHist"); + if (ret >= 0) { + cnt_fd = coobpf_map_find(net_health->obj, "outCnt"); + dist_fd = coobpf_map_find(net_health->obj, "netHist"); + } return ret; } @@ -71,7 +73,7 @@ static int cal_dist(unsigned long* values) { static int get_count(unsigned long* values) { int key; static unsigned long rec[2]; - unsigned long now[2]; + unsigned long now[2] = {0, 0}; key = 0; coobpf_key_value(cnt_fd, &key, &now[0]); diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c index d9d9428b10db56ea0f3ebf5928f53f4a2be464b7..6be0e8ab8eb7b3cc81902fbeae962ab824523a4f 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c @@ -26,12 +26,15 @@ struct liphdr { BPF_PERF_OUTPUT(perf, 1024); BPF_STACK_TRACE(stack, MAX_ENTRY); -BPF_HASH(outCnt, int, u64, NET_RETRANS_TYPE_MAX); +BPF_ARRAY(outCnt, u64, NET_RETRANS_TYPE_MAX + 1); static inline void addCnt(int k, u64 val) { + k += 1; u64 *pv = bpf_map_lookup_elem(&outCnt, &k); if (pv) { __sync_fetch_and_add(pv, val); + } else { + bpf_map_update_elem(&outCnt, &k, &val, BPF_ANY); } } diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c index 415a9cfb97bb38114ec6a8934c3c5836d2320129..7a20ceda17c604752860c42359bee78cb7715c9c 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -49,16 +49,26 @@ int init(void *arg) printf("net_retrans plugin install.\n"); ret = LOAD_SKEL_OBJECT(net_retrans, perf); - cnt_fd = coobpf_map_find(net_retrans->obj, "outCnt"); - stack_fd = coobpf_map_find(net_retrans->obj, "stack"); + if (ret >= 0) { + cnt_fd = coobpf_map_find(net_retrans->obj, "outCnt"); + stack_fd = coobpf_map_find(net_retrans->obj, "stack"); + } return ret; } static int get_count(unsigned long *locals) { int i = 0; - - for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { - coobpf_key_value(cnt_fd, &i, &locals[i]); + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(cnt_fd, &key, &key_next) == 0) { + coobpf_key_value(cnt_fd, &key_next, &value); + locals[i ++] = value; + if (i > NET_RETRANS_TYPE_MAX) { + break; + } + key = key_next; } return i; } @@ -69,7 +79,7 @@ static int cal_retrans(unsigned long *values) { unsigned long locals[NET_RETRANS_TYPE_MAX]; get_count(locals); - for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + for (i = NET_RETRANS_TYPE_RTO; i < NET_RETRANS_TYPE_MAX; i ++) { values[i] = locals[i] - rec[i]; rec[i] = locals[i]; } @@ -78,7 +88,7 @@ static int cal_retrans(unsigned long *values) { int call(int t, struct unity_lines *lines) { int i; - unsigned long values[NET_RETRANS_TYPE_MAX]; + unsigned long values[NET_RETRANS_TYPE_MAX] = {0}; struct unity_line* line; budget = t; //release log budget @@ -88,10 +98,9 @@ int call(int t, struct unity_lines *lines) { unity_set_table(line, "net_retrans_count"); cal_retrans(values); - for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { - unity_set_value(line, i, net_title[i], values[i]); + for (i = NET_RETRANS_TYPE_RTO; i < NET_RETRANS_TYPE_MAX; i ++) { + unity_set_value(line, i - NET_RETRANS_TYPE_RTO, net_title[i], values[i]); } - return 0; } @@ -101,9 +110,7 @@ void deinit(void) DESTORY_SKEL_BOJECT(net_retrans); } -#define LOG_MAX 256 -static char log[LOG_MAX]; - +#define LOG_MAX 1024 static int transIP(unsigned long lip, char *result, int size) { inet_ntop(AF_INET, (void *) &lip, result, size); return 0; @@ -169,6 +176,7 @@ static const char * resetActive(int stack_fd, struct data_t *e){ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { char sip[32]; char dip[32]; + char log[LOG_MAX]; transIP(e->ip_src, sip, 32); transIP(e->ip_dst, dip, 32); @@ -182,30 +190,30 @@ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { case NET_RETRANS_TYPE_SYN: case NET_RETRANS_TYPE_SYN_ACK: { - char buf[LOG_MAX - 1]; - snprintf(buf, LOG_MAX - 1, "rcv_nxt:%u, rcv_wup:%u, snd_nxt:%u, snd_una:%u, copied_seq:%u, " + char buf[LOG_MAX/2]; + snprintf(buf, LOG_MAX/2, "rcv_nxt:%u, rcv_wup:%u, snd_nxt:%u, snd_una:%u, copied_seq:%u, " "snd_wnd:%u, rcv_wnd:%u, lost_out:%u, packets_out:%u, retrans_out:%u, " "sacked_out:%u, reordering:%u", e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, e->sacked_out, e->reordering ); - strncat(log, buf, LOG_MAX -1); + strncat(log, buf, LOG_MAX - 1 - strlen(log)); } break; case NET_RETRANS_TYPE_RST: - strncat(log, "noport", LOG_MAX - 1); + strncat(log, "noport", LOG_MAX - 1 - strlen(log)); break; case NET_RETRANS_TYPE_RST_SK: { const char *type = resetSock(stack_fd, e); - strncat(log, type, LOG_MAX - 1); + strncat(log, type, LOG_MAX - 1 - strlen(log)); } break; case NET_RETRANS_TYPE_RST_ACTIVE: { const char *type = resetActive(stack_fd, e); - strncat(log, type, LOG_MAX - 1); + strncat(log, type, LOG_MAX - 1 - strlen(log)); } break; default: diff --git a/source/tools/monitor/unity/collector/plugin/pmu_events/Makefile b/source/tools/monitor/unity/collector/plugin/pmu_events/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..2d093de697640e13f3e316ed69e435c87cec8e83 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/pmu_events/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := pmu_events.o +SO := libpmu_events.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) diff --git a/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.c b/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.c new file mode 100644 index 0000000000000000000000000000000000000000..9525f1807291a07613ba64f7a0052b04dd0ffa4b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.c @@ -0,0 +1,370 @@ +#include "pmu_events.h" + +struct pmu_events { + int nr_cpus; + struct pcpu_hw_info *pcpu_hwi; +}; + +long nr_cpus; +double summary[NR_EVENTS]; +struct pcpu_hw_info *gpcpu_hwi; +struct pmu_events *glb_pme; +char *events_str[] = {"cpu_cycles", "instructions", "ref_cycles", + "llc_load_ref", "llc_load_miss", + "llc_store_ref", "llc_store_miss"}; +char *value_str[] = {"cycles", "instructions", "CPI", + "llc_load_ref", "llc_load_miss", "LLC_LMISS_RATE" + "llc_store_ref", "llc_store_miss", "LLC_SMIRSS_RATE"}; +/*char origpath[]="/mnt/host/sys/fs/cgroup/perf_event/system.slice/"; */ +char *origpath = NULL; /* defalt to host events */ + +static int init_fail; +static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, + int cpu, int group_fd, unsigned long flags) +{ + int ret; + + ret = syscall(__NR_perf_event_open, hw_event, pid, cpu, + group_fd, flags); + return ret; +} + +int create_hw_events(struct pcpu_hw_info *pc_hwi) +{ + int cpu, i, j, group_last, idx_fail; + int ret, pid, group_leader; + struct hw_info *hwi, *leader; + unsigned long flags = 0; + + hwi = pc_hwi->hwi; + struct perf_event_attr attr = { + .freq = 0, + .disabled = 1, + .sample_period = 1000*1000*1000, + }; + + cpu = pc_hwi->cpu; + pid = pc_hwi->pid; + flags = pc_hwi->flags; + leader = NULL; + group_leader = -1; + j = 0; + group_last = groupidx[0]; + for (i = 0; i < NR_EVENTS; i++) { + /* The next PERF types */ + if (groupidx[i] != group_last) { + group_leader = -1; + group_last = groupidx[i]; + } + attr.type = hw_types[i]; + attr.config = hw_configs[i]; +#ifdef DEBUG + if (cpu == 0) + printf("cpu=%d, type=%d, conf=%llu, pid=%d, gld=%d, flags=%d\n", + cpu, attr.type, attr.config, pid, group_leader, flags); +#endif + hwi[i].fd = perf_event_open(&attr, pid, cpu, group_leader, flags); + if (hwi[i].fd <= 0) { + int ret = errno; + if (ret == ENODEV) { + printf("cpu may OFF LINE\n"); + } else { + fprintf(stderr, "WARN:%s cpu%d event %s \n", + strerror(ret), cpu, events_str[i]); + break; + } + } + hwi[i].leader = group_leader; + /* group leader */ + if (group_leader == -1) { + group_leader = hwi[i].fd; + } + } + if (ret) { + idx_fail = i; + goto fail_open; + } + for (i = 0; i < NR_EVENTS; i++) { + int ret = 0; + if (hwi[i].leader == -1) { + ret = ioctl(hwi[i].fd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); + if (ret < 0) { + idx_fail = i; + printf("FAIL:ioctl_RESET %s fd=%d fail\n", events_str[i], hwi[i].fd); + goto fail_ioctl; + } + ret = ioctl(hwi[i].fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); + if (ret < 0) { + idx_fail = i; + printf("FAIL:ioctl_ENABLE %s fd=%d fail\n", events_str[i], hwi[i].fd); + goto fail_ioctl; + } + } + } +fail_ioctl: + for (i = 0; i < idx_fail; i++) { + if ((hwi[i].leader == -1) && (hwi[i].fd > 0)) + ioctl(hwi[i].fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + idx_fail = NR_EVENTS; + } +fail_open: + for (i = 0; i < idx_fail; i++) { + if (hwi[i].fd > 0) + close(hwi[i].fd); + } + return ret; +} + +struct pmu_events *pme_new(int ncpu) +{ + int ret, fd; + struct pmu_events *pmue; + struct pcpu_hw_info *pcpu_hwi; + + pmue = calloc(sizeof(struct pmu_events), 1); + if (!pmue) { + ret = errno; + fprintf(stderr, "%s :malloc pmu_events fail\n", strerror(ret)); + return NULL; + } + pcpu_hwi = calloc(sizeof(struct pcpu_hw_info), nr_cpus); + if (!pcpu_hwi) { + ret = errno; + fprintf(stderr, "%s :alloc pcpu_hw_info fail\n", strerror(ret)); + free(pmue); + return NULL; + } + pmue->nr_cpus = ncpu; + pmue->pcpu_hwi = pcpu_hwi; +} + +int init(void * arg) +{ + int i, ret, flags, cgroup_fd; + struct pmu_events *pmue; + struct pcpu_hw_info *pcpu_hwi; + + nr_cpus = sysconf(_SC_NPROCESSORS_CONF); + if (nr_cpus <= 0) { + ret = errno; + printf("WARN: pmu_events install FAIL sysconf\n"); + init_fail = ret; + return 0; + } + + pmue = pme_new(nr_cpus); + if (pmue && pmue->pcpu_hwi) { + pcpu_hwi = pmue->pcpu_hwi; + glb_pme = pmue; + } else { + init_fail = -1; + return 0; + } +#if 0 + pmue = (struct pmu_events *)arg; + cgroup_fd = pmue->cgroup_fd; +#endif + if (origpath) { + cgroup_fd = open(origpath, O_RDONLY); + if (cgroup_fd < 0) { + printf(" open %s fail\n", origpath); + init_fail = cgroup_fd; + return 0; + } + flags = PERF_FLAG_PID_CGROUP; + } else { + cgroup_fd = -1; + flags = 0; + } + for (i = 0; i < nr_cpus; i++) { + pcpu_hwi[i].cpu = i; + pcpu_hwi[i].pid = cgroup_fd; + pcpu_hwi[i].flags = flags; + ret = create_hw_events(&pcpu_hwi[i]); + if (ret) { + init_fail = ret; + return 0; + } + } + printf("pmu_events plugin install.\n"); + init_fail = 0; + return 0; +} + +void collect(struct pcpu_hw_info *phw, double *sum) +{ + int i, n; + double *delta = phw->values; + struct hw_info *hw; + + hw = phw->hwi; + for (i = 0; i < NR_EVENTS; i++) { + if (hw[i].fd < 0) + continue; + hw[i].prv_cnt = hw[i].count; + n = read(hw[i].fd, &hw[i].count, sizeof(hw[i].count)); + if (n < 0) + continue; + delta[i] = hw[i].count - hw[i].prv_cnt; + sum[i] += delta[i]; +#ifdef DEBUG + if (phw->cpu == 0) + printf("cpu%d %s prev=%llu, now=%llu\n", + phw->cpu, events_str[i], + hw[i].prv_cnt, hw[i].count); +#endif + } +#ifdef DEBUG + printf("cpu%d cpi=%f\n", phw->cpu, (float)delta[0]/(float)delta[1]); +#endif +} + +int fill_line_percpu(struct unity_line *line, double *summ, char *index) +{ + int i; + + unity_set_index(line, 0, "mode", index); + for (i = 0; i < NR_EVENTS; i++) + unity_set_value(line, i, events_str[i], summ[i]); + unity_set_value(line, i++, "cpi", summ[CYCLES]/summ[INSTRUCTIONS]); + unity_set_value(line, i++, "ipc", summ[INSTRUCTIONS]/summ[CYCLES]); + unity_set_value(line, i++, "llc_cache_mpi", + (summ[LLC_LOAD_MISS]+summ[LLC_STORE_MISS])/summ[INSTRUCTIONS]); + unity_set_value(line, i++, "llc_rmiss_rate", + summ[LLC_LOAD_MISS]/summ[LLC_LOAD_REF]); + unity_set_value(line, i++, "llc_wmiss_rate", + summ[LLC_STORE_MISS]/summ[LLC_STORE_REF]); + unity_set_value(line, i++, "llc_miss_rate", + (summ[LLC_LOAD_MISS]+summ[LLC_STORE_MISS])/ + (summ[LLC_LOAD_REF]+summ[LLC_STORE_REF])); +} + +int fill_line(struct unity_line *line, double *summ, char *index) +{ + int i; + + unity_set_index(line, 0, "mode", index); + for (i = 0; i < NR_EVENTS; i++) + unity_set_value(line, i, events_str[i], summ[i]); + + unity_set_value(line, i++, "cpi", + summ[INSTRUCTIONS]==0?0:summ[CYCLES]/summ[INSTRUCTIONS]); + unity_set_value(line, i++, "ipc", + summ[CYCLES]==0?0:summ[INSTRUCTIONS]/summ[CYCLES]); + unity_set_value(line, i++, "llc_cache_mpi", + summ[INSTRUCTIONS]==0?0: + (summ[LLC_LOAD_MISS]+summ[LLC_STORE_MISS])/summ[INSTRUCTIONS]); + unity_set_value(line, i++, "llc_rmiss_rate", + summ[LLC_LOAD_REF]==0?0:summ[LLC_LOAD_MISS]/summ[LLC_LOAD_REF]); + unity_set_value(line, i++, "llc_wmiss_rate", + summ[LLC_STORE_REF]==0?0:summ[LLC_STORE_MISS]/summ[LLC_STORE_REF]); + unity_set_value(line, i++, "llc_miss_rate", + (summ[LLC_LOAD_REF]+summ[LLC_STORE_REF])==0?0: + (summ[LLC_LOAD_MISS]+summ[LLC_STORE_MISS])/ + (summ[LLC_LOAD_REF]+summ[LLC_STORE_REF])); +} + +int call(int t, struct unity_lines* lines) +{ + int i; + char index[16] = {0}; + struct unity_line* line; + double summ[NR_EVENTS]; + struct pcpu_hw_info *pcp_hw; + + if (init_fail) { + return init_fail; + } + pcp_hw = glb_pme->pcpu_hwi; + for (i = 0; i < nr_cpus; i++) { + collect(&pcp_hw[i], summ); + } + unity_alloc_lines(lines, 1+nr_cpus); + line = unity_get_line(lines, 0); + unity_set_table(line, "pmu_events"); + fill_line(line, summ, "node"); + + for (i = 0; i < nr_cpus; i++) { + line = unity_get_line(lines, 1+i); + unity_set_table(line, "pmu_events_percpu"); + snprintf(index, sizeof(index), "percpu%d", i); + fill_line(line, pcp_hw[i].values, index); + } + return 0; +} + +void remove_events(struct pcpu_hw_info *pcpu_hwi) +{ + + int i; + struct hw_info *hw; + + hw = pcpu_hwi->hwi; + for (i = 0; i < NR_EVENTS; i++) { + if ((hw[i].leader == -1) && (hw[i].fd > 0)) + ioctl(hw[i].fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP); + if (hw[i].fd > 0) { + close(hw[i].fd); + } + } +} + +void deinit(void) +{ + int i; + struct pcpu_hw_info *pcp_hw; + + if (glb_pme) { + pcp_hw = glb_pme->pcpu_hwi; + if (pcp_hw) { + for (i = 0; i < nr_cpus; i++) { + remove_events(&pcp_hw[i]); + } + free(pcp_hw); + } + free(glb_pme); + } + printf("pmu_events plugin uninstall\n"); +} + +#ifdef DEBUG +/*#if 1 */ + +/* for dev/selftest */ +int call_debug(void) +{ + int i; + struct pcpu_hw_info *pcp_hw; + + pcp_hw = glb_pme->pcpu_hwi; + for (i = 0; i < nr_cpus; i++) { + collect(&pcp_hw[i], summary); + } +} +int main(int argc, char *argv[]) +{ + int i = 4; + + init(NULL); + call_debug(); + while(i > 0) { + sleep(1); + memset(summary, 0, sizeof(summary)); + call_debug(); + if (summary[INSTRUCTIONS]) + printf("CPI=%f\n", summary[CYCLES]/summary[INSTRUCTIONS]); + if (summary[INSTRUCTIONS]) + printf("RCPI=%f\n", summary[REF_CYCLES]/summary[INSTRUCTIONS]); + if (summary[LLC_LOAD_REF]) + printf("LLC_LOAD_MISS RATE =%f\n", summary[LLC_LOAD_MISS]/summary[LLC_LOAD_REF]); + if (summary[LLC_STORE_REF]) + printf("LLC_STORE_MISS RATE =%f\n", summary[LLC_STORE_MISS]/summary[LLC_STORE_REF]); + if ((summary[LLC_LOAD_REF]+summary[LLC_STORE_REF]) != 0) + printf("LLC_MISS RATE=%f\n", + (summary[LLC_LOAD_MISS]+summary[LLC_STORE_MISS])/ + (summary[LLC_LOAD_REF]+summary[LLC_STORE_REF])); + i--; + } + deinit(); +} +#endif diff --git a/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.h b/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.h new file mode 100644 index 0000000000000000000000000000000000000000..3406c5450d6473df9fc4d0ebbdffdf3fafb8edda --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/pmu_events/pmu_events.h @@ -0,0 +1,109 @@ +#ifndef UNITY_PMU_EVENT_H +#define UNITY_PMU_EVENT_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../plugin_head.h" +#define NR_GROUP 3 +#define NR_EVENTS 7 +#define NR_CELL 2 + + +#ifdef DEBUG +/* for test */ +__u32 hw_types[] = { + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, + PERF_TYPE_SOFTWARE, +}; + +__u64 hw_configs[] = { + PERF_COUNT_SW_CPU_CLOCK, + PERF_COUNT_SW_CONTEXT_SWITCHES, + PERF_COUNT_SW_TASK_CLOCK, + PERF_COUNT_SW_PAGE_FAULTS, + PERF_COUNT_SW_PAGE_FAULTS_MIN, + PERF_COUNT_SW_CPU_MIGRATIONS, + PERF_COUNT_SW_PAGE_FAULTS_MAJ, +}; +#endif + +__u32 hw_types[] = { + PERF_TYPE_HARDWARE, + PERF_TYPE_HARDWARE, + PERF_TYPE_HARDWARE, + PERF_TYPE_HW_CACHE, + PERF_TYPE_HW_CACHE, + PERF_TYPE_HW_CACHE, + PERF_TYPE_HW_CACHE, +}; + +__u64 hw_configs[] = { + PERF_COUNT_HW_CPU_CYCLES, + PERF_COUNT_HW_INSTRUCTIONS, + PERF_COUNT_HW_REF_CPU_CYCLES, + PERF_COUNT_HW_CACHE_LL << 0 | + (PERF_COUNT_HW_CACHE_OP_READ << 8) | + (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), + PERF_COUNT_HW_CACHE_LL << 0 | + (PERF_COUNT_HW_CACHE_OP_READ << 8) | + (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), + PERF_COUNT_HW_CACHE_LL << 0 | + (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | + (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), + PERF_COUNT_HW_CACHE_LL << 0 | + (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | + (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), +}; + +int groupidx[NR_EVENTS] = { + 0,0,0, + 1,1, + 2,2 +}; + +enum { + CYCLES, + INSTRUCTIONS, + REF_CYCLES, + LLC_LOAD_REF, + LLC_LOAD_MISS, + LLC_STORE_REF, + LLC_STORE_MISS, +}; + +struct hw_info { + int fd, leader; + __u32 type; + __u64 config; + __u64 count, prv_cnt; +}; + +struct pcpu_hw_info { + pid_t pid; + int cpu; + unsigned long flags; + double values[NR_EVENTS]; + struct hw_info hwi[NR_EVENTS]; +}; + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + +#endif //UNITY_PMU_EVENT_H diff --git a/source/tools/monitor/unity/collector/plugin/proto_sender.c b/source/tools/monitor/unity/collector/plugin/proto_sender.c index 552e6068910b5ed9e65b005311bba83898a44a67..36a5dd450858f366c27a4d925d0d8bec0e58e605 100644 --- a/source/tools/monitor/unity/collector/plugin/proto_sender.c +++ b/source/tools/monitor/unity/collector/plugin/proto_sender.c @@ -5,6 +5,8 @@ #include "proto_sender.h" #define _GNU_SOURCE #include +#include +#include #include #define PROTO_QUEUE_SIZE 64 @@ -165,9 +167,11 @@ int proto_send_proc(void* msg, struct beeQ* q) { return ret; } +// 这是接收外部非lua 推送队列的操作 static int proto_recv_setup(struct beeQ* q) { lua_State *L; struct beeQ* pushQ = (struct beeQ*)(q->qarg); + prctl(PR_SET_NAME, (unsigned long)"proto_recv"); L = proto_sender_lua(pushQ); if (L == NULL) { diff --git a/source/tools/monitor/unity/collector/plugin/sum_retrans/Makefile b/source/tools/monitor/unity/collector/plugin/sum_retrans/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d0c3391a855a9e62347a513e73d253a4c8cced4f --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/sum_retrans/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := sum_retrans.bpf.c +csrcs := sum_retrans.c +so := libsum_retrans.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..16ac99107678b8a6ad0b831e110093c6f25c2cbf --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.bpf.c @@ -0,0 +1,47 @@ +// +// Created by 廖肇燕 on 2023/4/11. +// +#define BPF_NO_GLOBAL_DATA +#include +#include + +BPF_HASH(dips, u32, u64, 1024); +BPF_HASH(inums, u32, u64, 1024); + +static inline u32 read_ns_inum(struct sock *sk) +{ + if (sk) { + return BPF_CORE_READ(sk, __sk_common.skc_net.net, ns.inum); + } + return 0; +} + +static void inc_value(struct bpf_map* maps, u32 k) { + u64 *pv = bpf_map_lookup_elem(maps, &k); + if (pv) { + __sync_fetch_and_add(pv, 1); + } else { + u64 v = 1; + bpf_map_update_elem(maps, &k, &v, BPF_ANY); + } +} + +struct tcp_retrans_args { + u64 pad; + u64 skb; + u64 sk; + u16 sport; + u16 dport; + u32 sip; + u32 dip; +}; +SEC("tracepoint/tcp/tcp_retransmit_skb") +int tcp_retransmit_skb_hook(struct tcp_retrans_args *args){ + struct sock *sk = (struct sock *)args->sk; + u32 inum = read_ns_inum(sk); + u32 ip = args->dip; + + inc_value((struct bpf_map *)&inums, inum); + inc_value((struct bpf_map *)&dips, ip); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.c b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.c new file mode 100644 index 0000000000000000000000000000000000000000..d411e59d53071b0f37d3f50dc9da73cb6a961fac --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.c @@ -0,0 +1,113 @@ +// +// Created by 廖肇燕 on 2023/4/11. +// + +#include "sum_retrans.h" +#include "../bpf_head.h" +#include "sum_retrans.skel.h" + +#include +#include + +#include +#include +#include + + +DEFINE_SEKL_OBJECT(sum_retrans); +static int inum_fd = 0; +static int dip_fd = 0; +int init(void *arg) +{ + int ret; + printf("sum_retrans plugin install.\n"); + ret = LOAD_SKEL_OBJECT(sum_retrans, perf); + if (ret >= 0) { + inum_fd = coobpf_map_find(sum_retrans->obj, "inums"); + dip_fd = coobpf_map_find(sum_retrans->obj, "dips"); + } + return ret; +} + +#define BUFF_SIZE 64 +#define LOG_MAX 4096 +static char log[LOG_MAX]; + +static int transIP(unsigned long lip, char *result, int size) { + inet_ntop(AF_INET, (void *) &lip, result, size); + return 0; +} + +static void pack_dip() { + char ips[32]; + char buff[BUFF_SIZE]; + unsigned long value; + unsigned int ip, ip_next; + + log[0] = '\0'; + ip = 0; + while (coobpf_key_next(dip_fd, &ip, &ip_next) == 0) { + bpf_map_lookup_elem(dip_fd, &ip_next, &value); + ip = ip_next; + + transIP(ip, ips, 32); + snprintf(buff, BUFF_SIZE, "%s:%ld,", ips, value); + strncat(log, buff, LOG_MAX - 1 - strlen(log)); + ip = ip_next; + } + + ip = 0; + while (coobpf_key_next(dip_fd, &ip, &ip_next) == 0) { + bpf_map_delete_elem(dip_fd, &ip_next); + ip = ip_next; + } +} + +static void pack_inum() { + char buff[BUFF_SIZE]; + unsigned long value; + unsigned int inum, inum_next; + + log[0] = '\0'; + + inum = 0; + while (coobpf_key_next(inum_fd, &inum, &inum_next) == 0) { + bpf_map_lookup_elem(inum_fd, &inum_next, &value); + snprintf(buff, BUFF_SIZE, "%u:%ld,", inum_next, value); + strncat(log, buff, LOG_MAX - 1 - strlen(log)); + inum = inum_next; + } + + inum = 0; + while (coobpf_key_next(inum_fd, &inum, &inum_next) == 0) { + bpf_map_delete_elem(inum_fd, &inum_next); + inum = inum_next; + } +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line* line; + + pack_dip(); + if (strlen(log) > 0) { + unity_alloc_lines(lines, 2); // 预分配好 + + line = unity_get_line(lines, 0); + unity_set_table(line, "retrans_dip"); + unity_set_log(line, "ip_log", log); + + pack_inum(); + line = unity_get_line(lines, 1); + unity_set_table(line, "retrans_inum"); + unity_set_log(line, "inum_log", log); + } + + return 0; +} + +void deinit(void) +{ + printf("sum_retrans plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(sum_retrans); +} diff --git a/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.h b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.h new file mode 100644 index 0000000000000000000000000000000000000000..4faecd9a3ec06c061b25523644c3dba9963a836c --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/sum_retrans/sum_retrans.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/4/11. +// + +#ifndef UNITY_SUM_RETRANS_H +#define UNITY_SUM_RETRANS_H + +#endif //UNITY_SUM_RETRANS_H diff --git a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c index 169e5062fb04c8a5c371ed77a42b60cc7f86e5c0..b276a925e73a88c13cc00cf4acae15721bd27577 100644 --- a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c +++ b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c @@ -30,6 +30,7 @@ static int sample_thread_func(struct beeQ* q, void * arg) { unity_set_value(line, 1, "value2", 2.0 + value); unity_set_log(line, "log", "hello world."); beeQ_send(q, lines); + value += 1.1; ret = sleep(5); if (ret > 0) { // interrupt by signal break; diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c index 850111668bbfb4e91dd34c2831163e259989c2eb..323d2097e410b9b455cacd242a2991d6ff284466 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c @@ -138,8 +138,11 @@ int BPF_KPROBE(account_process_tick, struct task_struct *p, int user_tick) bpf_get_current_comm(&event.comm, sizeof(event.comm)); event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); latp->last_perf_event = now; - bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + if (latp->recorded == 0) { + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(event)); + latp->recorded = 1; + } } } } else { diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h index f7280f4bf2088230af297d819002d13cb238924b..4f411ad91fbdbf8393fc71cecbe26afae30e7e6a 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h @@ -14,7 +14,7 @@ struct args { struct latinfo { unsigned long long last_seen_need_resched_ns; unsigned long long last_perf_event; - int ticks_without_resched; + int recorded, ticks_without_resched; }; #ifndef __VMLINUX_H__ diff --git a/source/tools/monitor/unity/collector/plugin/virtiostat/Makefile b/source/tools/monitor/unity/collector/plugin/virtiostat/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..51b1e9daa818094ea2bf67754226ea5a58400978 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtiostat/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := virtiostat.bpf.c +csrcs := virtiostat.c +so := libvirtiostat.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.bpf.c b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..de54bb42dedf44bf736badb97037f77cbdc08745 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.bpf.c @@ -0,0 +1,105 @@ +// +// Created by 廖肇燕 on 2023/5/7. +// + +#define BPF_NO_GLOBAL_DATA + +#include +#include +#include "virtiostat.h" + +BPF_HASH(stats, u64, virtio_stat_t, 256); + +struct virtio_device_id { + u32 device; + u32 vendor; +}; + +struct virtio_device { + int index; + bool failed; + bool config_enabled; + bool config_change_pending; + spinlock_t config_lock; + struct device dev; + struct virtio_device_id id; + void *config; + void *vringh_config; + struct list_head vqs; + u64 features; + void *priv; +}; + +struct virtqueue { + struct list_head list; + void (*callback)(struct virtqueue *vq); + const char *name; + struct virtio_device *vdev; + unsigned int index; + unsigned int num_free; + void *priv; +}; + +static inline void add_value(virtio_stat_t *vs, struct scatterlist **sgs, + u32 out_sgs, u32 in_sgs) { + vs->out_sgs += out_sgs; + vs->in_sgs += in_sgs; +} + +static inline void record(struct virtqueue *vq, struct scatterlist **sgs, + unsigned int out_sgs, unsigned int in_sgs) +{ + virtio_stat_t newvs = {0}; + virtio_stat_t *vs; + u64 key = (u64)vq; + + vs = bpf_map_lookup_elem(&stats, &key); + if (!vs) { + bpf_probe_read_kernel_str(newvs.driver, sizeof(newvs.driver), BPF_CORE_READ(vq, vdev, dev.driver, name)); + bpf_probe_read_kernel_str(newvs.dev, sizeof(newvs.dev), BPF_CORE_READ(vq, vdev, dev.kobj.name)); + bpf_probe_read_kernel_str(newvs.vqname, sizeof(newvs.vqname), BPF_CORE_READ(vq, name)); + + add_value(&newvs, sgs, out_sgs, in_sgs); + bpf_map_update_elem(&stats, &key, &newvs, BPF_ANY); + } else { + add_value(vs, sgs, out_sgs, in_sgs); + } +} + +SEC("kprobe/virtqueue_add_sgs") +int trace_virtqueue_add_sgs(struct pt_regs *ctx) +{ + struct virtqueue *vq = (struct virtqueue *)PT_REGS_PARM1(ctx); + struct scatterlist **sgs = (struct scatterlist **)PT_REGS_PARM2(ctx); + u32 out_sgs = PT_REGS_PARM3(ctx); + u32 in_sgs = PT_REGS_PARM4(ctx); + record(vq, sgs, out_sgs, in_sgs); + return 0; +} + +SEC("kprobe/virtqueue_add_outbuf") +int trace_virtqueue_add_outbuf(struct pt_regs *ctx) +{ + struct virtqueue *vq = (struct virtqueue *)PT_REGS_PARM1(ctx); + struct scatterlist **sgs = (struct scatterlist **)PT_REGS_PARM2(ctx); + record(vq, sgs, 1, 0); + return 0; +} + +SEC("kprobe/virtqueue_add_inbuf") +int trace_virtqueue_add_inbuf(struct pt_regs *ctx) +{ + struct virtqueue *vq = (struct virtqueue *)PT_REGS_PARM1(ctx); + struct scatterlist **sgs = (struct scatterlist **)PT_REGS_PARM2(ctx); + record(vq, sgs, 0, 1); + return 0; +} + +SEC("kprobe/virtqueue_add_inbuf_ctx") +int trace_virtqueue_add_inbuf_ctx(struct pt_regs *ctx) +{ + struct virtqueue *vq = (struct virtqueue *)PT_REGS_PARM1(ctx); + struct scatterlist **sgs = (struct scatterlist **)PT_REGS_PARM2(ctx); + record(vq, sgs, 0, 1); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.c b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.c new file mode 100644 index 0000000000000000000000000000000000000000..cbe5c03d41d0c46a68f95a79219052d2a7b53cf1 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.c @@ -0,0 +1,54 @@ +// +// Created by 廖肇燕 on 2023/5/7. +// + +#include "virtiostat.h" +#include "../bpf_head.h" +#include "virtiostat.skel.h" + +DEFINE_SEKL_OBJECT(virtiostat); +static int stats_fd = 0; + +int init(void *arg) +{ + int ret; + printf("virtiostat plugin install.\n"); + ret = LOAD_SKEL_OBJECT(virtiostat, perf); + if (ret >= 0) { + stats_fd = coobpf_map_find(virtiostat->obj, "stats"); + } + return ret; +} + +#define BUF_MAX 128 +#define LOG_MAX 4096 +void walk_virtio(struct unity_lines *lines) { + struct unity_line *line; + unsigned long key, next; + struct virtio_stat stat; + char log[LOG_MAX] = {'\0'}; + + key = 0; + while (coobpf_key_next(stats_fd, &key, &next) == 0) { + char buf[BUF_MAX]; + bpf_map_lookup_elem(stats_fd, &next, &stat); + key = next; + snprintf(buf, BUF_MAX, "driver:%s,dev:%s,name:%s,in:%d,out:%d;", stat.driver, stat.dev, stat.vqname, stat.in_sgs, stat.out_sgs); + strncat(log, buf, LOG_MAX - 1 - strlen(log)); + } + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "virtios"); + unity_set_log(line, "log", log); +} + +int call(int t, struct unity_lines *lines) { + walk_virtio(lines); + return 0; +} + +void deinit(void) +{ + printf("virtiostat plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(virtiostat); +} diff --git a/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.h b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.h new file mode 100644 index 0000000000000000000000000000000000000000..2e037b85b9ddf57d702e4d28379ab5814dc2e83d --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtiostat/virtiostat.h @@ -0,0 +1,18 @@ +// +// Created by 廖肇燕 on 2023/5/7. +// + +#ifndef UNITY_VIRTIOSTAT_H +#define UNITY_VIRTIOSTAT_H + +#define CMPMAX 16 + +typedef struct virtio_stat { + char driver[CMPMAX]; + char dev[12]; + char vqname[12]; + unsigned int in_sgs; + unsigned int out_sgs; +} virtio_stat_t; + +#endif //UNITY_VIRTIOSTAT_H diff --git a/source/tools/monitor/unity/collector/plugin/virtout/Makefile b/source/tools/monitor/unity/collector/plugin/virtout/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f0e8a47f3f89b8f918ce8aaf7422838616af5cc6 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := virtout.bpf.c +csrcs := virtout.c +so := libvirtout.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c b/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..69ca70ad1e3939c92715e6cc62e1def21bfbd5aa --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.bpf.c @@ -0,0 +1,68 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// +#define BPF_NO_GLOBAL_DATA + +#include +#include +#include "virtout.h" + +#define MAX_ENTRY 128 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +#define PERIOD_TIME (10 * 1000 * 1000ULL) +#define THRESHOLD_TIME (200 * 1000 * 1000ULL) + +BPF_ARRAY(virtdist, u64, 20); +BPF_PERF_OUTPUT(perf, 1024); +BPF_STACK_TRACE(stack, MAX_ENTRY); +BPF_PERCPU_ARRAY(perRec, u64, 2); + +static inline u64 get_last(int index) { + u64 *pv = bpf_map_lookup_elem(&perRec, &index); + if (pv) { + return *pv; + } + return 0; +} + +static inline void save_last(int index, u64 t) { + bpf_map_update_elem(&perRec, &index, &t, BPF_ANY); +} + +static inline void check_time(struct bpf_perf_event_data *ctx, + int index, + struct bpf_map* event) { + u64 t = ns(); + u64 last = get_last(index); + s64 delta; + + save_last(index, t); + delta = t - last; + + if (last && delta > 2 * PERIOD_TIME) { + delta -= PERIOD_TIME; + int res = delta / PERIOD_TIME; + hist10_push((struct bpf_map_def *)&virtdist, res * 10); + if (delta >= THRESHOLD_TIME) { + struct task_struct* task = (struct task_struct *)bpf_get_current_task(); + struct data_t data = {}; + + data.pid = pid(); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + data.delta = delta; + data.us = t / 1000; + data.cpu = cpu(); + bpf_get_current_comm(&data.comm, TASK_COMM_LEN); + bpf_perf_event_output(ctx, event, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + } +} + +SEC("perf_event") +int sw_clock(struct bpf_perf_event_data *ctx) +{ + check_time(ctx, 1, (struct bpf_map *)&perf); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.c b/source/tools/monitor/unity/collector/plugin/virtout/virtout.c new file mode 100644 index 0000000000000000000000000000000000000000..6034db5a305e83e59ab06c6b3058e5d65e7ef57e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.c @@ -0,0 +1,228 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#include +#define COOLBPF_PERF_THREAD +#include "../bpf_head.h" +#include "virtout.h" +#include "virtout.skel.h" + +#include +#include +#include +#include +#include + + +#define CPU_DIST_INDEX 4 +#define DIST_ARRAY_SIZE 20 + +static volatile int budget = 0; // for log budget +static int dist_fd = 0; +static int stack_fd = 0; +static int perf_fds[1024]; + +int proc(int stack_fd, struct data_t *e, struct unity_line *line); +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + int ret; + if (budget > 0) { + struct data_t *e = (struct data_t *)data; + struct beeQ *q = (struct beeQ *)ctx; + struct unity_line *line; + struct unity_lines *lines = unity_new_lines(); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + ret = proc(stack_fd, e, line); + if (ret >= 0) { + beeQ_send(q, lines); + } + budget --; + } +} + +static void close_perf_fds(void) { + int i; + for (i = 0; i < 1024; i ++) { + if (perf_fds[i] > 0) { + close(perf_fds[i]); + perf_fds[i] = 0; + } + } +} + +static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, + int cpu, int group_fd, int flags) +{ + int ret; + + ret = syscall(__NR_perf_event_open, hw_event, pid, cpu, + group_fd, flags); + return ret; +} + +DEFINE_SEKL_OBJECT(virtout); +static struct bpf_prog_skeleton *search_progs(const char* func) { + struct bpf_object_skeleton *s; + int i; + + s = virtout->skeleton; + for (i = 0; i < s->prog_cnt; i ++) { + if (strcmp(s->progs[i].name, func) == 0) { + return &(s->progs[i]); + } + } + return NULL; +} + +static int setup_perf_events(const char* func) { + int nr_cpus = libbpf_num_possible_cpus(); + int i; + struct bpf_link *link; + struct bpf_prog_skeleton *progs; + + progs = search_progs(func); + + struct perf_event_attr perf_attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CPU_CLOCK, + .freq = 0, + .sample_period = 10 * 1000 * 1000, + }; + + for (i = 0; i < nr_cpus; i ++) { + perf_fds[i] = perf_event_open(&perf_attr, -1, i, -1, 0); + if (perf_fds[i] < 0) { + if (errno == ENODEV) { + printf("skip offline cpu id: %d\n", i); + continue; + } else { + perror("syscall failed."); + close_perf_fds(); + return -1; + } + } + + link = bpf_program__attach_perf_event(*(progs->prog), perf_fds[i]); + if (!link) { + perror("attach failed."); + close_perf_fds(); + return -1; + } + *(progs->link) = link; + } + printf("setup virout OK.\n"); + return 0; +} + +int init(void *arg) { + int ret; + printf("virtout plugin install.\n"); + + ret = LOAD_SKEL_OBJECT(virtout, perf); + if (ret < 0) { + return ret; + } + ret = setup_perf_events("sw_clock"); + if (ret < 0) { + return ret; + } + dist_fd = coobpf_map_find(virtout->obj, "virtdist"); + stack_fd = coobpf_map_find(virtout->obj, "stack"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[key] = value; + if (key > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE] = {0}; + + size = get_dist(locals); + for (i = 0; i < CPU_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +int call(int t, struct unity_lines *lines) { + int i; + unsigned long values[CPU_DIST_INDEX]; + const char *names[] = { "ms100", "s1", "s10", "so"}; + struct unity_line* line; + + budget = t; + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "virtout_dist"); + + cal_dist(values); + for (i = 0; i < CPU_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + return 0; +} + + +void deinit(void) +{ + printf("virout plugin uninstall.\n"); + close_perf_fds(); + DESTORY_SKEL_BOJECT(virtout); +} + +#define LOG_MAX 4096 +int proc(int stack_fd, struct data_t *e, struct unity_line *line) { + int i; + char log[LOG_MAX] = {'\0'}; + unsigned long addr[128]; + int id = e->stack_id; //last stack + struct ksym_cell* cell; + + snprintf(log, LOG_MAX, "task:%d(%s);us:%ld;cpu:%d;delayed:%ld;callstack:", e->pid, e->comm, e->us, e->cpu, e->delta); + coobpf_key_value(stack_fd, &id, &addr); + + for (i = 0; i < 128; i ++) { + if (addr[i] > 0) { + cell = ksym_search(addr[i]); + if (cell != NULL) { + strncat(log, cell->func, LOG_MAX - 1 - strlen(log)); + } else { + strncat(log, "!nil", LOG_MAX - - 1 - strlen(log)); + } + strncat(log, ",", LOG_MAX - 1 - strlen(log)); + } else { + break; + } + } + unity_set_table(line, "virtout_log"); + unity_set_log(line, "log", log); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/virtout/virtout.h b/source/tools/monitor/unity/collector/plugin/virtout/virtout.h new file mode 100644 index 0000000000000000000000000000000000000000..42822d9694975db3566b81250da18d07d3e036f9 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/virtout/virtout.h @@ -0,0 +1,19 @@ +// +// Created by 廖肇燕 on 2023/4/10. +// + +#ifndef UNITY_VIRTOUT_H +#define UNITY_VIRTOUT_H + +#define TASK_COMM_LEN 16 + +struct data_t { + int pid; + int cpu; + unsigned long us; + unsigned int stack_id; + unsigned long delta; + char comm[TASK_COMM_LEN]; +}; + +#endif //UNITY_VIRTOUT_H diff --git a/source/tools/monitor/unity/collector/pluginManager.lua b/source/tools/monitor/unity/collector/pluginManager.lua new file mode 100644 index 0000000000000000000000000000000000000000..501fea88b0d4bde067a4ec0bb86b55ee8ee94f20 --- /dev/null +++ b/source/tools/monitor/unity/collector/pluginManager.lua @@ -0,0 +1,69 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/21 8:30 AM +--- + +require("common.class") +local CpluginManager = class("pluginManager") +local CguardSched = require("collector.guard.guardSched") +local Cplugin = require("collector.plugin") +local system = require("common.system") + +function CpluginManager:_init_(procffi, proto_q, resYaml, tid, jperiod) + local res = resYaml + self:setProcSys(procffi, res.config) + + self._sig_cffi = procffi["cffi"] + self._sig_cffi.ffi_plugin_init() + + self._plugins = {} + self._names = {} + self:setup(res, proto_q) + self._guardSched = CguardSched.new(tid, self._plugins, self._names, jperiod) + + self._resYaml = resYaml -- for add function + self._proto_q = proto_q +end + +function CpluginManager:_del_() + self._sig_cffi.plugin_stop() + for _, plugin in ipairs(self._plugins) do + plugin = nil + end +end + +function CpluginManager:setup(resYaml, proto_q) + local pluginFFI = require("collector.native.plugincffi") + local plugins = resYaml.plugins + + for _, plugin in ipairs(plugins) do + local so = plugin.so + if so then + table.insert(self._plugins, Cplugin.new(resYaml, pluginFFI, proto_q, so)) + table.insert(self._names, so) + end + end +end + +function CpluginManager:add(name, loops) + local pluginFFI = require("collector.native.plugincffi") + if not system:valueIsIn(self._names, name) then + table.insert(self._plugins, Cplugin.new(self._resYaml, pluginFFI, self._proto_q, name, loops)) + table.insert(self._names, name) + end +end + +function CpluginManager:setProcSys(procFFI, config) + local proc = config["proc_path"] or "/" + local sys = config["sys_path"] or "/" + + procFFI.cffi.ffi_set_unity_proc(procFFI.ffi.string(proc)) + procFFI.cffi.ffi_set_unity_sys(procFFI.ffi.string(sys)) +end + +function CpluginManager:proc(t, lines) + self._guardSched:proc(t, lines) +end + +return CpluginManager diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua index 23a2f4defc854729dc982aa8e03a8800b506de3e..f40439683f1cce9a880884b073fdf16db921ab0b 100644 --- a/source/tools/monitor/unity/collector/pod_allocpage.lua +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -10,22 +10,30 @@ local unistd = require("posix.unistd") local dirent = require("posix.dirent") local stdlib = require("posix.stdlib") local stat = require("posix.sys.stat") -local cjson = require("cjson") -local json = cjson.new() local CkvProc = require("collector.kvProc") local CvProc = require("collector.vproc") local pystring = require("common.pystring") local dockerinfo = require("common.dockerinfo") - +local json = require("cjson") local CPodAlloc = class("podalloc", CkvProc) +local ChttpCli = require("httplib.httpCli") function CPodAlloc:_init_(proto, pffi, mnt, pFile) CkvProc._init_(self, proto, pffi, mnt, pFile , "pod_alloc") self._ffi = require("collector.native.plugincffi") - self.proc_fs, self.sys_fs, self.pods_fs, self.root_fs = dockerinfo:get_hostfs() + self.root_fs = mnt + self.proc_fs = mnt .. "/proc/" self.name_space = {} self.pod_mem = {} + self.inode_link = {} + self.link_pid = {} + self.cgroup_pod = {} + self.allpods = {} + self.blacklist = {["arms-prom"] = 1, ["kube-system"] = 1, ["kube-public"] = 1, ["kube-node-lease"] = 1} + -- self.blacklist= {} self.total = 0 + self.count = 0 + self.cgroup_count = 0 end function CPodAlloc:file_exists(file) @@ -37,23 +45,29 @@ function CPodAlloc:file_exists(file) end end -function CPodAlloc:switch_ns(pid) - local pid_ns = self.proc_fs .. pid .. "/ns/net" - if not self:file_exists(pid_ns) then return end +function CPodAlloc:switch_ns(file) + if file == nil then file = self.proc_fs .. "1/ns/net" end + if not self:file_exists(file) then return end - local f = fcntl.open(pid_ns,fcntl.O_RDONLY) - self._ffi.C.setns(f,0) + local f = fcntl.open(file,fcntl.O_RDONLY) + if f == nil then return false end + local res = self._ffi.C.setns(f,0) + if res == -1 then return false end unistd.close(f) + return true end function CPodAlloc:get_container_info(did) - local res = "unknow" + local restable = {} local podname = did local podns = did local cname = did - res = dockerinfo:get_inspect(did) + res = dockerinfo:get_inspect(did, self.root_fs) + res = pystring:strip(res) + if #res == 0 then return podname,podns end local restable = json.decode(res) + if not restable then return podname end if #restable > 0 then restable = restable[1] end @@ -81,58 +95,53 @@ function CPodAlloc:get_container_info(did) podns = restable['status']['labels']['io.kubernetes.pod.namespace'] end if pystring:startswith(podname,"/") then podname=string.sub(podname,2,-1) end - if not self.pod_mem[podname] then - self.pod_mem[podname] = {} - self.pod_mem[podname]["allocpage"] = 0 - self.pod_mem[podname]["podns"] = podns - self.pod_mem[podname]["podname"] = podname - end - return podname + return podname,podns end function CPodAlloc:get_pidalloc() - local pods = {} local dockerids = {} - for net,pidn in pairs(self.name_space) do - if pidn == "self" then pidn = "1" end - - self:switch_ns(pidn) + for inode,nsfile in pairs(self.name_space) do + local ns_res = self:switch_ns(nsfile) + if not ns_res then goto continue end -- local env = posix.getenv() -- env["PROC_ROOT"] = self.proc_fs - stdlib.setenv("PROC_ROOT",self.proc_fs) - local pfile = io.popen("ss -anp","r") + local pfile = io.popen("ss -an| grep -E 'tcp|udp|raw' | awk '$3 > 0'","r") io.input(pfile) for line in io.lines() do repeat - local proto,recv,task,pid = string.match(line,"(%S*)%s*%S*%s*(%d*).*users:%S*\"(%S*)\",pid=(%d*)") - if not proto or not recv or not task or not pid then break end + local proto,recv,srcip,srcport,dstip,dstport = string.match(line,"(%S*)%s*%S*%s*(%d*)%s+%d*%s*(%S*):(%d*)%s*(%S*):(%d*)") + if not proto or not recv then break end if proto ~="tcp" and proto ~="udp" and proto ~="raw" then break end - + local index = srcip..srcport..dstip..dstport + if not self.link_pid[index] then break end recv = tonumber(recv) + local pid = self.link_pid[index] local dockerid = "" if not dockerids[pid] then - dockerid = dockerinfo:get_dockerid(pid) + dockerid = dockerinfo:get_dockerid(pid, self.root_fs) if dockerid == "unknow" then break end dockerids[pid] = dockerid else dockerid = dockerids[pid] end - local podname = dockerid - if not pods[dockerid] then - podname = self:get_container_info(dockerid) - pods[dockerid] = podname + local podns = dockerid + if not self.cgroup_pod[dockerid] then + podname,podns = self:get_container_info(dockerid) + self.cgroup_pod[dockerid] = {["podname"]=podname, ["podns"]=podns} + self.cgroup_count = self.cgroup_count + 1 + if recv < 1024 and podname == dockerid then break end + else - podname = pods[dockerid] + podname = self.cgroup_pod[dockerid]["podname"] + podns = self.cgroup_pod[dockerid]["podns"] end - if recv < 1024 and podname == dockerid then break end - if not self.pod_mem[podname] then self.pod_mem[podname] = {} self.pod_mem[podname]["allocpage"] = 0 - self.pod_mem[podname]["podns"] = podname + self.pod_mem[podname]["podns"] = podns self.pod_mem[podname]["podname"] = podname end self.pod_mem[podname]["allocpage"] = self.pod_mem[podname]["allocpage"] + recv @@ -140,51 +149,117 @@ function CPodAlloc:get_pidalloc() until true end pfile:close() - self:switch_ns("1") + self:switch_ns() stdlib.setenv("PROC_ROOT","") + ::continue:: end end function CPodAlloc:scan_namespace() - local root = self.proc_fs - for pid in dirent.files(root) do - repeat - if pystring:startswith(pid,".") then break end - if not self:file_exists(self.proc_fs .. pid .. "/comm") then break end - - local proc_ns = self.proc_fs .. pid .. "/ns/net" - if not self:file_exists(proc_ns) then break end + local files = {"/var/run/docker/netns/","/var/run/netns/"} + for _,nsfile in pairs(files) do + if CPodAlloc:file_exists(self.root_fs ..nsfile) then + for file in dirent.files(self.root_fs .. nsfile) do + if not pystring:startswith(file,".") then + local fs = stat.lstat(self.root_fs .. nsfile .. file) + local inode = fs.st_ino + if not self.name_space[inode] then self.name_space[inode] = pystring:strip(self.root_fs .. nsfile .. file) end + end + end + end +end +end - local slink = unistd.readlink(proc_ns) - if not slink then break end - if not string.find(slink,"net") then break end +function CPodAlloc:get_fdlink() + for pid in dirent.files(self.proc_fs) do + if not pystring:startswith(pid,".") then + if self:file_exists(self.proc_fs .. pid .. "/fd") then + for fd in dirent.files(self.proc_fs .. pid .. "/fd") do + local res = stat.stat(self.proc_fs .. pid .. "/fd/" .. fd) + if res and stat.S_ISSOCK(res.st_mode) then + if self.inode_link[tostring(res.st_ino)] then self.link_pid[self.inode_link[tostring(res.st_ino)]] = pid end + end + end + end + end + end +end - local inode = string.match(slink,"%[(%S+)%]") - if not inode then break end +function CPodAlloc:get_link() + local files = {"/net/tcp", "/net/udp", "/net/raw"} + for _,file in pairs(files) do + local lines = io.lines(self.proc_fs .. file) + lines() + for line in lines do + local srcip, srcport, dstip, dstport, inode =string.match(line,"%s*%d+:%s*([0-9A-F]+):([0-9A-F]+)%s+([0-9A-F]+):([0-9A-F]+)%s+[0-9A-f]+%s+[0-9A-F]+:[0-9A-F]+%s+[0-9A-F]+:[0-9A-F]+%s+[0-9A-F]+%s+%d+%s+%d+%s+(%d+)") + if inode and inode ~= 0 then + srcip = tonumber(srcip:sub(7,8),16) .. "." .. tonumber(srcip:sub(5,6),16) .. "." .. tonumber(srcip:sub(3,4),16) .. "." .. tonumber(srcip:sub(1,2),16) + srcport = tonumber(srcport,16) + dstip = tonumber(dstip:sub(7,8),16) .. "." .. tonumber(dstip:sub(5,6),16) .. "." .. tonumber(dstip:sub(3,4),16) .. "." .. tonumber(dstip:sub(1,2),16) + dstport = tonumber(dstport,16) + local index = srcip..srcport..dstip..dstport + if not self.inode_link[inode] then self.inode_link[inode] = index end + end + end + end +end - if not self.name_space[inode] then self.name_space[inode] = pystring:strip(pid) end - if not self:file_exists(root .. self.name_space[inode] .. "/comm") then self.name_space[inode] = pystring:strip(pid) end - until true +function CPodAlloc:get_allpods() + local url = "http://127.0.0.1:10255/pods" + local cli = ChttpCli.new() + local content = cli:get(url) + local obj = cli:jdecode(content.body) + for _,v in pairs(obj['items']) do + if not self.blacklist[v['metadata']['namespace']] then self.allpods[v['metadata']['name']] = v['metadata']['namespace'] end end end function CPodAlloc:proc(elapsed, lines) CvProc.proc(self) + self.count = self.count + 1 + if (self.count-1) % 4 ~= 0 then + for k,v in pairs(self.allpods) do + local cell = nil + if self.pod_mem[k] then + cell = {{name="pod_allocpage", value=self.pod_mem[k]['allocpage']/1024}} + else + cell = {{name="pod_allocpage", value=0}} + end + local label = {{name="podname",index=k,}, {name="podns",index = v,},} + self:appendLine(self:_packProto("pod_alloc", label, cell)) + end + local cell = {{name="pod_allocpage_total", value=self.total/1024}} + self:appendLine(self:_packProto("pod_alloc", nil, cell)) + self:push(lines) + return + end self.name_space = {} self.pod_mem = {} + self.inode_link = {} + self.link_pid = {} + self.allpods = {} + if self.cgroup_count > 1000 then self.cgroup_pod = {} self.cgroup_count = 0 end self.total = 0 + + self:get_link() + self:get_fdlink() + self:get_allpods() + -- for k,v in pairs(self.link_pid) do print(k,v) end self:scan_namespace() self:get_pidalloc() - for k,v in pairs(self.pod_mem) do - local cell = {{name="pod_allocpage", value=v['allocpage']/1024}} - local label = {{name="podname",index=v['podname'],}, {name="namespace",index = v['podns'],},} + for k,v in pairs(self.allpods) do + local cell = nil + if self.pod_mem[k] then + cell = {{name="pod_allocpage", value=self.pod_mem[k]['allocpage']/1024}} + else + cell = {{name="pod_allocpage", value=0}} + end + local label = {{name="podname",index=k,}, {name="podns",index = v,},} self:appendLine(self:_packProto("pod_alloc", label, cell)) end local cell = {{name="pod_allocpage_total", value=self.total/1024}} self:appendLine(self:_packProto("pod_alloc", nil, cell)) - self:push(lines) end - return CPodAlloc diff --git a/source/tools/monitor/unity/collector/postEngine/engine.lua b/source/tools/monitor/unity/collector/postEngine/engine.lua new file mode 100644 index 0000000000000000000000000000000000000000..4ec4fefb1354ecc720098b206e389366fa0cdce7 --- /dev/null +++ b/source/tools/monitor/unity/collector/postEngine/engine.lua @@ -0,0 +1,126 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/25 4:21 PM +--- + +require("common.class") +local CprotoData = require("common.protoData") +local postQue = require("beeQ.postQue.postQue") +local CvProto = require("collector.vproto") +local pystring = require("common.pystring") +local system = require("common.system") +local cjson = require("cjson.safe") +local CexecBase = require("collector.postEngine.execBase") +local CexecDiag = require("collector.postEngine.execDiag") +local Cengine = class("engine", CvProto) + +local diagExec = { + io_hang = {block = 60, time = 15, cmd = "../../../iosdiag", + report = {title = "iosdiag", + files = {"/var/log/sysak/iosdiag/hangdetect/result.log.stat", + "/var/log/sysak/iosdiag/hangdetect/result.log.seq"}}}, + net_edge = {block = 5 * 60, time = 60, so = {virtiostat = 5 * 3}}, +} + +function Cengine:_init_(que, proto_q, fYaml, tid) + CvProto._init_(self, CprotoData.new(que)) + self._que = que + self._fYaml = fYaml + self._tid = tid + self._task = nil + + local res = system:parseYaml(fYaml) + self._resDiag = res.diagnose + self._diags = {} +end + +function Cengine:setMainloop(main) + self._main = main +end + +function Cengine:setTask(taskMons) + self._task = taskMons +end + +function Cengine:run(e, res, diag) + local args = res.args + local second = res.second or diag.time + if diag.cmd then + local exec = CexecDiag.new(diag.cmd, args, second, self._que, diag.report, res.uid) + exec:addEvents(e) + end + local so = diag.so + if so then + for plugin, loop in pairs(so) do + print(plugin, loop) + self._main.soPlugins:add(plugin, loop) + end + end + self._diags[res.exec] = diag.block +end + +function Cengine:pushTask(e, msgs) + local events = pystring:split(msgs, '\n') + for _, msg in ipairs(events) do + print(msg) + local res = cjson.decode(msg) + local cmd = res.cmd + if cmd == "mon_pid" then + self._task:add(res.pid, res.loop) + elseif cmd == "exec" then -- exec a cmd + local execCmd = res.exec + local args = res.args or {} + local second = res.second or 1 + local exec = CexecBase.new(execCmd, args, second) + exec:addEvents(e) + elseif cmd == "diag" and self._resDiag then + local exec = res.exec + local diag = self._resDiag[exec] + if diag then + system:dumps(diag) + if self._diags[exec] then + print("cmd " .. exec .. " is blocking.") + else + self:run(e, res, diag) + end + end + end + end +end + +function Cengine:proc(t, event, msgs) + local lines = self._proto:protoTable() + + CvProto.proc(self, t) + + self:packLog("post_que", "post", cjson.encode(msgs)) + local bytes = self._proto:encode(lines) + self._proto:que(bytes) + + self:pushTask(event, msgs) +end + +function Cengine:checkDiag() + local toDel = {} + for k, v in pairs(self._diags) do + if v > 0 then + self._diags[k] = v - 1 + else + table.insert(toDel, k) + end + end + for _, k in ipairs(toDel) do + self._diags[k] = nil + end +end + +function Cengine:work(t, event) + local msgs = postQue.pull() + if msgs then + self:proc(t, event, msgs) + end + self:checkDiag() +end + +return Cengine diff --git a/source/tools/monitor/unity/collector/postEngine/execBase.lua b/source/tools/monitor/unity/collector/postEngine/execBase.lua new file mode 100644 index 0000000000000000000000000000000000000000..77b7cb94542d5586208c96d1c692d9abecd98892 --- /dev/null +++ b/source/tools/monitor/unity/collector/postEngine/execBase.lua @@ -0,0 +1,73 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/12 00:01 +--- + +require("common.class") +local exec = require("common.exec") +local pwait = require("posix.sys.wait") +local unistd = require("posix.unistd") +local pystring = require("common.pystring") +local system = require("common.system") + +local CexecBase = class("execBase") +local interval = 5 --- poll for every 5 second + +local function checkChild(ppid, pid) + local path = "/proc/" .. pid .. "/status" + local ret = false + if unistd.access(path) then + local f = io.open(path) + + for line in f:lines() do + if pystring:startswith(line, "PPid:") then + local _, s = pystring:split(line, ":", 1) + if tonumber(pystring:strip(s)) == ppid then + ret = true + end + break + end -- end startswith + end -- end for + + f:close() + end + return ret +end + +function CexecBase:_init_(cmd, args, seconds) + self.cmd = cmd + self._cnt = 0 + self._loop = seconds / interval + + self._ppid = unistd.getpid() + self._pid = exec.run(cmd, args) +end + +function CexecBase:addEvents(e) + e:addEvent(self.cmd, self, interval, true, self._loop) +end + +function CexecBase:work() + local cnt = self._cnt + local pid, stat, exit = pwait.wait(self._pid, pwait.WNOHANG) + + if exit then + return -1, exit + end + if cnt >= self._loop then + if pid == nil then + error("wait failed " .. stat .. exit) + end + if not exit then -- process not exit + print("force to kill " .. self._pid) + if checkChild(self._ppid, self._ppid) then -- confirm child process + exec.kill(self._pid) + end + end + return -1, nil -- delete from task list. + end + self._cnt = cnt + 1 +end + +return CexecBase diff --git a/source/tools/monitor/unity/collector/postEngine/execDiag.lua b/source/tools/monitor/unity/collector/postEngine/execDiag.lua new file mode 100644 index 0000000000000000000000000000000000000000..adb6d7ef41e5e20df2a9d60c0bcfe7bc1d2945f9 --- /dev/null +++ b/source/tools/monitor/unity/collector/postEngine/execDiag.lua @@ -0,0 +1,75 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/3 23:04 +--- + +require("common.class") +local system = require("common.system") +local unistd = require("posix.unistd") +local cjson = require("cjson.safe") +local CprotoData = require("common.protoData") +local CexecBase = require("collector.postEngine.execBase") + +local CexecDiag = class("execDiag", CexecBase) + +function CexecDiag:_init_(cmd, args, seconds, que, report, uid) + CexecBase._init_(self, cmd, args, seconds) + system:dumps(args) + self._proto = CprotoData.new(que) + self._report = report + if uid then + self._uid = uid + else + self._uid = system:guid() + end +end + +function CexecDiag:work() + local ret, exit = CexecBase.work(self) + if ret == -1 and exit then -- -1 means should delete from exec list. exit and report need to report + local title = "diag_result" + local logs + if self._report then + local c = 0 + local ss = {} + for _, fName in ipairs(self._report.files) do + if unistd.access(fName) then + c = c + 1 + local f = io.open(fName, 'r') + ss[c] = f:read("*a") + end + end + + local stream = table.concat(ss) + logs = { + { + name = "uid", + log = cjson.encode(self._uid), + }, { + name = "diag", + log = cjson.encode(stream) + } + } + title = self._report.title + else + logs = { + { + name = "uid", + log = cjson.encode(self._uid), + } + } + end + local line = {line = title, log = logs} + + local lines = self._proto:protoTable() + table.insert(lines.lines, line) + system:dumps(lines) + + local bytes = self._proto:encode(lines) + self._proto:que(bytes) + end + return ret +end + +return CexecDiag \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/postPlugin/postPlugin.lua b/source/tools/monitor/unity/collector/postPlugin/postPlugin.lua new file mode 100644 index 0000000000000000000000000000000000000000..b485c05d724d24fe25558f2a1db0d04dc20f2c3d --- /dev/null +++ b/source/tools/monitor/unity/collector/postPlugin/postPlugin.lua @@ -0,0 +1,20 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/26 10:24 PM +--- + +require("common.class") + +local CtaskMons = require("collector.postPlugin.taskMons") +local CpostPlugin = class("") + +function CpostPlugin:_init_(proto, pffi, rYaml) + self.tasks = CtaskMons.new(proto, pffi, rYaml) +end + +function CpostPlugin:proc(elapsed, lines) + self.tasks:proc(elapsed, lines) +end + +return CpostPlugin diff --git a/source/tools/monitor/unity/collector/postPlugin/taskMon.lua b/source/tools/monitor/unity/collector/postPlugin/taskMon.lua new file mode 100644 index 0000000000000000000000000000000000000000..b0f409fcd4bb07c5097c219c44b94545e1378262 --- /dev/null +++ b/source/tools/monitor/unity/collector/postPlugin/taskMon.lua @@ -0,0 +1,141 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/27 11:46 AM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +local system = require("common.system") + +local CtaskMon = class("taskMon", CvProc) + +local function getInfo(fstat) + local s + for line in io.lines(fstat) do + s = line + end + local ss = pystring:rsplit(s, ") ", 1) + local comm = pystring:split(ss[1], "(", 1) + + local rest = ss[2] + local vs = pystring:split(rest, " ") + local user, sys = tonumber(vs[12]), tonumber(vs[13]) + return comm[2], user, sys +end + +local function getStatus(fstatus) + local ctxt1, ctxt2 = 0, 0 + for line in io.lines(fstatus) do + if pystring:startswith(line, "voluntary_ctxt_switches") then + local kvs = pystring:split(line) + ctxt1 = tonumber(kvs[2]) + elseif pystring:startswith(line, "nonvoluntary_ctxt_switches") then + local kvs = pystring:split(line) + ctxt2 = tonumber(kvs[2]) + end + end + return ctxt1, ctxt2 +end + +function CtaskMon:_init_(proto, pffi, mnt, pid, loop) + CvProc._init_(self, proto, pffi, mnt, nil) + self._loop = loop or -1 + self._fstat = mnt .. 'proc/' .. pid .. '/stat' + self._fstatus = mnt .. 'proc/' .. pid .. '/status' + self._comm, self._lastUser, self._lastSys = getInfo(self._fstat) + self._ls = { + { + name = "pid", + index = tostring(pid), + }, + { + name = "comm", + index = self._comm, + }, + } + self._ctxt, self._non_ctxt = getStatus(self._fstatus) +end + +function CtaskMon:stat() + local s + for line in io.lines(self._fstat) do + s = line + end + if s then + local ss = pystring:rsplit(s, ") ", 1) + local rest = ss[2] + local vs = pystring:split(rest, " ") + local user, sys = tonumber(vs[12]), tonumber(vs[13]) + local vsize, rss = tonumber(vs[21]), tonumber(vs[22]) + + local ret = { + { + name = "vsize", + value = vsize, + }, + { + name = "rss", + value = rss * 4096, + }, + { + name = "user", + value = user - self._lastUser + }, + { + name = "sys", + value = sys - self._lastSys + }, + } + self._lastUser = user + self._lastSys = sys + return ret + else + return nil + end +end + +function CtaskMon:status(vs) + for line in io.lines(self._fstatus) do + if pystring:startswith(line, "voluntary_ctxt_switches") then + local kvs = pystring:split(line) + local v = tonumber(kvs[2]) + table.insert(vs, {name="voluntary_ctxt_switches", value = v - self._ctxt}) + self._ctxt = v + elseif pystring:startswith(line, "nonvoluntary_ctxt_switches") then + local kvs = pystring:split(line) + local v = tonumber(kvs[2]) + table.insert(vs, {name="nonvoluntary_ctxt_switches", value = v - self._non_ctxt}) + self._non_ctxt = v + end + end +end + +function CtaskMon:proc(elapsed, lines) + local vs + + CvProc.proc(self) + local ls = {{ + name = "pid", + index = tostring(self.pid) + }} + + vs = self:stat() + if vs == nil then + return -1 + end + self:status(vs) + system:dumps(vs) + self:appendLine(self:_packProto("mon_task", self._ls, vs)) + self:push(lines) + + if self._loop > 0 then + self._loop = self._loop - 1 + if self._loop <= 0 then + return -1 + end + end +end + +return CtaskMon diff --git a/source/tools/monitor/unity/collector/postPlugin/taskMons.lua b/source/tools/monitor/unity/collector/postPlugin/taskMons.lua new file mode 100644 index 0000000000000000000000000000000000000000..8a08bb95dbada96e0a120f795d9a9915d311d8e7 --- /dev/null +++ b/source/tools/monitor/unity/collector/postPlugin/taskMons.lua @@ -0,0 +1,58 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/27 1:52 PM +--- + +require("common.class") +local CtaskMon = require("collector.postPlugin.taskMon") +local unistd = require("posix.unistd") +local CvProto = require("collector.vproto") +local CtaskMons = class("taskMons", CvProto) + +function CtaskMons:_init_(proto, pffi, rYaml) + CvProto._init_(self, proto) + self._pffi = pffi + self._mnt = rYaml.config.proc_path + self._limit = rYaml.config.limit.tasks or 10 + self._taskList = {} +end + +function CtaskMons:add(pid, loop) + local lines = self._proto:protoTable() + CvProto.proc(self, 1) + + if #self._taskList >= self._limit then + self:packLog("post_req", "post", "task pids already overflow.") + else + if self._taskList[pid] then -- already in + self:packLog("post_req", "post", "pid " .. pid .. " is already in mon list.") + else -- check if in proc + local dPath = self._mnt .. 'proc/' .. pid + local res = unistd.access(dPath, 'r') + if res then -- exist + self._taskList[pid] = CtaskMon.new(self._proto, self._pffi, self._mnt, pid, loop) + self:packLog("post_req", "post", "add pid " .. pid .. " to monitor.") + else + self:packLog("post_req", "post", "pid " .. pid .. " is not exist.") + end + end + end + self:push(lines) + + local bytes = self._proto:encode(lines) + self._proto:que(bytes) +end + +function CtaskMons:proc(elapsed, lines) + local res + + for pid, task in pairs(self._taskList) do + res = task:proc(elapsed, lines) + if res == -1 then + self._taskList[pid] = nil + end + end +end + +return CtaskMons diff --git a/source/tools/monitor/unity/collector/proc_bled.lua b/source/tools/monitor/unity/collector/proc_bled.lua new file mode 100644 index 0000000000000000000000000000000000000000..4ba427d6511fc25b87d0a8259d29c178249ddbbc --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_bled.lua @@ -0,0 +1,30 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/21 12:08 AM +--- +--- + +require("common.class") +local CvProc = require("collector.vproc") + +local CprocBled = class("procArp", CvProc) + +function CprocBled:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "dummy") + self._record = 1 +end + +function CprocBled:proc(elapsed, lines) + print("this only a test case for bled " .. self._record) + self._record = self._record + 1 + if self._record > 5 then + print("trig full cpu.") + local t = os.time() + 2 + while os.time() < t do + local c = 1 + end + end +end + +return CprocBled diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index e324c0d56c989f2972c2b045cf743453ee8b3e88..cf5f4c4479041d749d73db507293845a9672f213 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -23,7 +23,8 @@ function CprocBuddyinfo:proc(elapsed, lines) CvProc.proc(self) self._protoTable.vs = {} local buddyinfo = {} - for line in io.lines(self.pFile) do + local fInfo = io.open(self.pFile, "r") + for line in fInfo:lines() do if string.find(line,"Normal") then local subline = pystring:split(line,"Normal",1)[2] for num in string.gmatch(subline, "%d+") do @@ -32,9 +33,11 @@ function CprocBuddyinfo:proc(elapsed, lines) break end end + fInfo:close() - if not buddyinfo then - for line in io.lines(self.pFile) do + if #buddyinfo == 0 then + fInfo = io.open(self.pFile, "r") + for line in fInfo:lines() do if string.find(line,"DMA32") then local subline = pystring:split(line,"DMA32",1)[2] for num in string.gmatch(subline, "%d+") do @@ -43,6 +46,7 @@ function CprocBuddyinfo:proc(elapsed, lines) break end end + fInfo:close() end for k,v in pairs(buddyinfo) do diff --git a/source/tools/monitor/unity/collector/proc_diskstats.lua b/source/tools/monitor/unity/collector/proc_diskstats.lua index a8535414e6e969e6319884bc31581a0c233f4e4b..173895e26100eda3193d5c9328aaedb41459b5f7 100644 --- a/source/tools/monitor/unity/collector/proc_diskstats.lua +++ b/source/tools/monitor/unity/collector/proc_diskstats.lua @@ -86,7 +86,7 @@ function CprocDiskstats:_calcDiff(disk_name, now, last, elapsed) cell = { name = "busy", - value = (now["time"] - now["time"]) / elapsed + value = (now["time"] - last["time"]) / elapsed } table.insert(protoTable.vs, cell) self:appendLine(protoTable) diff --git a/source/tools/monitor/unity/collector/proc_interrupts.lua b/source/tools/monitor/unity/collector/proc_interrupts.lua index 9b66c3447edb5c46caaf01dca5c17098f79cb355..fc2269ddb41d3470956474a69d582cc1d2998f4b 100644 --- a/source/tools/monitor/unity/collector/proc_interrupts.lua +++ b/source/tools/monitor/unity/collector/proc_interrupts.lua @@ -8,12 +8,17 @@ require("common.class") local pystring = require("common.pystring") local CvProc = require("collector.vproc") local unistd = require("posix.unistd") +local system = require("common.system") local Cinterrupts = class("interrupts", CvProc) function Cinterrupts:_init_(proto, pffi, mnt, pFile) CvProc._init_(self, proto, pffi, mnt, pFile or "proc/interrupts") - self._cpus = unistd.sysconf(84) + local err, errno + self._cpus, err, errno = unistd.sysconf(84) + if err then + system:posixError("sysconf failed", err, errno) + end end function Cinterrupts:proc(elapsed, lines) diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index a9e79e10f3f6956f62fa3d963844cbc45c825622..262d54330db519ea2af937099caf95355a18e431 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -17,10 +17,123 @@ function CprocSnmpStat:_init_(proto, pffi, mnt, pFile) self._rec = nil end +local ipHeads = {"InReceives", "OutRequests"} +function CprocSnmpStat:ipCount(titles, values) + local vs = {} + for i = 1, titles.no do + local cell = self._ffi.string(titles.s[i]) + if cell == "Forwarding" then + table.insert(vs, {name = cell, value = tonumber(values.value[i - 1])}) + elseif system:valueIsIn(ipHeads, cell) then + local name = "v_ip" .. cell + local v = tonumber(values.value[i - 1]) + if self[name] then + table.insert(vs, {name = cell, value = v - self[name]}) + end + self[name] = v + end + end + self:appendLine(self:_packProto("net_ip_count", nil, vs)) +end + +local icmpHeads = {"InMsgs", "InErrors", "OutMsgs", "OutErrors"} +function CprocSnmpStat:icmpCount(titles, values) + local vs = {} + for i = 1, titles.no do + local cell = self._ffi.string(titles.s[i]) + if system:valueIsIn(icmpHeads, cell) then + local name = "v_icmp" .. cell + local v = tonumber(values.value[i - 1]) + if self[name] then + table.insert(vs, {name = cell, value = v - self[name]}) + end + self[name] = v + end + end + if #vs > 0 then + self:appendLine(self:_packProto("net_icmp_count", nil, vs)) + end +end + +local udpHeads = {"InDatagrams", "InErrors", "OutDatagrams", + "RcvbufErrors", "SndbufErrors", "NoPorts"} +function CprocSnmpStat:udpCount(titles, values) + local vs = {} + for i = 1, titles.no do + local cell = self._ffi.string(titles.s[i]) + if system:valueIsIn(udpHeads, cell) then + local name = "v_udp" .. cell + local v = tonumber(values.value[i - 1]) + if self[name] then + table.insert(vs, {name = cell, value = v - self[name]}) + end + self[name] = v + end + end + if #vs > 0 then + self:appendLine(self:_packProto("net_udp_count", nil, vs)) + end +end + +local tcpHeads = {"InSegs", "OutSegs", "RetransSegs", "InErrs"} +function CprocSnmpStat:tcpCount(titles, values) + local vs = {} + for i = 1, titles.no do + local cell = self._ffi.string(titles.s[i]) + if cell == "CurrEstab" then + table.insert(vs, {name = cell, value = tonumber(values.value[i - 1])}) + elseif system:valueIsIn(tcpHeads, cell) then + local name = "v_tcp" .. cell + local v = tonumber(values.value[i - 1]) + if self[name] then + table.insert(vs, {name = cell, value = v - self[name]}) + end + self[name] = v + end + end + if #vs > 0 then + self:appendLine(self:_packProto("net_tcp_count", nil, vs)) + end +end + +local tcpExtHeads = {"TCPSynRetrans", "ListenDrops", "ListenOverflows", + "SyncookiesSent", "SyncookiesRecv", "SyncookiesFailed", + "ActiveOpens", "PassiveOpens"} +function CprocSnmpStat:tcpExtCount(titles, values) + local vs = {} + for i = 1, titles.no do + local cell = self._ffi.string(titles.s[i]) + if system:valueIsIn(tcpExtHeads, cell) then + local name = "v_tcpExt" .. cell + local v = tonumber(values.value[i - 1]) + if self[name] then + table.insert(vs, {name = cell, value = v - self[name]}) + end + self[name] = v + end + end + if #vs > 0 then + self:appendLine(self:_packProto("net_tcp_ext_count", nil, vs)) + end +end + function CprocSnmpStat:createTable(titles, values, now) local head = string.gsub(self._ffi.string(titles.s[0]), ":", "") assert(self._ffi.string(titles.s[0]), self._ffi.string(values.s)) - for i=1, titles.no do + + if head == "Tcp" then + self:tcpCount(titles, values) + elseif head == "Udp" then + self:udpCount(titles, values) + elseif head == "Ip" then + self:ipCount(titles, values) + elseif head == "Icmp" then + self:icmpCount(titles, values) + elseif head == "TcpExt" then + self:tcpExtCount(titles, values) + end + + for i = 1, titles.no do local cell = self._ffi.string(titles.s[i]) local low = string.lower(cell) for j, mon in ipairs(self._cellsMon) do diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index f1ad61e37e66e7d0dc95a2e8f0b6f64f738e5bf3..d497c6a5bc99451e7be1f506f306dd43dbf2170d 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -6,18 +6,20 @@ require("common.class") local CvProc = require("collector.vproc") - +local unistd = require("posix.unistd") local CprocStatm = class("procStatm", CvProc) function CprocStatm:_init_(proto, pffi, mnt, pFile) CvProc._init_(self, proto, pffi, mnt, pFile or nil) + local pid = unistd.getpid() + self._fstatm = "/proc/" .. pid .. "/statm" end function CprocStatm:proc(elapsed, lines) CvProc.proc(self) local heads = {"size", "resident", "shared", "text", "lib", "data", "dt"} local c = 0 - for line in io.lines("/proc/self/statm") do + for line in io.lines(self._fstatm) do local vs = {} local data = self._ffi.new("var_long_t") assert(self._cffi.var_input_long(self._ffi.string(line), data) == 0) diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 05024bc8e2ebfd9a608aa97f838ffdd6324b7366..60a1a54483ac06747baad5f92df6aded6f4a60be 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -5,12 +5,12 @@ --- require("common.class") -local system = require("common.system") +local CvProto = require("collector.vproto") -local CvProc = class("collector.vproc") +local CvProc = class("collector.vproc", CvProto) function CvProc:_init_(proto, pffi, mnt, pFile) - self._proto = proto + CvProto._init_(self, proto) self._cffi = pffi["cffi"] self._ffi = pffi["ffi"] mnt = mnt or "/" @@ -18,29 +18,4 @@ function CvProc:_init_(proto, pffi, mnt, pFile) self.pFile = mnt .. pFile end -function CvProc:proc(elapsed) - self._lines = self._proto:protoTable() -end - -function CvProc:appendLine(line) - table.insert(self._lines["lines"], line) -end - -function CvProc:copyLine(line) - self:appendLine(system:deepcopy(line)) -end - -function CvProc:push(lines) - local c = #lines["lines"] -- not for #lines - for _, line in ipairs(self._lines["lines"]) do - c = c + 1 - lines["lines"][c] = line - end - self._lines = nil -end - -function CvProc:_packProto(head, labels, vs, log) - return {line = head, ls = labels, vs = vs, log = log} -end - return CvProc \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/vproto.lua b/source/tools/monitor/unity/collector/vproto.lua new file mode 100644 index 0000000000000000000000000000000000000000..403a5f8aae38c122a69b675662e2ac02e34ecd0d --- /dev/null +++ b/source/tools/monitor/unity/collector/vproto.lua @@ -0,0 +1,52 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/25 5:28 PM +--- + +require("common.class") +local system = require("common.system") + +local CvProto = class("collector.vproto") + +function CvProto:_init_(proto) + self._proto = proto +end + +function CvProto:proc(elapsed) + self._lines = self._proto:protoTable() +end + +function CvProto:appendLine(line) + --assert(not self._lines, "your class should call CvProto.proc at first.") + table.insert(self._lines["lines"], line) +end + +function CvProto:copyLine(line) + self:appendLine(system:deepcopy(line)) +end + +function CvProto:push(lines) + local c = #lines["lines"] -- not for #lines + for _, line in ipairs(self._lines["lines"]) do + c = c + 1 + lines["lines"][c] = line + end + self._lines = nil +end + +function CvProto:_packProto(head, labels, vs, log) + return {line = head, ls = labels, vs = vs, log = log} +end + +function CvProto:packLog(title, name, msg) + local logList = {} + + logList[1] = { + name = name, + log = msg + } + self:appendLine(self:_packProto(title, nil, nil, logList)) +end + +return CvProto diff --git a/source/tools/monitor/unity/common/dockerinfo.lua b/source/tools/monitor/unity/common/dockerinfo.lua index 9bbce2ced08f0681ad39532d23692996b615f81b..2322eee5ad59a8309bfb350e702220e9daea4d82 100644 --- a/source/tools/monitor/unity/common/dockerinfo.lua +++ b/source/tools/monitor/unity/common/dockerinfo.lua @@ -4,15 +4,16 @@ --- DateTime: 2023/02/08 17:00 PM --- -dockerinfo = {} +local dockerinfo = {} local posix = require("posix") -local cjson = require("cjson") -local json = cjson.new() local pystring = require("common.pystring") local stat = require("posix.sys.stat") +local unistd = require("posix.unistd") +local system = require("common.system") +local api = require("httplib.dockerApi") -function file_exists(file) - local f=stat.lstat(file) +local function file_exists(file) + local f=unistd.access(file) if f ~= nil then return true else @@ -20,31 +21,14 @@ function file_exists(file) end end -function dockerinfo:get_hostfs() - local proc_fs="/mnt/host/proc/" - local sys_fs="/mnt/host/sys/" - local pods_fs="/mnt/host/var/lib/kubelet/pods/" - local root_fs = "/mnt/host/" - if file_exists(proc_fs) then - return proc_fs, sys_fs, pods_fs, root_fs - end - proc_fs="/proc/" - sys_fs="/sys/" - pods_fs="/var/lib/kubelet/pods/" - root_fs = "/" - return proc_fs, sys_fs, pods_fs, root_fs -end - -function get_runtimesock() - local root_fs = "" - _, _, _, root_fs = dockerinfo:get_hostfs() +local function get_runtimesock(root_fs) local runtime = "docker" local runtime_sock = root_fs .. "var/run/docker.sock" - local sock={"var/run/docker.sock","run/containerd/containerd.sock", "var/run/dockershim.sock"} + local sock={"var/run/docker.sock","run/podman/podman.sock","run/containerd/containerd.sock", "var/run/dockershim.sock"} for _,runtimex in pairs(sock) do if file_exists(root_fs .. runtimex) then runtime_sock = root_fs .. runtimex - if not string.find(runtime_sock,"docker.sock") then + if not string.find(runtime_sock,"docker.sock") and not string.find(runtime_sock,"podman.sock") then runtime = "crictl" end end @@ -52,17 +36,76 @@ function get_runtimesock() return runtime,runtime_sock end -function dockerinfo:get_inspect(did) - local runtime,runtime_sock = get_runtimesock() +local function get_container_inspect(did, root_fs) + local runtime, runtime_sock = get_runtimesock(root_fs) + local d_api = api.new('localhost', runtime_sock) + local res = d_api:inspect_container(did) + if res then res = res.body end + return res +end + +local function get_crictl_inspect(did, root_fs) + local runtime, runtime_sock = get_runtimesock(root_fs) + local cmd = runtime .. " -r " .. runtime_sock .. " inspect " .. did .. " 2>/dev/null " + local f = io.popen(cmd,"r") + local res = f:read("*a") + f:close() + return res +end + +function dockerinfo:get_podname_pid(pid, root_fs) + local did = dockerinfo:get_dockerid(pid, root_fs) + return dockerinfo:get_podname_did(did,root_fs) +end + +function dockerinfo:get_podname_did(did, root_fs) + local restable = dockerinfo:get_inspect(did,root_fs) + local podname = did + local podns = did + local cname = did + + if not restable then return did end + if #restable > 0 then + restable = restable[1] + end + if restable['Config'] then + local config = restable['Config'] + if config['Labels'] then + local label = config['Labels'] + if label['io.kubernetes.pod.name'] then + podname = label['io.kubernetes.pod.name'] + end + if label['io.kubernetes.container.name'] then + cname = label['io.kubernetes.container.name'] + end + if label['io.kubernetes.pod.namespace'] then + podns = label['io.kubernetes.pod.namespace'] + end + end + if podname == did and restable['Name'] then + cname = restable['Name'] + podname = restable['Name'] + end + elseif restable['status'] then + podname = restable['status']['labels']['io.kubernetes.pod.name'] + cname = restable['status']['labels']['io.kubernetes.container.name'] + podns = restable['status']['labels']['io.kubernetes.pod.namespace'] + end + if pystring:startswith(podname,"/") then podname=string.sub(podname,2,-1) end + return podname +end + +function dockerinfo:get_inspect(did, root_fs) + local runtime,runtime_sock = get_runtimesock(root_fs) if runtime == "docker" then - return get_container_inspect(did) + return get_container_inspect(did, root_fs) else - return get_crictl_inspect(did) + return get_crictl_inspect(did, root_fs) end end -function dockerinfo:get_dockerid(pid) - local proc_fs = dockerinfo:get_hostfs() +function dockerinfo:get_dockerid(pid, root_fs) + local proc_fs = root_fs .. "/proc/" local idstring = "unknow" if not file_exists(proc_fs .. pid .. "/cgroup") then return idstring end local cmd = "cat " .. proc_fs .. pid .. "/cgroup 2>/dev/null | grep memory:" @@ -70,11 +113,15 @@ function dockerinfo:get_dockerid(pid) local res = pfile:read("*a") pfile:close() - if not string.find(res,"kubepods") and not string.find(res,"docker%-") then return idstring end + if not string.find(res,"kubepods") and not string.find(res,"docker%-") and not string.find(res,"libpod") then return idstring end if string.find(res,"docker%-") then idstring = pystring:split(res,"docker-")[2] elseif string.find(res,"cri%-containerd%-") then idstring = pystring:split(res,"cri-containerd-")[2] + elseif string.find(res,"libpod%-conmon%-") then + idstring = pystring:split(res,"libpod-conmon-")[2] + elseif string.find(res,"libpod%-") then + idstring = pystring:split(res,"libpod-")[2] else local tmp = pystring:split(res,"/",10) idstring = tmp[#tmp] @@ -83,22 +130,4 @@ function dockerinfo:get_dockerid(pid) return idstring end -function get_container_inspect(did) - local runtime, runtime_sock = get_runtimesock() - local cmd = "curl --silent -XGET --unix-socket " .. runtime_sock .. " http://localhost/containers/" .. did .. "/json 2>/dev/null " - local f = io.popen(cmd,"r") - local res = f:read("*a") - f:close() - return res -end - -function get_crictl_inspect(did) - local runtime, runtime_sock = get_runtimesock() - local cmd = runtime .. " -r " .. runtime_sock .. " inspect " .. did .. " 2>/dev/null " - local f = io.popen(cmd,"r") - local res = f:read("*a") - f:close() - return res -end - return dockerinfo diff --git a/source/tools/monitor/unity/common/exec.lua b/source/tools/monitor/unity/common/exec.lua new file mode 100644 index 0000000000000000000000000000000000000000..ad98d63994f36c42c64a86ca8078739a373586a2 --- /dev/null +++ b/source/tools/monitor/unity/common/exec.lua @@ -0,0 +1,33 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/4 10:50 +--- + +local unistd = require("posix.unistd") +local pwait = require("posix.sys.wait") +local signal = require("posix.signal") + +local module = {} + +function module.run(cmd, args) + local pid, err = unistd.fork() + if pid > 0 then -- for self + print("pid: " .. pid) + return pid + elseif pid == 0 then -- for child + local errno + prctl_death_kill() + _, err, errno = unistd.exec(cmd, args) + assert(not errno, "exec failed." .. err .. errno) + else + error("fork report" .. err) + end +end + +function module.kill(pid) + signal.kill(pid, signal.SIGKILL) -- force to kill task + pwait.wait(pid) -- wait task +end + +return module diff --git a/source/tools/monitor/unity/common/fifo.lua b/source/tools/monitor/unity/common/fifo.lua new file mode 100644 index 0000000000000000000000000000000000000000..76dfd3075fd964f592251cbc80eb6d242c584208 --- /dev/null +++ b/source/tools/monitor/unity/common/fifo.lua @@ -0,0 +1,68 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/29 9:08 PM +--- + +require("common.class") + +local Cfifo = class("fifo") + +function Cfifo:_init_(maxLen) + self._max = maxLen or 16 + self.list = {} + self._head = 0 + self._tail = -1 + self._count = 0 +end + +function Cfifo:push(v) + local tail = self._tail + 1 + local count = self._count + 1 + self._tail = tail + self._count = count + + self.list[tail] = v + if count > self._max then + self:pop() + end +end + +function Cfifo:pop() + local count = self._count + if count == 0 then + return nil + else + local head = self._head + local v = self.list[head] + self.list[head] = nil + self._head = head + 1 + self._count = count - 1 + return v + end +end + +function Cfifo:len() + return self._count +end + +function Cfifo:capacity() + return self._max +end + +function Cfifo:value(index) + index = index + self._head - 1 + return self.list[index] +end + +function Cfifo:toList() + local c = 0 + local res = {} + for _, v in pairs(self.list) do + c = c + 1 + res[c] = v + end + return res +end + +return Cfifo diff --git a/source/tools/monitor/unity/common/fifoNum.lua b/source/tools/monitor/unity/common/fifoNum.lua new file mode 100644 index 0000000000000000000000000000000000000000..ecc9ece38d56f521bb4aebbb0baccffe09997b11 --- /dev/null +++ b/source/tools/monitor/unity/common/fifoNum.lua @@ -0,0 +1,61 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/29 11:58 PM +--- + +require("common.class") + +local Cfifo = require("common.fifo") + +local CfifoNum = class("fifoNum", Cfifo) + +function CfifoNum:_init_(maxLen) + Cfifo._init_(self, maxLen) +end + +function CfifoNum:max() + assert(self._count > 0, "fifo count should > 0") + local res, index + + res, index = self.list[self._head], 1 + for k, v in pairs(self.list) do + if v > res then + res = v + index = k - self._head + 1 + end + end + return res, index +end + +function CfifoNum:min() + assert(self._count > 0, "fifo count should > 0") + local res, index + + res, index = self.list[self._head], 1 + for k, v in pairs(self.list) do + if v < res then + res = v + index = k - self._head + 1 + end + end + return res, index +end + +function CfifoNum:sum() + local sum = 0 + + for _, v in pairs(self.list) do + sum = sum + v + end + return sum +end + +function CfifoNum:average() + assert(self._count > 0, "fifo count should > 0") + local sum = self:sum() + + return sum / self._count +end + +return CfifoNum \ No newline at end of file diff --git a/source/tools/monitor/unity/common/inotifies.lua b/source/tools/monitor/unity/common/inotifies.lua new file mode 100644 index 0000000000000000000000000000000000000000..d8b33c784f23322e43c0b1ef0c579179d9e9fe4c --- /dev/null +++ b/source/tools/monitor/unity/common/inotifies.lua @@ -0,0 +1,67 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/24 2:34 PM +--- + +require("common.class") + +local Cinotifies = class("inotifies") +local dirent = require("posix.dirent") +local pstat = require("posix.sys.stat") +local system = require("common.system") +local inotify = require('inotify') + +local function walk_dirs(path, dirs) + local files = dirent.files(path) + for f in files do + if string.sub(f, 1, 1) ~= '.' then + local full = table.concat({path, f}, "/") + local lStat = pstat.lstat(full) + if pstat.S_ISDIR(lStat.st_mode) > 0 then + table.insert(dirs, full) + walk_dirs(full, dirs) + end + end + end +end + +local function mon_dirs(path) + local handle = inotify.init() + local ws = {} + local c = 0 + + local dirs = {path} + walk_dirs(path, dirs) + + for _, dir in ipairs(dirs) do + local w = handle:addwatch(dir, inotify.IN_CREATE, inotify.IN_MOVE, inotify.IN_DELETE) + if w > 0 then + c = c + 1 + ws[c] = w + else + error("add " .. dir .. " to watch failed.") + end + end + + system:fdNonBlocking(handle:getfd()) + return handle, ws +end + +function Cinotifies:_init_(path) + self._handle, self._ws = mon_dirs(path) +end + +function Cinotifies:_del_() + for _, w in ipairs(self._ws) do + self._handle:rmwatch(w) + end + self._handle:close() +end + +function Cinotifies:isChange() + local events = self._handle:read() + return #events > 0 +end + +return Cinotifies diff --git a/source/tools/monitor/unity/common/iplib.lua b/source/tools/monitor/unity/common/iplib.lua new file mode 100644 index 0000000000000000000000000000000000000000..8fe36962774ddf359230f09e919e7bae264350e8 --- /dev/null +++ b/source/tools/monitor/unity/common/iplib.lua @@ -0,0 +1,88 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/9 23:39 +--- + +require("common.class") +local pystring = require("common.pystring") +local system = require("common.system") +local bit = require("bit") + +local Ciplib = class("iplib") + +local function toIP(s) + return tonumber(s, 16) +end + +local function get_routes(mnt) + local c = 0 + local routes = {} + for line in io.lines(mnt .. "proc/net/route") do + if c > 0 then + local cells = pystring:split(line) + local route = { + ["if"] = cells[1], + dst = toIP(cells[2]), + gw = toIP(cells[3]), + mask = toIP(cells[8]), + flag = tonumber(cells[4]), + } + routes[c] = route + end + c = c + 1 + end + + table.sort(routes, function(a, b) return a.mask > b.mask end) + return routes +end + +function Ciplib:_init_(mnt) + self._routes = get_routes(mnt) +end + +function Ciplib:toInt(ip) + local num = 0 + if ip and type(ip)=="string" then + local o1, o2, o3, o4 = ip:match("(%d+)%.(%d+)%.(%d+)%.(%d+)") + num = 2^24*o4 + 2^16*o3 + 2^8*o2 + o1 + end + return num +end + +function Ciplib:toIp(n) + if n then + n = tonumber(n) + local n1 = math.floor(n / (2^24)) + local n2 = math.floor((n - n1*(2^24)) / (2^16)) + local n3 = math.floor((n - n1*(2^24) - n2*(2^16)) / (2^8)) + local n4 = math.floor((n - n1*(2^24) - n2*(2^16) - n3*(2^8))) + return pystring:join(".", {n4, n3, n2, n1}) + end + return "0.0.0.0" +end + +function Ciplib:getRoute(ip) + local num = self:toInt(ip) + for _, route in pairs(self._routes) do + local mask = route.mask + if bit.band(mask, num) == bit.band(mask, route.dst) then + return route + end + end + return nil +end + +function Ciplib:routeIp(ip) + local route = self:getRoute(ip) + if route then + if bit.band(route.flag, 0x02) == 0x02 then -- need gate + return self:toIp(route.gw) + else -- direct + return ip + end + end + return nil +end + +return Ciplib diff --git a/source/tools/monitor/unity/common/lineParse.lua b/source/tools/monitor/unity/common/lineParse.lua index 33d6486a0e831608f41ce0956c6e87539b1be50e..a70e0d35758dbd0fc55508750e37c5d3695ce511 100644 --- a/source/tools/monitor/unity/common/lineParse.lua +++ b/source/tools/monitor/unity/common/lineParse.lua @@ -10,6 +10,8 @@ local system = require("common.system") local module = {} local json = cjson.new() +json.encode_escape_forward_slash(false) + local function parseLabel(sls, ls) local lss = pystring:split(sls, ",") for _, cell in ipairs(lss) do @@ -79,11 +81,12 @@ end function module.pack(title, ls, vs) local line = title - if system:keyCount(ls) > 0 then + if ls and system:keyCount(ls) > 0 then local lss = {} local c = 0 for k, v in pairs(ls) do c = c + 1 + v = string.gsub(v, "%s", "_") lss[c] = table.concat({k, v}, "=") end line = line .. ',' .. pystring:join(",", lss) @@ -99,6 +102,7 @@ function module.pack(title, ls, vs) c = c + 1 vss[c] = table.concat({k, json.encode(v)}, "=") else + system:dumps(tStr) error("bad value type for " .. tStr) end end @@ -106,4 +110,34 @@ function module.pack(title, ls, vs) return line end +function module.packs(line) + local cells + local title = line.line + local ls = {} + local vs = {} + + cells = line.ls + if cells then + for _, cell in ipairs(cells) do + ls[cell.name] = cell.index + end + end + + cells = line.vs + if cells then + for _, cell in ipairs(cells) do + vs[cell.name] = cell.value + end + end + + cells = line.log + if cells then + for _, cell in ipairs(cells) do + vs[cell.name] = cell.log + end + end + + return module.pack(title, ls, vs) +end + return module diff --git a/source/tools/monitor/unity/common/ping.lua b/source/tools/monitor/unity/common/ping.lua new file mode 100644 index 0000000000000000000000000000000000000000..1e9315a10d008ca46550da3eba50e75c86ca7741 --- /dev/null +++ b/source/tools/monitor/unity/common/ping.lua @@ -0,0 +1,65 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/10 00:50 +--- + +require("common.class") + +local psocket = require("posix.sys.socket") +local ptime = require("posix.sys.time") +local poll = require("posix.poll") +local unistd = require("posix.unistd") +local system = require("common.system") +local Cping = class("ping") + +function Cping:_init_(ip, dev) + assert(psocket.SOCK_RAW and psocket.SO_BINDTODEVICE, "not support raw socket yet.") + self.ip = ip + + local fd, err = psocket.socket(psocket.AF_INET, psocket.SOCK_RAW, psocket.IPPROTO_ICMP) + assert(fd, err) + + local ok, err = psocket.setsockopt(fd, psocket.SOL_SOCKET, psocket.SO_BINDTODEVICE, dev) + assert(ok, err) + + system:fdNonBlocking(fd) + + self.fd = fd +end + +function Cping:_del_() + if self.fd then + unistd.close(self.fd) + end +end + +local function diff_us(t1, t0) + local sum1 = t1.tv_sec * 1e6 + t1.tv_usec + local sum0 = t0.tv_sec * 1e6 + t0.tv_usec + return sum1 - sum0 +end + +function Cping:ping() + local fd = self.fd + local data = string.char(0x08, 0x00, 0x89, 0x98, 0x6e, 0x63, 0x00, 0x04, 0x00) + local t0 = ptime.gettimeofday() + local ok, err = psocket.sendto(fd, data, {family=psocket.AF_INET, addr=self.ip, port=0}) + assert(ok, err) + + local r = poll.rpoll(fd, 300) + if r == 0 then + print("receive over time.") + return -1 + elseif r == 1 then + local data, sa = psocket.recvfrom(fd, 1024) + local t1 = ptime.gettimeofday() + assert(data, sa) + return diff_us(t1, t0) + else + print("closed.") + return -2 + end +end + +return Cping diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index a499ee85cafe28c4bc836128f696a2595f802e8f..84e5e8b57abffc0ccc727a7d18263c3e59dff0db 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -337,22 +337,7 @@ function pystring:partition(s, del) end end -function pystring:partition(s, del) - local result = {} - del = del or " " - local delimiter = setupDelimiter(del) - local iBeg, iEnd = string.find(s, delimiter) - if iBeg then - result[1] = string.sub(s, 1, iBeg - 1) - result[2] = del - result[3] = string.sub(s, iEnd + 1) - return result - else - return nil - end -end - -function pystring:reverseTable(t) +local function reverseTable(t) local n = #t for i = 1, n / 2 do t[i], t[n + 1 - i] = t[n + 1 - i], t[i] @@ -389,7 +374,7 @@ function pystring:rsplit(s, delimiter, n) end end --return result - pystring:reverseTable(result) + reverseTable(result) return result end diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index ce4f77033eea77f5dd4c4adf4cab230d5c6d5ce6..b1220ade32855aaf4e15b827236141876e7cb90b 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -6,13 +6,30 @@ local socket = require("socket") local serpent = require("common.serpent") - local system = {} function system:sleep(t) socket.select(nil, nil, t) end +function system:fdNonBlocking(fd) + local res + local fcntl = require("posix.fcntl") + local bit = require("bit") + + local flag, err, errno = fcntl.fcntl(fd, fcntl.F_GETFL) + if flag then + res, err, errno = fcntl.fcntl(fd, fcntl.F_SETFL, bit.bor(flag, fcntl.O_NONBLOCK)) + if res then + return 0 + else + system:posixError("fcntl set failed", err, errno) + end + else + system:posixError("fcntl get failed", err, errno) + end +end + function system:deepcopy(object) local lookup_table = {} local function _copy(object) @@ -40,6 +57,13 @@ function system:dumps(t) print(serpent.block(t)) end +function system:reverseTable(t) + local n = #t + for i = 1, n / 2 do + t[i], t[n + 1 - i] = t[n + 1 - i], t[i] + end +end + function system:keyIsIn(tbl, key) if type(tbl) ~= "table" then return false @@ -174,4 +198,31 @@ function system:posixError(msg, err, errno) error(s) end +function system:Enum(tbl, index) + local eTbl = {}; + index = index or 0; + for i, v in ipairs(tbl) do + eTbl[v] = index + i; + end + return eTbl; +end + +function system:guid() + math.randomseed(os.time()) + local seed={'e','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'} + local tb={} + for i = 1,32 do + table.insert(tb,seed[math.random(1,16)]) + end + + local sid = table.concat(tb) + return string.format('%s-%s-%s-%s-%s', + string.sub(sid,1,8), + string.sub(sid,9,12), + string.sub(sid,13,16), + string.sub(sid,17,20), + string.sub(sid,21,32) + ) +end + return system \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/asyncDns.lua b/source/tools/monitor/unity/httplib/asyncDns.lua new file mode 100644 index 0000000000000000000000000000000000000000..d327571cb1144cc174bfcddf9d9f4f768b5a248f --- /dev/null +++ b/source/tools/monitor/unity/httplib/asyncDns.lua @@ -0,0 +1,115 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 11:41 +--- + +require("common.class") +local psocket = require("posix.sys.socket") +local unistd = require("posix.unistd") +local pystring = require("common.pystring") +local system = require("common.system") + +local CasyncDns = class("coDns") + +local function lookup_server() + local f = io.open("/etc/resolv.conf") + local server = "" + for line in f:lines() do + if pystring:startswith(line, "nameserver") then + local res = pystring:split(line) + server = res[2] + break + end + end + + f:close() + return server +end + +function CasyncDns:_init_() + self._server = lookup_server() + assert(#self._server > 0) +end + +local function packQuery(domain) + local cnt = 0 + local queries = {} + local head = string.char( + 0x12, 0x34, -- Query ID + 0x01, 0x00, -- Standard query + 0x00, 0x01, -- Number of questions + 0x00, 0x00, -- Number of answers + 0x00, 0x00, -- Number of authority records + 0x00, 0x00 -- Number of additional records + ) + cnt = cnt + 1 + queries[cnt] = head + + local names = pystring:split(domain, ".") + for _, name in ipairs(names) do + cnt = cnt + 1 + queries[cnt] = string.char(string.len(name)) + cnt = cnt + 1 + queries[cnt] = name + end + cnt = cnt + 1 + local tail = string.char( + 0x00, -- End of domain name + 0x00, 0x01, -- Type A record + 0x00, 0x01 -- Class IN + ) + queries[cnt] = tail + + local query = table.concat(queries) + return query +end + +function CasyncDns:waitReq(fd) + local res, msg + local toWake = coroutine.yield() + local e = coroutine.yield() + local ip + if e.ev_close > 0 then + print("force to close socket.") + elseif e.ev_in > 0 then + res = psocket.recvfrom(fd, 512) + if res then + ip = string.format("%d.%d.%d.%d", string.byte(res, -4, -1)) + end + else + print("bad socket.") + end + res, msg = coroutine.resume(toWake, ip) + assert(res, msg) + g_lb:co_exit(fd) +end + +function CasyncDns:dns_lookup(domain) + self._coWake = nil + local res, msg + if domain == "localhost" then + return "127.0.0.1" + end + + local fd, err, errno = psocket.socket(psocket.AF_INET, psocket.SOCK_DGRAM, 0) + if not fd then + system:posixError("new socket failed.", err, errno) + end + local tDist = {family=psocket.AF_INET, addr=self._server, port=53} + + local co = g_lb:co_add(fd, self.waitReq) + local query = packQuery(domain) + local len, err, errno = psocket.sendto(fd, query, tDist) + if not len then + system:posixError("socket send failed.", err, errno) + end + + res, msg = coroutine.resume(co, self, fd) -- send fd + assert(res, msg) + res, msg = coroutine.resume(co, coroutine.running()) -- send to wake + assert(res, msg) + return coroutine.yield() +end + +return CasyncDns \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/asyncHttp.lua b/source/tools/monitor/unity/httplib/asyncHttp.lua new file mode 100644 index 0000000000000000000000000000000000000000..37ffa8ed0afb45d3c8c1f70097700a87646ea952 --- /dev/null +++ b/source/tools/monitor/unity/httplib/asyncHttp.lua @@ -0,0 +1,374 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 15:38 +--- + +require("common.class") +local ChttpComm = require("httplib.httpComm") +local pystring = require("common.pystring") +local psocket = require("posix.sys.socket") +local unistd = require("posix.unistd") +local bit = require("bit") +local fcntl = require("posix.fcntl") +local CasyncDns = require("httplib.asyncDns") + +local CasyncHttp = class("asyncHttp", ChttpComm) + +local ip_pattern = "(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d?)" + +local function match_ip(ip) + local d1, d2, d3, d4 = ip:match(ip_pattern) + if d1 and d2 and d3 and d4 then + local num1, num2, num3, num4 = tonumber(d1), tonumber(d2), tonumber(d3), tonumber(d4) + if num1 >= 0 and num1 <= 255 and num2 >= 0 and num2 <= 255 and num3 >= 0 and num3 <= 255 and num4 >= 0 and num4 <= 255 then + return true + end + end + return false +end + +local function getIp(domain) + local ip + if match_ip(domain) then + ip = domain + else + local dns = CasyncDns.new() + ip = dns:dns_lookup(domain) + end + return ip +end + +local function fdNonBlocking(fd) + local res + local flag, err, errno = fcntl.fcntl(fd, fcntl.F_GETFL) + if flag then + res, err, errno = fcntl.fcntl(fd, fcntl.F_SETFL, bit.bor(flag, fcntl.O_NONBLOCK)) + if res then + return + else + print(string.format("fcntl set failed, report:%d, %s", err, errno)) + return -1 + end + else + print(string.format("fcntl get failed, report:%d, %s", err, errno)) + return -1 + end +end + +local function setupFd() + local fd, err, errno + fd, err, errno = psocket.socket(psocket.AF_INET, psocket.SOCK_STREAM, 0) + if fd then -- for socket + if fdNonBlocking(fd) == -1 then + unistd.close(fd) + return -1 + end + return fd + else + print(string.format("socket create failed, report:%d, %s", err, errno)) + return -1 + end +end + +local function tryConnect(fd, tConn) + local res, err, errno + + res, err, errno = psocket.connect(fd, tConn) + if not res then + if errno == 115 then + return 1 + else + print(string.format("socket connect failed, report:%d, %s", err, errno)) + return + end + else + return 0 + end +end + +function CasyncHttp:_init_() + ChttpComm._init_(self) +end + +function CasyncHttp:pack(method, domain, uri, headers, body) + local line = self:packCliHead(method, uri) + headers.Host = domain + headers = self:packCliHeaders(headers, #body) + return pystring:join("\r\n", {line, headers, body}) +end + +local function waitDataRest(fread, rest, tReq) + local len = 0 + local tStream = {tReq.data} + local c = #tStream + while len < rest do + local s = fread() + if s then + len = len + #s + c = c + 1 + tStream[c] = s + else + return -1 + end + end + tReq.data = pystring:join("", tStream) + return 0 +end + +local function waitChuckData(fread, s, size) + while true do + if #s >= size + 2 then + return s + end + local add = fread() + if add then + s = s .. add + else + return nil + end + end +end + +local function waitChuckSize(fread, s) + while true do + if string.find(s, "\r\n") then + return s + end + local add = fread() + if add then + s = s .. add + else + return nil + end + end +end + +local function readChunks(fread, tReq) + local cells = {} + local s = tReq.data + local ssize, size + local len = 1 + local bodies, body + + while true do + if len == 0 then + break + end + s = waitChuckSize(fread, s) + if s then + size, s = unpack(pystring:split(s, "\r\n", 1)) + len = tonumber(size, 16) + bodies = waitChuckData(fread, s, len) + if bodies then + body = string.sub(bodies, 1, len) + s = string.sub(bodies, len + 2) + table.insert(cells, body) + else + return -2 + end + else + return -1 + end + end + tReq.data = pystring:join("", cells) + return 0 +end + +local function waitHttpRest(fread, tReq) + if tReq.header["content-length"] then + local lenData = #tReq.data + local lenInfo = tonumber(tReq.header["content-length"]) + + local rest = lenInfo - lenData + if rest > 10 * 1024 * 1024 then -- limit max data len + return -1 + end + + if waitDataRest(fread, rest, tReq) < 0 then + return -2 + end + else -- chunk mode + if #tReq.data > 0 then + if readChunks(fread, tReq) < 0 then + return -3 + end + end + end + return 0 +end + +local function waitHttpHead(fread) + local stream = "" + while true do + local s = fread() + if s then + stream = stream .. s + if string.find(stream, "\r\n\r\n") then + return stream + end + else + return nil + end + end +end + +function CasyncHttp:parse(fread, stream) + local tStatus = pystring:split(stream, "\r\n", 1) + if #tStatus < 2 then + print("bad stream format.") + return nil + end + + local stat, heads = unpack(tStatus) + local tStat = pystring:split(stat, " ") + if #tStat < 3 then + print("bad stat: "..stat) + return nil + end + + local vers, code, descr = unpack(tStat) + local tReq = { + vers = vers, + code = code, + descr = descr + } + + local tHead = pystring:split(heads, "\r\n\r\n", 1) + if #tHead < 2 then + print("bad head: " .. heads) + return nil + end + local headers, data = unpack(tHead) + local tHeader = pystring:split(headers, "\r\n") + local header = {} + for _, s in ipairs(tHeader) do + local tKv = pystring:split(s, ":", 1) + if #tKv < 2 then + print("bad head kv value: " .. s) + return nil + end + local k, v = unpack(tKv) + k = string.lower(k) + header[k] = pystring:lstrip(v) + end + tReq.header = header + tReq.data = data + if waitHttpRest(fread, tReq) < 0 then + return nil + end + return tReq +end + +function CasyncHttp:result(fread) + local stream = waitHttpHead(fread) + if stream == nil then -- read return stream or error code or nil + return nil + end + return self:parse(fread, stream) +end + +local function checkConnect(fd, connecting, toWake) + local res, msg + if connecting > 0 then + local e = coroutine.yield() + if e.ev_out > 0 then + g_lb:mod_fd(fd, 0) + else + res, msg = coroutine.resume(toWake, "connect failed.") + assert(res, msg) + return -1 + end + end +end + +function CasyncHttp:procStream(fd, stream, toWake) + local res, msg + res = g_lb:write(fd, stream) + if res then + local fread = g_lb:read(fd) + local tReq = self:result(fread) + res, msg = coroutine.resume(toWake, tReq.data) + assert(res, msg) + else + res, msg = coroutine.resume(toWake, "write failed.") + assert(res, msg) + end + g_lb:co_exit(fd) +end + +function CasyncHttp:_get(fd) + local toWake, domain, uri, headers, body, connecting = coroutine.yield() + + if checkConnect(fd, connecting, toWake) == -1 then + g_lb:co_exit(fd) + return + end + local stream = self:pack('GET', domain, uri, headers, body) + self:procStream(fd, stream, toWake) +end + +function CasyncHttp:connect(domain, uri, port, headers, body, cb) + port = port or 80 + local ip = getIp(domain) + if #ip == 0 then + return "request domain failed." + end + local tConn = {family=psocket.AF_INET, addr=ip, port=port} + + local fd = setupFd() + if fd > 0 then + local connect = tryConnect(fd, tConn) + if connect then + local res, msg + + local co = g_lb:co_add(fd, cb) + if connect > 0 then + g_lb:mod_fd(fd, 1) + end + res, msg = coroutine.resume(co, self, fd) + assert(res, msg) + res, msg = coroutine.resume(co, coroutine.running(), domain, uri, headers, body, connect) + assert(res, msg) + return coroutine.yield() + else + return "connect to domain failed." + end + end +end + +function CasyncHttp:get(domain, uri, port) + return self:connect(domain, uri, port, {}, "", self._get) +end + +function CasyncHttp:_put(fd) + local toWake, domain, uri, headers, body, connecting = coroutine.yield() + + if checkConnect(fd, connecting, toWake) == -1 then + g_lb:co_exit(fd) + return + end + local stream = self:pack('PUT', domain, uri, headers, body) + self:procStream(fd, stream, toWake) +end + +function CasyncHttp:put(domain, uri, headers, body, port) + return self:connect(domain, uri, port, headers, body, self._put) +end + +function CasyncHttp:_post(fd) + local toWake, domain, uri, headers, body, connecting = coroutine.yield() + + if checkConnect(fd, connecting, toWake) == -1 then + g_lb:co_exit(fd) + return + end + local stream = self:pack('POST', domain, uri, headers, body) + self:procStream(fd, stream, toWake) +end + +function CasyncHttp:post(domain, uri, headers, body, port) + return self:connect(domain, uri, port, headers, body, self._post) +end + +return CasyncHttp diff --git a/source/tools/monitor/unity/httplib/asyncOSS.lua b/source/tools/monitor/unity/httplib/asyncOSS.lua new file mode 100644 index 0000000000000000000000000000000000000000..b7aa9aa0976937d71f85bf493de0310eb24ef9d8 --- /dev/null +++ b/source/tools/monitor/unity/httplib/asyncOSS.lua @@ -0,0 +1,61 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 20:07 +--- + +require("common.class") +local system = require("common.system") +local pystring = require("common.pystring") +local sha1 = require("sha1") +local base64 = require("base64") +local CasyncHttp = require("httplib.asyncHttp") + +local CasyncOSS = class("asyncOSS", CasyncHttp) + +function CasyncOSS:_init_(res) + CasyncHttp._init_(self) + self._bucket = res.bucket + self._endPoint = res.endPoint + self._ak = base64.decode(res.ak) + self._sk = base64.decode(res.sk) +end + +function CasyncOSS:sign(cType, date, uri) + local ss = { + "PUT", -- VERB + "", -- md5 + cType, -- Content-Type + date, -- Date + uri, -- + } + local s = table.concat(ss, '\n') + return base64.encode(sha1.hmac_binary(self._sk, s)) +end + +function CasyncOSS:auth(cType, date, uri) + local ss = {"OSS ", self._ak, ":", self:sign(cType, date, uri)} + return table.concat(ss) +end + +function CasyncOSS:put(uuid, stream) + local bucket = self._bucket + local uri = os.date("/%Y/%m") .. "/" .. uuid + + local uris = table.concat({"/", bucket, uri}) + local date = system:timeRfc1123(os.time()) + local cType = "application/octet-stream" + local host = bucket .. "." .. self._endPoint + + local headers = { + Host = host, + ["Content-Type"] = cType, + ["Content-Length"] = #stream, + date = date, + authorization = self:auth(cType, date, uris) + } + + CasyncHttp.put(self, host, uri, headers, stream) +end + +return CasyncOSS \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/cmonCli.lua b/source/tools/monitor/unity/httplib/cmonCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..3f2101375cbb7504cf629754a77df762705becf8 --- /dev/null +++ b/source/tools/monitor/unity/httplib/cmonCli.lua @@ -0,0 +1,22 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/30 1:22 AM +--- + +require("common.class") + +local sha1 = require("sha1") +local lz4 = require("lz4") +local md5 = require("md5") +local base64 = require("base64") +local system = require("common.system") +local pystring = require("common.pystring") + +local ChttpCli = require("httplib.httpCli") +local CcmonCli = class("cmonCli", ChttpCli) + +function CcmonCli:_init_(cpath, proxy) + ChttpCli._init_(proxy) + +end \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/coCli.lua b/source/tools/monitor/unity/httplib/coCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..327c111df8961ac62ad55ca8f28ac2bc6c52b960 --- /dev/null +++ b/source/tools/monitor/unity/httplib/coCli.lua @@ -0,0 +1,222 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/3 17:46 +--- + +require("common.class") +local enumStat = require("httplib.enumStat") +local system = require("common.system") +local unistd = require("posix.unistd") +local CprotoData = require("common.protoData") +require("struct") +local CcoCli = class("coFrame") + +function CcoCli:_init_(fd) + self._bfd = fd + self._efd = self:_installFFI() + self._proto = CprotoData.new(nil) +end + +function CcoCli:_del_() + if self._efd then + self.cffi.deinit(self._efd) + end +end + +function CcoCli:_installFFI() + local ffi = require("beaver.native.beavercffi") + + self.ffi = ffi.ffi + self.cffi = ffi.cffi + + local efd = self.cffi.init(self._bfd) + assert(efd > 0) + return efd +end + +local socketWakeTbl = { + enumStat.connected, + enumStat.closed, +} + +function CcoCli:filterLines(cli, lines, body) + return cli:trans(lines, body, nil) +end + +function CcoCli:coQueFunc(cli, cffi, efd) + local body + local ok, msg + while true do + local lines = coroutine.yield() + local stat = cli.status + if lines and #lines.lines > 0 then + body = self:filterLines(cli, lines, body) + end + if body then --> has data to send. + if cli.co == nil and stat == enumStat.closed then -- not active + local co = coroutine.create(function(c, o, fd) cli.work(c, o, fd) end) + cli.co = co + ok, msg = coroutine.resume(co, cli, cffi, efd) + if not ok then + error(string.format("cli.co run failed %s", msg)) + end + elseif stat == enumStat.connected then --> already connected + ok, msg = coroutine.resume(cli.co, body) + if not ok then + error(string.format("cli.co run failed %s", msg)) + end + body = nil + else --> other stat, only record + ok = nil + --print("http stat:", stat, enumStat.connected) + end + end + end +end + +function CcoCli:checkOvertime(cli, co, ffi) + local ok, msg + if cli.status == enumStat.connecting and cli:checkTime() >= 2 then + local e = ffi.new("native_event_t") + e.fd = cli.fd + e.ev_close = 1 + ok, msg = coroutine.resume(cli.co, e) --> to stop cli connecting + if not ok then + error(string.format("cli.co run failed %s", msg)) + end + coroutine.resume(co, "") --> notify que + if not ok then + error(string.format("coOut run failed %s", msg)) + end + end +end + +local function err_fd(e) + if e.ev_close > 0 then + error("pipe out closed, process need stop.") + end +end + +local function pipe_error(err, errno) + error(string.format("read from pipe error: %s, %d", err, errno)) +end + +local function read_stream(fd) + local out + while true do + local e = coroutine.yield(out) + out = nil -- clear out + err_fd(e) + local size, err, errno = unistd.read(fd, 4) + if size then + local c = 0 + local len = struct.unpack(" 0 then + self:_pollFd(bfd, cli, nes, coIn, coOut) + else + self:checkOvertime(cli, coOut, ffi) + end + end +end + +function CcoCli:poll(cli) + local _, msg = pcall(self._poll, self, cli) + print(msg) + return 0 +end + +return CcoCli diff --git a/source/tools/monitor/unity/httplib/coHttpCli.lua b/source/tools/monitor/unity/httplib/coHttpCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..9718385d2e36764b01e00e2b55265005ecf7db89 --- /dev/null +++ b/source/tools/monitor/unity/httplib/coHttpCli.lua @@ -0,0 +1,542 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/3 11:42 +--- + +require("common.class") +local ChttpComm = require("httplib.httpComm") +local enumStat = require("httplib.enumStat") +local pystring = require("common.pystring") +local socket = require("posix.sys.socket") +local luaSocket = require("socket") +local unistd = require("posix.unistd") +local system = require("common.system") +local bit = require("bit") +local fcntl = require("posix.fcntl") + +local CcoHttpCli = class("coHttpCli", ChttpComm) + +local ip_pattern = "(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d?)" + +local function match_ip(ip) + local d1, d2, d3, d4 = ip:match(ip_pattern) + if d1 and d2 and d3 and d4 then + local num1, num2, num3, num4 = tonumber(d1), tonumber(d2), tonumber(d3), tonumber(d4) + if num1 >= 0 and num1 <= 255 and num2 >= 0 and num2 <= 255 and num3 >= 0 and num3 <= 255 and num4 >= 0 and num4 <= 255 then + return true + end + end + return false +end + +local function checkInt(errno, fd) + if errno == 4 then -- Interrupted system cal + if fd > 0 then + unistd.close(fd) + end + error("cli interrupt.") + end +end + +local function getIp(host) + local ip + if match_ip(host) then + ip = host + else + ip = luaSocket.dns.toip(host) + end + return ip +end + +local function setTimeOut(fd) + local ok, errmsg = socket.setsockopt(fd, socket.SOL_SOCKET, socket.SO_SNDTIMEO, 1, 0) + if ok then + return + else + print("set sock time out failed " .. errmsg) + return -1 + end +end + +local function fdNonBlocking(fd) + local res + local flag, err, errno = fcntl.fcntl(fd, fcntl.F_GETFL) + if flag then + res, err, errno = fcntl.fcntl(fd, fcntl.F_SETFL, bit.bor(flag, fcntl.O_NONBLOCK)) + if res then + return + else + checkInt(errno, fd) + print(string.format("fcntl set failed, report:%d, %s", err, errno)) + return -1 + end + else + checkInt(errno, fd) + print(string.format("fcntl get failed, report:%d, %s", err, errno)) + return -1 + end +end + +local function installFd(ip, port) + local fd, res, err, errno + fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + + if fd then -- for socket + if fdNonBlocking(fd) then + return + end + local tConn = {family=socket.AF_INET, addr=ip, port=port} + res, err, errno = socket.connect(fd, tConn) + if res then + return fd, res + elseif errno == 115 then -- in process + return fd, errno + else + checkInt(errno, fd) + unistd.close(fd) + print(string.format("socket connect failed, report:%d, %s", err, errno)) + end + else -- socket failed + checkInt(errno, 0) + print(string.format("socket create failed, report:%d, %s", err, errno)) + end +end + +local function setupSocket(host, port) + local ip = getIp(host) + if ip then + return installFd(ip, port) + else + print("get ip failed.") + return nil + end +end + +function CcoHttpCli:_init_(fYaml, persistent) + local res = system:parseYaml(fYaml) + local pushTo = res.pushTo + self._host = pushTo.host + self._port = pushTo.port or 80 + self._url = pushTo.url or "/" + self._persistent = persistent + + local Cidentity = require("beaver.identity") + local inst = Cidentity.new(fYaml) + self._instance = inst:id() + + self.status = enumStat.closed +end + +function CcoHttpCli:connect() + local fd, stat + fd, stat = setupSocket(self._host, self._port) + if fd then + self.fd = fd + if stat == 0 then + self.status = enumStat.connected + else + self.status = enumStat.connecting + end + return true + end + return false +end + +function CcoHttpCli:echo(tReq) + print("clid get " .. #tReq.data) +end + +function CcoHttpCli:trans(msgs, body, filter) + return "" +end + +function CcoHttpCli:addInstance(line) -- add instance id for line index. + local cells = line.ls + local hasInstance = false + + if cells then + for _, cell in ipairs(cells) do + if cell.name == "instance" then + hasInstance = true + end + end + end + + if not hasInstance then + local cell = { + name = "instance", + index = self._instance + } + if cells then + table.insert(cells, cell) + else + line.ls = {cell} + end + end +end + +function CcoHttpCli:pack(body) + local line = self:packCliHead('GET', self._url) + local head = { + Host = self._host, + ["Accept-Encoding"] = "null" + } + local heads = self:packCliHeaders(head) + return pystring:join("\r\n", {line, heads, ""}) +end + +function CcoHttpCli:waitConnected(cffi, efd) + local e + local res + local fd = self.fd + if self.status == enumStat.connecting then -- need wait until connected + res = cffi.mod_fd(efd, fd, 1) + checkInt(-res, fd) + if res < 0 then + print("mod_fd socket failed") + return false + end + + e = coroutine.yield() + if e.ev_close > 0 then + print("socket closed.") + return false + elseif e.ev_out > 0 then -- in flag + local val, err, errno = socket.getsockopt(fd, socket.SOL_SOCKET, socket.SO_ERROR) + if val == nil then + print(string.format("getsockopt failed, report:%d, %s", err, errno)) + return false + elseif val == 0 then + self.status = enumStat.connected + res = cffi.mod_fd(efd, fd, 0) -- back to wait mod, + checkInt(-res, fd) + if res < 0 then + print("mod_fd socket failed") + return false + end + return true + else + checkInt(errno, fd) + print("getsockopt value error.") + return false + end + end + end + return true +end + +function CcoHttpCli:exit(cffi, efd, fd) + local res = cffi.del_fd(efd, fd) -- remove for epoll + unistd.close(fd) -- closed + checkInt(-res, fd) + self.co = nil + self.fd = nil -- do not use any more +end + +local function waitDataRest(fread, rest, tReq) + local len = 0 + local tStream = {tReq.data} + local c = #tStream + while len < rest do + local s = fread() + if s then + len = len + #s + c = c + 1 + tStream[c] = s + else + return -1 + end + end + tReq.data = pystring:join("", tStream) + return 0 +end + +local function waitChuckData(fread, s, size) + while true do + if #s >= size + 2 then + return s + end + local add = fread() + if add then + s = s .. add + else + return nil + end + end +end + +local function waitChuckSize(fread, s) + while true do + if string.find(s, "\r\n") then + return s + end + local add = fread() + if add then + s = s .. add + else + return nil + end + end +end + +local function readChunks(fread, tReq) + local cells = {} + local s = tReq.data + local ssize, size + local len = 1 + local bodies, body + + while true do + if len == 0 then + break + end + s = waitChuckSize(fread, s) + if s then + size, s = unpack(pystring:split(s, "\r\n", 1)) + len = tonumber(size, 16) + bodies = waitChuckData(fread, s, len) + if bodies then + body = string.sub(bodies, 1, len) + s = string.sub(bodies, len + 2) + table.insert(cells, body) + else + return -2 + end + else + return -1 + end + end + tReq.data = pystring:join("", cells) + return 0 +end + +local function waitHttpRest(fread, tReq) + if tReq.header["content-length"] then + local lenData = #tReq.data + local lenInfo = tonumber(tReq.header["content-length"]) + print("len: " .. lenInfo) + + local rest = lenInfo - lenData + if rest > 10 * 1024 * 1024 then -- limit max data len + return -1 + end + + if waitDataRest(fread, rest, tReq) < 0 then + return -2 + end + else -- chunk mode + if #tReq.data > 0 then + if readChunks(fread, tReq) < 0 then + return -3 + end + end + end + return 0 +end + +local function waitHttpHead(fread) + local stream = "" + while true do + local s = fread() + if s then + stream = stream .. s + if string.find(stream, "\r\n\r\n") then + return stream + end + else + return nil + end + end +end + +function CcoHttpCli:parse(fread, stream) + local tStatus = pystring:split(stream, "\r\n", 1) + if #tStatus < 2 then + print("bad stream format.") + return nil + end + + local stat, heads = unpack(tStatus) + local tStat = pystring:split(stat, " ") + if #tStat < 3 then + print("bad stat: "..stat) + return nil + end + + local vers, code, descr = unpack(tStat) + local tReq = { + vers = vers, + code = code, + descr = descr + } + + local tHead = pystring:split(heads, "\r\n\r\n", 1) + if #tHead < 2 then + print("bad head: " .. heads) + return nil + end + local headers, data = unpack(tHead) + local tHeader = pystring:split(headers, "\r\n") + local header = {} + for _, s in ipairs(tHeader) do + local tKv = pystring:split(s, ":", 1) + if #tKv < 2 then + print("bad head kv value: " .. s) + return nil + end + local k, v = unpack(tKv) + k = string.lower(k) + header[k] = pystring:lstrip(v) + end + tReq.header = header + tReq.data = data + if waitHttpRest(fread, tReq) < 0 then + return nil + end + return tReq +end + +function CcoHttpCli:result(fread) + local stream = waitHttpHead(fread) + if stream == nil then -- read return stream or error code or nil + return nil + end + return self:parse(fread, stream) +end + +function CcoHttpCli:closureRead(fd, maxLen) + maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max + local function readFd() + local e = coroutine.yield() + if e.ev_close > 0 then + return nil + elseif e.ev_in > 0 then + local s, err, errno + s, err, errno = socket.recv(fd, maxLen) + if s then + if #s > 0 then + maxLen = maxLen - #s + return s + else + return nil + end + else + checkInt(errno, fd) + print(string.format("socket recv failed, report:%d, %s", err, errno)) + return nil + end + else + system:dumps(e) + end + return nil + end + return readFd +end + +function CcoHttpCli:coWrite(cffi, efd, fd, stream) + local sent, err, errno + local res + + sent, err, errno = socket.send(fd, stream) + if sent ~= nil then + if sent < #stream then -- send buffer may full + res = cffi.mod_fd(efd, fd, 1) -- epoll write ev + checkInt(-res, fd) + if res < 0 then + print("mod_fd socket failed") + return false + end + + while sent < #stream do + print("send.", sent, #stream) + local e = coroutine.yield() + if e.ev_close > 0 then + return false + elseif e.ev_out then + stream = string.sub(stream, sent + 1) + if stream == nil then + return true + end + sent, err, errno = socket.send(fd, stream) + if sent == nil then + if errno == 11 then -- EAGAIN ? + goto continue + end + checkInt(errno, fd) + print(string.format("socket send failed, report:%d, %s", err, errno)) + return false + end + else -- need to read ? may something error or closed. + return false + end + ::continue:: + end + res = cffi.mod_fd(efd, fd, 0) -- epoll read ev only + checkInt(-res, fd) + if res < 0 then + print("mod_fd socket failed") + return false + end + end + return true + else + checkInt(errno, fd) + print(string.format("socket send failed, report:%d, %s", err, errno)) + return false + end +end + +function CcoHttpCli:checkTime() + return os.time() - self.online +end + +function CcoHttpCli:work(cffi, efd) + self.online = os.time() + + if not self:connect() then -- need to connect + return + end + + local fd = self.fd + + local res = cffi.add_fd(efd, fd) + checkInt(-res, fd) + if res < 0 then + print("add fd to epoll failed.") + goto failed + end + + if not self:waitConnected(cffi, efd) then + goto failed + end + + self.online = os.time() + while true do + local body = coroutine.yield() + self.status = enumStat.sending + local s = self:pack(body) + if not self:coWrite(cffi, efd, fd, s) then + goto failed + end + self.status = enumStat.receiving + local fread = self:closureRead(fd) + local tReq = self:result(fread) + if tReq then + self.status = enumStat.connected + self:echo(tReq) + else + goto failed + end + if not self._persistent then + goto failed + end + end + + ::failed:: do + self.status = enumStat.closed + self:exit(cffi, efd, fd) + end +end + +return CcoHttpCli diff --git a/source/tools/monitor/unity/httplib/coInflux.lua b/source/tools/monitor/unity/httplib/coInflux.lua new file mode 100644 index 0000000000000000000000000000000000000000..6b00a4c930c8b53f66d0f86efb6d90213545ac9f --- /dev/null +++ b/source/tools/monitor/unity/httplib/coInflux.lua @@ -0,0 +1,56 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/5 23:49 +--- + +require("common.class") +local CcoHttpCli = require("httplib.coHttpCli") +local pystring = require("common.pystring") +local lineParse = require("common.lineParse") + +local CcoInflux = class("coInflux", CcoHttpCli) + +function CcoInflux:_init_(fYaml) + CcoHttpCli._init_(self, fYaml) +end + +function CcoInflux:echo(tReq) + if tReq.code ~= "204" then + print(tReq.code, tReq.data) + end +end + +function CcoInflux:trans(msgs, body, filter) + local res + local c = 0 + local lines + local bodies = {} + + lines = msgs.lines + for _, line in ipairs(lines) do + c = c + 1 + self:addInstance(line) + bodies[c] = lineParse.packs(line) + end + + res = pystring:join("\n", bodies) + if body and #body > 0 then + return pystring:join("\n", {body, res}) + else + return res + end +end + +function CcoInflux:pack(body) + local line = self:packCliHead('POST', self._url) + local head = { + Host = self._host, + ["Content-Type"] = "text/plain", + ["Content-Length"] = #body, + } + local heads = self:packCliHeaders(head) + return pystring:join("\r\n", {line, heads, body}) +end + +return CcoInflux diff --git a/source/tools/monitor/unity/httplib/enumStat.lua b/source/tools/monitor/unity/httplib/enumStat.lua new file mode 100644 index 0000000000000000000000000000000000000000..7e0a54d7babed69a6f2bf7ef3edca8b85eed363e --- /dev/null +++ b/source/tools/monitor/unity/httplib/enumStat.lua @@ -0,0 +1,17 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/4 17:15 +--- + +local system = require("common.system") + +local socketStatus = { + "connecting", + "connected", + "sending", + "receiving", + "closed" +} + +return system:Enum(socketStatus) diff --git a/source/tools/monitor/unity/httplib/httpApp.lua b/source/tools/monitor/unity/httplib/httpApp.lua index 42995cae63540d25163f542f42b92358e4871e75..996792efbde2ccbd057012e05c185ebf672f5c5e 100644 --- a/source/tools/monitor/unity/httplib/httpApp.lua +++ b/source/tools/monitor/unity/httplib/httpApp.lua @@ -14,14 +14,15 @@ function ChttpApp:_init_(frame) ChttpBase._init_(self) end -function ChttpApp:echo(tRet, keep) - local stat = self:packStat(200) +function ChttpApp:echo(tRet, keep, code) + code = code or 200 + local stat = self:packStat(code) local tHead = { ["Content-Type"] = "application/json", ["Connection"] = (keep and "keep-alive") or "close" } local body = self:jencode(tRet) - local headers = self:packHeaders(tHead, #body) + local headers = self:packServerHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 3ec1ff4f4c09243a7b1965358e4816db35e307cf..0de7ba31a391a9b5508b605da98d70a1da72d070 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -25,7 +25,7 @@ function ChttpBase:_installRe(path, frame) frame:registerRe(path, self) end -function ChttpBase:echo(tRet, keep) +function ChttpBase:echo(tRet, keep, code) error("ChttpBase:echo is a virtual function.") end @@ -48,8 +48,8 @@ end function ChttpBase:call(tReq) local keep = checkKeep(tReq) - local tRet = self._urlCb[tReq.path](tReq) - local res = self:echo(tRet, keep) + local tRet, code = self._urlCb[tReq.path](tReq) + local res = self:echo(tRet, keep, code) return res, keep end diff --git a/source/tools/monitor/unity/httplib/httpCli.lua b/source/tools/monitor/unity/httplib/httpCli.lua index c83888f7f7055f37379ea9ad3241f7e0db02a9af..5b25cf189f1a0400fa4186eeb31b463c4ff67b5f 100644 --- a/source/tools/monitor/unity/httplib/httpCli.lua +++ b/source/tools/monitor/unity/httplib/httpCli.lua @@ -31,6 +31,27 @@ function ChttpCli:get(Url) } end +function ChttpCli:put(Url, stream, header) + local headers = header or { Connection = 'close' } + local source = self._ltn12.source.string(stream) + local t = {} + local res, code, head = self._http.request{ + url = Url, + method = "PUT", + headers = headers, + source = source, + proxy = self._proxy, + sink = self._ltn12.sink.table(t) + } + local body = table.concat(t) + return { + res = res, + code = code, + head = head, + body = body + } +end + function ChttpCli:post(Url, reqs, header) local headers = header or { Connection = 'close' } local source = self._ltn12.source.string(reqs) @@ -61,4 +82,12 @@ function ChttpCli:postTable(Url, t) return self:post(Url, req, headers) end +function ChttpCli:postLine(Url, line) + local headers = { + ["Content-Type"] = "text/plain", + ["Content-Length"] = #line, + } + return self:post(Url, line, headers) +end + return ChttpCli diff --git a/source/tools/monitor/unity/httplib/httpComm.lua b/source/tools/monitor/unity/httplib/httpComm.lua index 3fdff02b5ebb8c3e6c5a8fa3b63737c28c6852b3..436f6ef54c54af2611a42857b93e8986864129bf 100644 --- a/source/tools/monitor/unity/httplib/httpComm.lua +++ b/source/tools/monitor/unity/httplib/httpComm.lua @@ -4,6 +4,8 @@ --- DateTime: 2022/12/19 10:46 PM --- +--- refer to https://blog.csdn.net/a19881029/article/details/14002273 + require("common.class") local pystring = require("common.pystring") local sockerUrl = require("socket.url") @@ -13,6 +15,8 @@ local ChttpComm = class("httplib.httpComm") local cjson = require("cjson.safe") local json = cjson.new() +json.encode_escape_forward_slash(false) + local function codeTable() return { [100] = "Continue", @@ -73,19 +77,19 @@ function ChttpComm:parsePath(path) return parseParams(res) end -local function originHeader() +local function originServerHeader() return { - server = "beaver/0.0.2", + server = "beaver/0.0.3", date = os.date("%a, %d %b %Y %H:%M:%S %Z", os.time()), } end -function ChttpComm:packHeaders(headTable, len) -- just for http out. +function ChttpComm:packServerHeaders(headTable, len) -- just for http out. local lines = {} if not headTable["Content-Length"] then headTable["Content-Length"] = len end - local origin = originHeader() + local origin = originServerHeader() local c = 0 for k, v in pairs(origin) do @@ -97,13 +101,52 @@ function ChttpComm:packHeaders(headTable, len) -- just for http out. c = c + 1 lines[c] = table.concat({k, v}, ": ") end - return pystring:join("\r\n", lines) .. "\r\n" + + c = c + 1 + lines[c] = "" + return pystring:join("\r\n", lines) end local codeStrTable = codeTable() -function ChttpComm:packStat(code) +function ChttpComm:packStat(code) -- only for server. local t = {"HTTP/1.1", code, codeStrTable[code]} return pystring:join(" ", t) end +local function originCliHeader() + return { + ["User-Agent"] = "beaverCli/0.0.2", + Connection = "Keep-Alive", + } +end + +function ChttpComm:packCliHeaders(headTable, len) + len = len or 0 + local lines = {} + if not headTable["Content-Length"] and len > 0 then + headTable["Content-Length"] = len + end + local origin = originCliHeader() + + local c = 0 + for k, v in pairs(origin) do + c = c + 1 + lines[c] = table.concat({k, v}, ": ") + end + + for k, v in pairs(headTable) do + c = c + 1 + lines[c] = table.concat({k, v}, ": ") + end + + c = c + 1 + lines[c] = "" + return pystring:join("\r\n", lines) +end + +function ChttpComm:packCliHead(method, url) + local t = {method, url, "HTTP/1.1"} + return pystring:join(" ", t) +end + return ChttpComm diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index 91db3e8824d05686f379d7e9166c9eca2d342e97..3d63de08415c7773bb29906aeb9673ff83fbbb67 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -100,22 +100,23 @@ local function htmlPack(title, content) return pystring:join("", bodies) end -function ChttpHtml:pack(cType, keep, body) - local stat = self:packStat(200) +function ChttpHtml:pack(cType, keep, body, code) + code = code or 200 + local stat = self:packStat(code) local tHead = { ["Content-Type"] = cType, ["Connection"] = (keep and "keep-alive") or "close" } - local headers = self:packHeaders(tHead, #body) + local headers = self:packServerHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end -function ChttpHtml:echo(tRet, keep) +function ChttpHtml:echo(tRet, keep, code) local cType = tRet.type or "text/html" local body = htmlPack(tRet.title, tRet.content) - return self:pack(cType, keep, body) + return self:pack(cType, keep, body, code) end return ChttpHtml diff --git a/source/tools/monitor/unity/httplib/httpPlain.lua b/source/tools/monitor/unity/httplib/httpPlain.lua index 81a6b39dad355f430c12345ff4b5452b612e0d90..fc0f075e66d1e03a2b6f805cbd38babd3aaffe19 100644 --- a/source/tools/monitor/unity/httplib/httpPlain.lua +++ b/source/tools/monitor/unity/httplib/httpPlain.lua @@ -14,14 +14,15 @@ function ChttpPlain:_init_(frame) ChttpBase._init_(self) end -function ChttpPlain:echo(tRet, keep) - local stat = self:packStat(200) +function ChttpPlain:echo(tRet, keep, code) + code = code or 200 + local stat = self:packStat(code) local tHead = { ["Content-Type"] = "text/plain", ["Connection"] = (keep and "keep-alive") or "close" } local body = tRet.text - local headers = self:packHeaders(tHead, #body) + local headers = self:packServerHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end diff --git a/source/tools/monitor/unity/httplib/influxCli.lua b/source/tools/monitor/unity/httplib/influxCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..a1eef70cf77b2fa17ee77dd36bf398eb6d36201b --- /dev/null +++ b/source/tools/monitor/unity/httplib/influxCli.lua @@ -0,0 +1,25 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/30 1:28 AM +--- + +require("common.class") +local ChttpCli = require("httplib.httpCli") + +local CinfluxCli = class("influxCli", ChttpCli) + +function CinfluxCli:_init_(url, proxy) + self._url = url + ChttpCli._init_(self, proxy) +end + +function CinfluxCli:puts(s) + local headers = { + ["Content-Type"] = "text/plain", + ["Content-Length"] = #s, + } + return self:post(self._url, s, headers) +end + +return CinfluxCli diff --git a/source/tools/monitor/unity/httplib/ossCli.lua b/source/tools/monitor/unity/httplib/ossCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..e3a436a61226ff2707fb3a2cf4e88737725d4253 --- /dev/null +++ b/source/tools/monitor/unity/httplib/ossCli.lua @@ -0,0 +1,62 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/23 20:43 +--- + +local sha1 = require("sha1") +local md5 = require("md5") +local base64 = require("base64") +local system = require("common.system") +local pystring = require("common.pystring") + +local ChttpCli = require("httplib.httpCli") +local CossCli = class("ossCli", ChttpCli) + +function CossCli:_init_(endPoint, bucket, ak, sk) + ChttpCli._init_(self) + self._endPoint = endPoint + self._bucket = bucket + self._ak = ak + self._sk = sk +end + +function CossCli:sign(cType, date, uri) + local ss = { + "PUT", -- VERB + "", -- md5 + cType, -- Content-Type + date, -- Date + uri, -- + } + local s = table.concat(ss, '\n') + return base64.encode(sha1.hmac_binary(self._sk, s)) +end + +function CossCli:auth(cType, date, uri) + local ss = {"OSS ", self._ak, ":", self:sign(cType, date, uri)} + return table.concat(ss) +end + +function CossCli:put(uri, stream, cType) + local bucket = self._bucket + local uris = table.concat({"/", bucket, uri}) + cType = cType or "application/octet-stream" + + local host = bucket .. "." .. self._endPoint + local date = system:timeRfc1123(os.time()) + local header = { + Host = host, + ["Content-Type"] = cType, + ["Content-Length"] = #stream, + date = date, + authorization = self:auth(cType, date, uris) + } + print(header.authorization) + local url = table.concat({"http://", self._endPoint, uri}) + print(url) + + return ChttpCli.put(self, url, stream, header) +end + +return CossCli diff --git a/source/tools/monitor/unity/test/bees/run.sh b/source/tools/monitor/unity/test/bees/run.sh index 37055941e5db172dcf2f65bcca4b52416404d7e1..29f6a84eb19d3d35020ce8751d47dfb898c6351e 100755 --- a/source/tools/monitor/unity/test/bees/run.sh +++ b/source/tools/monitor/unity/test/bees/run.sh @@ -13,7 +13,6 @@ cd ../../beeQ || exit 1 [ ! -d ../lib ] && mkdir ../lib cp ./lib/*.so ../lib cp ../tsdb/native/*.so ../lib -rm -rf ../bin [ ! -d ../bin ] && mkdir ../bin cp unity-mon ../bin diff --git a/source/tools/monitor/unity/test/curl/diag/ioHang.py b/source/tools/monitor/unity/test/curl/diag/ioHang.py new file mode 100644 index 0000000000000000000000000000000000000000..12e762287e72c3968f069ae7ad17efa9b52ecd72 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/diag/ioHang.py @@ -0,0 +1,8 @@ +import requests +import json +import uuid + +url = "http://127.0.0.1:8400/api/trig" +vs = {"cmd": "diag", "exec": "io_hang", "args": ["hangdetect", "vda"], "uid": str(uuid.uuid4())} +res = requests.post(url, json=vs) +print(res) diff --git a/source/tools/monitor/unity/test/curl/diag/jRuntime.py b/source/tools/monitor/unity/test/curl/diag/jRuntime.py new file mode 100644 index 0000000000000000000000000000000000000000..ece07f9c0cea036fa4905e326ba81a0ce2a66e36 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/diag/jRuntime.py @@ -0,0 +1,8 @@ +import requests +import json +import uuid + +url = "http://127.0.0.1:8400/api/trig" +vs = {"cmd": "diag", "exec": "jruntime", "args": ["-d", "5", "--top", "2"], "uid": str(uuid.uuid4())} +res = requests.post(url, json=vs) +print(res) diff --git a/source/tools/monitor/unity/test/curl/diag/netCli.py b/source/tools/monitor/unity/test/curl/diag/netCli.py new file mode 100644 index 0000000000000000000000000000000000000000..16ab2153b0cc0e5a633dabe9975e0d554ff450d5 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/diag/netCli.py @@ -0,0 +1,8 @@ +import requests +import json +import uuid + +url = "http://127.0.0.1:8400/api/trig" +vs = {"cmd": "diag", "exec": "net_edge", "args": ["11.0.145.174", "host", "tcp port 80"], "uid": str(uuid.uuid4())} +res = requests.post(url, json=vs) +print(res) diff --git a/source/tools/monitor/unity/test/curl/dns/base.lua b/source/tools/monitor/unity/test/curl/dns/base.lua new file mode 100644 index 0000000000000000000000000000000000000000..8dc44f1a0bf4e917ddc938520af5c6b9b9e46448 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/dns/base.lua @@ -0,0 +1,76 @@ +package.path = package.path .. ";../../../?.lua;" +local socket = require("socket") +local pystring = require("common.pystring") +local system = require("common.system") + +local function lookup_server() + local f = io.open("/etc/resolv.conf") + local server = "" + for line in f:lines() do + if pystring:startswith(line, "nameserver") then + local res = pystring:split(line) + server = res[2] + break + end + end + + f:close() + return server +end + +function dns_lookup(domain_name) + local udp = socket.udp() + local dst = lookup_server() + print(dst) + udp:setpeername(dst, 53) + udp:settimeout(5) + + local cnt = 0 + local queries = {} + local head = string.char( + 0x12, 0x34, -- Query ID + 0x01, 0x00, -- Standard query + 0x00, 0x01, -- Number of questions + 0x00, 0x00, -- Number of answers + 0x00, 0x00, -- Number of authority records + 0x00, 0x00 -- Number of additional records + ) + cnt = cnt + 1 + queries[cnt] = head + + local names = pystring:split(domain_name, ".") + for _, name in ipairs(names) do + cnt = cnt + 1 + queries[cnt] = string.char(string.len(name)) + cnt = cnt + 1 + queries[cnt] = name + end + cnt = cnt + 1 + local tail = string.char( + 0x00, -- End of domain name + 0x00, 0x01, -- Type A record + 0x00, 0x01 -- Class IN + ) + queries[cnt] = tail + + local query = table.concat(queries) + system:hexdump(query) + udp:send(query) + local response = udp:receive() + system:hexdump(response) + udp:close() + if response then + local ip_address = string.format("%d.%d.%d.%d", string.byte(response, -4, -1)) + return ip_address + else + return nil + end +end + +local domain_name = "www.baidu.com" +local ip_address = dns_lookup(domain_name) +if ip_address then + print(string.format("%s -> %s", domain_name, ip_address)) +else + print(string.format("Failed to resolve %s", domain_name)) +end \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/dns/base.py b/source/tools/monitor/unity/test/curl/dns/base.py new file mode 100644 index 0000000000000000000000000000000000000000..947979318582bca6ac9b0e816071be4096c28dcc --- /dev/null +++ b/source/tools/monitor/unity/test/curl/dns/base.py @@ -0,0 +1,38 @@ +import socket + + +def dns_lookup(domain_name): + dns_server = "100.100.2.136" + dns_port = 53 + query = bytearray() + query += bytearray.fromhex("AA AA") # Query ID + query += bytearray.fromhex("01 00") # Flags + query += bytearray.fromhex("00 01") # Questions + query += bytearray.fromhex("00 00") # Answers + query += bytearray.fromhex("00 00") # Authority records + query += bytearray.fromhex("00 00") # Additional records + for part in domain_name.split("."): + query += bytes([len(part)]) + query += part.encode() + query += bytearray.fromhex("00") # End of domain name + query += bytearray.fromhex("00 01") # Type A record + query += bytearray.fromhex("00 01") # Class IN + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(query, (dns_server, dns_port)) + response, _ = sock.recvfrom(1024) + sock.close() + + if response: + ip_address = ".".join(str(byte) for byte in response[-4:]) + return ip_address + else: + return None + + +domain_name = "www.baidu.com" +ip_address = dns_lookup(domain_name) +if ip_address: + print(f"{domain_name} -> {ip_address}") +else: + print(f"Failed to resolve {domain_name}") \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/dns/psock.lua b/source/tools/monitor/unity/test/curl/dns/psock.lua new file mode 100644 index 0000000000000000000000000000000000000000..460ac77a09e9bdacbee1b56a7f81497892c3faf3 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/dns/psock.lua @@ -0,0 +1,80 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 11:21 +--- + +package.path = package.path .. ";../../../?.lua;" +local psocket = require("posix.sys.socket") +local unistd = require("posix.unistd") +local pystring = require("common.pystring") +local system = require("common.system") + +local function lookup_server() + local f = io.open("/etc/resolv.conf") + local server = "" + for line in f:lines() do + if pystring:startswith(line, "nameserver") then + local res = pystring:split(line) + server = res[2] + break + end + end + + f:close() + return server +end + +function dns_lookup(domain_name) + local fd = psocket.socket(psocket.AF_INET, psocket.SOCK_DGRAM, 0) + local dst = lookup_server() + local tDist = {family=psocket.AF_INET, addr=dst, port=53} + + local cnt = 0 + local queries = {} + local head = string.char( + 0x12, 0x34, -- Query ID + 0x01, 0x00, -- Standard query + 0x00, 0x01, -- Number of questions + 0x00, 0x00, -- Number of answers + 0x00, 0x00, -- Number of authority records + 0x00, 0x00 -- Number of additional records + ) + cnt = cnt + 1 + queries[cnt] = head + + local names = pystring:split(domain_name, ".") + for _, name in ipairs(names) do + cnt = cnt + 1 + queries[cnt] = string.char(string.len(name)) + cnt = cnt + 1 + queries[cnt] = name + end + cnt = cnt + 1 + local tail = string.char( + 0x00, -- End of domain name + 0x00, 0x01, -- Type A record + 0x00, 0x01 -- Class IN + ) + queries[cnt] = tail + + local query = table.concat(queries) + psocket.sendto(fd, query, tDist) + local response = psocket.recvfrom(fd, 512) + system:hexdump(response) + unistd.close(fd) + if response then + local ip_address = string.format("%d.%d.%d.%d", string.byte(response, -4, -1)) + return ip_address + else + return nil + end +end + +local domain_name = "www.baidu.com" +local ip_address = dns_lookup(domain_name) +if ip_address then + print(string.format("%s -> %s", domain_name, ip_address)) +else + print(string.format("Failed to resolve %s", domain_name)) +end \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/forkRun.py b/source/tools/monitor/unity/test/curl/forkRun.py new file mode 100644 index 0000000000000000000000000000000000000000..7ab0ceb8032c5f36c030c64107dc3acba27e238c --- /dev/null +++ b/source/tools/monitor/unity/test/curl/forkRun.py @@ -0,0 +1,17 @@ + +import time +from outLine import CnfPut + + +def loop(nf): + i = 1 + while True: + nf.puts('forkRun,mode=java log="hello runtime.",count=%d' % i) + i += 1 + time.sleep(15) + + +if __name__ == "__main__": + time.sleep(2) + nf = CnfPut() + loop(nf) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/getBtf.lua b/source/tools/monitor/unity/test/curl/getBtf.lua new file mode 100644 index 0000000000000000000000000000000000000000..8e0afd0136833f2677e8202a9acb5c0a1671bb01 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/getBtf.lua @@ -0,0 +1,11 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/26 14:13 +--- + +package.path = package.path .. ";../../?.lua;" + +local CbtfLoader = require("collector.btfLoader") + +local btf = CbtfLoader.new(".") diff --git a/source/tools/monitor/unity/test/curl/hello.py b/source/tools/monitor/unity/test/curl/hello.py new file mode 100644 index 0000000000000000000000000000000000000000..c497ca2f47acc5395f3ad944dd7eb6a028e1f034 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/hello.py @@ -0,0 +1,7 @@ +import time +import os + +print(os.getcwd()) +while True: + print("hello.") + time.sleep(3) diff --git a/source/tools/monitor/unity/test/curl/influx.lua b/source/tools/monitor/unity/test/curl/influx.lua new file mode 100644 index 0000000000000000000000000000000000000000..188992c461a80b6fec5b3641848abbcc9a37ba20 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/influx.lua @@ -0,0 +1,13 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/31 01:30 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local CinfluxCli = require("httplib.influxCli") + +local cli = CinfluxCli.new("http://ld-wz9d17b514mg6kjkx-proxy-tsdb.lindorm.rds.aliyuncs.com:8242/api/v2/write?db=lua") +local res = cli:puts('virtout,instance=self log="task:0(swapper/13), cpu:13, delayed:162994631, callstack:mwait_idle_with_hints.constprop.0,intel_idle,cpuidle_enter_state,cpuidle_enter,cpuidle_idle_call,do_idle,cpu_startup_entry,secondary_startup_64_no_verify,"') +system:dumps(res) diff --git a/source/tools/monitor/unity/test/curl/ossTest.lua b/source/tools/monitor/unity/test/curl/ossTest.lua new file mode 100644 index 0000000000000000000000000000000000000000..7d5c537209eae377263695d42aa91bc3719f69d3 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/ossTest.lua @@ -0,0 +1,25 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 00:12 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local CossCli = require("httplib.ossCli") + +local function generate_string(size) + local str = "" + local i = 0 + local s = "111111111111111111111111111111112222222222222222222333333333333cccc344444444445" + while i < size do + i = i + #s + str = str .. s + end + return str +end + +local s = generate_string( 1024 * 1024) +local c = CossCli.new("oss-cn-shenzhen.aliyuncs.com", "netinfo-shenzhen", + "ak", "sk") +system:dumps(c:put("/long.txt", s)) diff --git a/source/tools/monitor/unity/test/curl/outLine.py b/source/tools/monitor/unity/test/curl/outLine.py new file mode 100644 index 0000000000000000000000000000000000000000..cd5ad58cfb4341acf3b186158c1bd021d84e6dad --- /dev/null +++ b/source/tools/monitor/unity/test/curl/outLine.py @@ -0,0 +1,23 @@ +import os +import socket + +PIPE_PATH = "/var/sysom/outline" +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self, path=PIPE_PATH): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = path + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check Netinfo is running.") + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut() + nf.puts('runtime,mode=java log="hello runtime."') diff --git a/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua b/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua index e44e05358d81a7e698a8f20531bab507a7b374a4..137c9a0dcac878bdedc426f97c4e07e52f717318 100644 --- a/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua +++ b/source/tools/monitor/unity/test/curl/poBeaver/poBeaver.lua @@ -63,22 +63,6 @@ function CpoBeaver:_install_fd(port, ip, backlog) end end -local function fdNonBlocking(fd) - local res, err, errno - local flag, err, errno = fcntl.fcntl(fd, fcntl.F_GETFL) - if flag then - res, err, errno = fcntl.fcntl(fd, fcntl.F_SETFL, bit.bor(flag, fcntl.O_NONBLOCK)) - if res then - return 0 - else - posixError("fcntl set failed", err, errno) - end - else - posixError("fcntl get failed", err, errno) - end -end - - function CpoBeaver:read(fd, maxLen) maxLen = maxLen or 1 * 1024 * 1024 -- signal conversation accept 1M stream max local function readFd() @@ -203,7 +187,7 @@ function CpoBeaver:accept(fd, e) elseif e.IN then local nfd, err, errno = socket.accept(fd) if nfd then - fdNonBlocking(nfd) + system:fdNonBlocking(nfd) self:co_add(nfd) else posixError("accept new socket failed", err, errno) diff --git a/source/tools/monitor/unity/test/curl/postCmd.lua b/source/tools/monitor/unity/test/curl/postCmd.lua new file mode 100644 index 0000000000000000000000000000000000000000..3bbf577a92878e926043ee0b9c53d5bef5b4a0bd --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postCmd.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/17 19:55 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/trig" +local req = {cmd = "exec", exec = "/usr/bin/pwd"} +local res = cli:postTable(url, req) +system:dumps(res) diff --git a/source/tools/monitor/unity/test/curl/postDiag.lua b/source/tools/monitor/unity/test/curl/postDiag.lua new file mode 100644 index 0000000000000000000000000000000000000000..8082eb06e1d81abf5c646b131a48ff2ec09b7a00 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postDiag.lua @@ -0,0 +1,16 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/4 13:21 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/trig" +--local req = {cmd = "diag", exec = "io_hang", args = {"hangdetect", "vda"}} +local req = {cmd = "diag", exec = "io_hang", args = {"hangdetect", "vda"}, uid = system:guid()} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/postJRuntime.lua b/source/tools/monitor/unity/test/curl/postJRuntime.lua new file mode 100644 index 0000000000000000000000000000000000000000..301fd11f2369929add40b13eb972dd2e130c834c --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postJRuntime.lua @@ -0,0 +1,16 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/23 00:11 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/trig" +--local req = {cmd = "diag", exec = "io_hang", args = {"hangdetect", "vda"}} +local req = {cmd = "diag", exec = "jruntime", args = {"-d", "5", "--top", "2"}, uid = system:guid()} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/postLine.lua b/source/tools/monitor/unity/test/curl/postLine.lua new file mode 100644 index 0000000000000000000000000000000000000000..e04791a91168261ab57b404481b8610bdf96c1e8 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postLine.lua @@ -0,0 +1,14 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/24 02:21 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/line" +local res = cli:postLine(url, "lineTable,index=abc value=1") +system:dumps(res) diff --git a/source/tools/monitor/unity/test/curl/postLine.py b/source/tools/monitor/unity/test/curl/postLine.py new file mode 100644 index 0000000000000000000000000000000000000000..9885d89c09d751534b74a3c390f3e4c159b22b92 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postLine.py @@ -0,0 +1,6 @@ +import requests + +url = "http://127.0.0.1:8400/api/line" +line = "lineTable,index=abc value=2" +res = requests.post(url, data=line) +print(res) diff --git a/source/tools/monitor/unity/test/curl/postNet.lua b/source/tools/monitor/unity/test/curl/postNet.lua new file mode 100644 index 0000000000000000000000000000000000000000..85b74a6aea3d3385653a0184792e14a6e50d96d1 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postNet.lua @@ -0,0 +1,16 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/22 17:00 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/trig" +--local req = {cmd = "diag", exec = "io_hang", args = {"hangdetect", "vda"}} +local req = {cmd = "diag", exec = "net_edge", args = {"172.16.0.118", "host", "tcp port 80"}, uid = system:guid()} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/postOSS.lua b/source/tools/monitor/unity/test/curl/postOSS.lua new file mode 100644 index 0000000000000000000000000000000000000000..8d37e291ee5d97335d751b9170b9abc775b1e15c --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postOSS.lua @@ -0,0 +1,18 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 22:12 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/oss" +local file = io.open("test.bin", "rb") +local content = file:read("*all") +file:close() +local req = {stream = content, uuid = system:guid()} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/postPy.lua b/source/tools/monitor/unity/test/curl/postPy.lua new file mode 100644 index 0000000000000000000000000000000000000000..d4085f0a4a8bfd1571d56be462e157d70204046a --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postPy.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/17 20:21 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/trig" +local req = {cmd = "exec", exec = "/usr/bin/python", args = {"../test/curl/hello.py",}, } +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/postQueTest.lua b/source/tools/monitor/unity/test/curl/postQueTest.lua new file mode 100644 index 0000000000000000000000000000000000000000..26e373279aee76b975422f021e5731a70f2a5f08 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/postQueTest.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/24 11:52 PM +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/que" +local req = {cmd = "mon_pid", pid = 15789, loop = 3} +local res = cli:postTable(url, req) +system:dumps(res) diff --git a/source/tools/monitor/unity/test/curl/reqdomain.lua b/source/tools/monitor/unity/test/curl/reqdomain.lua new file mode 100644 index 0000000000000000000000000000000000000000..22504856e3273f8da1d295d13e3c16d156f3fd01 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/reqdomain.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 14:15 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/dns" +local req = {domain = "cn.bing.com"} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/reqproxy.lua b/source/tools/monitor/unity/test/curl/reqproxy.lua new file mode 100644 index 0000000000000000000000000000000000000000..972fc7d46587b60c779a3829ed2a40dbbe9118bf --- /dev/null +++ b/source/tools/monitor/unity/test/curl/reqproxy.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/24 17:08 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ChttpCli = require("httplib.httpCli") + +local cli = ChttpCli.new() +local url = "http://127.0.0.1:8400/api/proxy" +local req = {host = "www.baidu.com", uri = "/"} +local res = cli:postTable(url, req) +system:dumps(res) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/curl/tInfluxCli.lua b/source/tools/monitor/unity/test/curl/tInfluxCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..f25d617c6ddc5fc5a16a221e9da2c9002585adb5 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/tInfluxCli.lua @@ -0,0 +1,18 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/6 00:14 +--- + +package.path = package.path .. ";../../?.lua;" +local unistd = require("posix.unistd") +local posix = require("posix") +local coCli = require("httplib.coCli") +local coInflux = require("httplib.coInflux") + +local fdIn, fdOut = unistd.pipe() +local frame = coCli.new(fdIn) +local c = coInflux.new("ld-wz9d17b514mg6kjkx-proxy-tsdb.lindorm.rds.aliyuncs.com", 8242, "/api/v2/write?db=lua") +frame:poll(c) +unistd.close(fdIn) +unistd.close(fdOut) diff --git a/source/tools/monitor/unity/test/curl/tco.lua b/source/tools/monitor/unity/test/curl/tco.lua new file mode 100644 index 0000000000000000000000000000000000000000..2c822eb740209789c2e6bf63133dfc970adb1f72 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/tco.lua @@ -0,0 +1,21 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/3 17:52 +--- + +package.path = package.path .. ";../../?.lua;" + +local system = require("common.system") +local unistd = require("posix.unistd") +local posix = require("posix") +local coCli = require("httplib.coCli") +local coHttpCli = require("httplib.coHttpCli") + +local fdIn, fdOut = unistd.pipe() +local frame = coCli.new(fdIn) +local c = coHttpCli.new("cn.bing.com") +--local c = coHttpCli.new("pylcc.openanolis.cn", 5008) +frame:poll(c) +unistd.close(fdIn) +unistd.close(fdOut) diff --git a/source/tools/monitor/unity/test/curl/tcoCli.lua b/source/tools/monitor/unity/test/curl/tcoCli.lua new file mode 100644 index 0000000000000000000000000000000000000000..2fd0a4cd53ac87ae3f9aa4070bddad9584259f81 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/tcoCli.lua @@ -0,0 +1,14 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/3 16:06 +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local coHttpCli = require("httplib.coHttpCli") + +--local cli = coHttpCli.new("pylcc.openanolis.cn", 5008) +local cli = coHttpCli.new("cn.bing.com") +cli:connect() +cli:get() diff --git a/source/tools/monitor/unity/test/lab/base.lua b/source/tools/monitor/unity/test/lab/base.lua new file mode 100644 index 0000000000000000000000000000000000000000..dab9a72bffc6391f3f5cb5526f759d2e5fb27511 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/base.lua @@ -0,0 +1,23 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/19 6:00 PM +--- + +require("common.class") + +local Cbase = class("base") + +function Cbase:_init_(name) + self.name = name +end + +function Cbase:hello() + return self.name +end + +function Cbase:_del_() + print("base del..." .. self.name) +end + +return Cbase diff --git a/source/tools/monitor/unity/test/lab/dirInotify.lua b/source/tools/monitor/unity/test/lab/dirInotify.lua new file mode 100644 index 0000000000000000000000000000000000000000..6604667216acbbb70a9a27051f8a2d26b5780004 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/dirInotify.lua @@ -0,0 +1,20 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/24 2:15 PM +--- + +package.path = package.path .. ";../../?.lua;" + +local Cinotifies = require("common.inotifies") +local system = require("common.system") + +local ino = Cinotifies.new("/tmp") + +while true do + system:sleep(1) + if ino:isChange() then + ino = Cinotifies.new("/tmp") + print("change.") + end +end diff --git a/source/tools/monitor/unity/test/lab/fileSync/fileSync.py b/source/tools/monitor/unity/test/lab/fileSync/fileSync.py new file mode 100644 index 0000000000000000000000000000000000000000..37b7369770ac754cd69c5a58c2e2998a776e07a4 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/fileSync/fileSync.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： fileSync.py + Description : + Author : liaozhaoyan + date： 2023/4/18 +------------------------------------------------- + Change Activity: + 2023/4/18: +------------------------------------------------- +""" +import os +import sys + + +FILE_SYNC = 0x010000 +FILE_DIRECT = 0x040000 + + +def getFlags(fdPath): + try: + with open(fdPath) as f: + for i, line in enumerate(f): + if line.startswith("flags:"): + _, ret = line.split(":", 1) + return ret.strip() + except IOError: + return 0 + + +def getFdName(pid, fd): + fdPath = "/proc/%s/fd/%s" % (pid, fd) + return os.readlink(fdPath) + + +def getCmd(pid): + path = "/proc/%s/cmdline" % pid + try: + with open(path) as f: + return f.read() + except IOError: + return "nil" + + +def checkFile(fileLink): + if fileLink.startswith("/") and not fileLink.startswith("/dev"): + return True + return False + + +def checkFlag(pid, fd, flag): + vFlag = int(flag) + link = getFdName(pid, fd) + if checkFile(link): + if vFlag & FILE_SYNC: + print("pid:%s, cmd:%s, file:%s, set sync flag" % (pid, getCmd(pid), link)) + if vFlag & FILE_DIRECT: + print("pid:%s, cmd:%s, file:%s, set direct flag" % (pid, getCmd(pid), link)) + + +def checkPid(pid): + path = "/proc/" + pid + "/fdinfo" + for fd in os.listdir(path): + fdPath = "/".join([path, fd]) + try: + flag = getFlags(fdPath) + except IOError: + continue + checkFlag(pid, fd, flag) + + +def walkGroup(path): + path += "/tasks" + try: + with open(path) as f: + for i, line in enumerate(f): + checkPid(line.strip()) + except IOError: + print("bad path.") + + +if __name__ == "__main__": + path = "/sys/fs/cgroup/pids/kubepods/besteffort/slot_976/231a6b41a7be49fdcecac835bc1487272ba8319b8106692d2134c34b025931fa" + if len(sys.argv) > 1: + path = sys.argv[1] + walkGroup(path) diff --git a/source/tools/monitor/unity/test/lab/hashList.lua b/source/tools/monitor/unity/test/lab/hashList.lua new file mode 100644 index 0000000000000000000000000000000000000000..b9dab8fb760a2c31c308f1445b463c47ed6eabe5 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/hashList.lua @@ -0,0 +1,48 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/21 18:25 +--- + +local aList = {'a', 'ab', 'abc'} +print(#aList, table.getn(aList)) +for _, v in ipairs(aList) do + print(v) +end + +aList[2] = nil +print(#aList, table.getn(aList)) +for _, v in ipairs(aList) do + print(v) +end + +for k, v in pairs(aList) do + print(k, v) +end + +local bList = { + a = 1, + b = 2, + c = 3 +} + +bList.b = nil + +print("list.") +aList = {'a', 'ab', 'abc'} +for i, v in ipairs(aList) do + print(v) + if i == 2 then + table.remove(aList, i) + end +end + +print("list reverse.") +aList = {'a', 'ab', 'abc'} +for i = #aList, 1, -1 do + print(aList[i]) + if i == 2 then + table.remove(aList, i) + end +end + diff --git a/source/tools/monitor/unity/test/lab/jencode.lua b/source/tools/monitor/unity/test/lab/jencode.lua new file mode 100644 index 0000000000000000000000000000000000000000..89fbb74db9a17bbd785e26168255d0e5d154946b --- /dev/null +++ b/source/tools/monitor/unity/test/lab/jencode.lua @@ -0,0 +1,13 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/18 15:10 +--- + +local cjson = require("cjson.safe") +local json = cjson.new() +json.encode_escape_forward_slash(false) + +local log = "task:0(swapper/13), cpu:13, delayed:162994631, callstack:mwait_idle_with_hints.constprop.0,intel_idle,cpuidle_enter_state,cpuidle_enter,cpuidle_idle_call,do_idle,cpu_startup_entry,secondary_startup_64_no_verify," + +print(json.encode(log)) diff --git a/source/tools/monitor/unity/test/lab/kmsg/kmsg.c b/source/tools/monitor/unity/test/lab/kmsg/kmsg.c new file mode 100644 index 0000000000000000000000000000000000000000..613fa03cd2a61300dd0b0177d60568cafdb1c4f0 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/kmsg/kmsg.c @@ -0,0 +1,88 @@ +// +// Created by 廖肇燕 on 2023/4/18. +// + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#define KMSG_LINE 8192 + +static int kmsg_set_block(int fd) { + int ret; + int flags = fcntl(fd, F_GETFL); + flags &= ~O_NONBLOCK; + ret = fcntl(fd, F_SETFL, flags); + return ret; +} + +int kmsg_thread_func(void) { + int fd; + int ret; + char buff[KMSG_LINE]; + + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + if (fd < 0) { + goto endOpen; + }/* + ret = lseek(fd, 0, SEEK_DATA); + if (ret < 0) { + perror("kmsg seek error"); + goto endLseek; + }*/ + + ret = 1; // strip old message. + while (ret > 0) { + ret = read(fd, buff, KMSG_LINE - 1); + buff[ret] = '\0'; + printf("%s\n", buff); + if (ret < 0) { + if (errno == EAGAIN) { + break; + } + perror("kmsg read failed."); + goto endRead; + } + } + + ret = kmsg_set_block(fd); + if (ret < 0) { + perror("kmsg set block failed."); + goto endBlock; + } + + while (1) { + + ret = read(fd, buff, KMSG_LINE - 1); + if (ret < 0) { + if (errno == EINTR) { + break; + } + perror("kmsg read2 failed."); + goto endRead; + } + buff[ret -1] = '\0'; + + printf("read: %s\n", buff); + } + + close(fd); + return 0; + endBlock: + endRead: + endLseek: + close(fd); + endOpen: + return 1; +} + +int main(void) { + kmsg_thread_func(); + return 0; +} \ No newline at end of file diff --git a/source/tools/monitor/unity/test/lab/listSun.lua b/source/tools/monitor/unity/test/lab/listSun.lua new file mode 100644 index 0000000000000000000000000000000000000000..815cadee79f7c26cefc46d48a0bebf78db0c5228 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/listSun.lua @@ -0,0 +1,9 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/12 17:54 +--- + +local aList = {1, 2, 3, sum=6} +print(#aList) +print(math.max(aList)) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/lab/objTest.lua b/source/tools/monitor/unity/test/lab/objTest.lua new file mode 100644 index 0000000000000000000000000000000000000000..6a5f252d837097cf990069b4cca510d8555cd018 --- /dev/null +++ b/source/tools/monitor/unity/test/lab/objTest.lua @@ -0,0 +1,30 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/19 5:55 PM +--- + +package.path = package.path .. ";../../?.lua;" + +require("common.class") +local system= require("common.system") + +local Cone = require("one") +local Ctwo = require("two") +local CThree = require("three") + +local one = Cone.new("1one") +local two = Ctwo.new("2two") +local three = CThree.new("3three") + +print(one:hello()) +assert(one:hello() == "1one") +assert(two:hello() == "2two") +assert(three:hello() == "3three") + +one:say() +two:say() +three:say() + +one = nil +two:say() \ No newline at end of file diff --git a/source/tools/monitor/unity/test/lab/one.lua b/source/tools/monitor/unity/test/lab/one.lua new file mode 100644 index 0000000000000000000000000000000000000000..50b37a00800cf05827ce6a3f7405b3a7bc0b5e4b --- /dev/null +++ b/source/tools/monitor/unity/test/lab/one.lua @@ -0,0 +1,20 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/19 5:58 PM +--- + +require("common.class") +local Cbase = require("base") + +Cone = class("one", Cbase) + +function Cone:_init_(name) + Cbase._init_(self, name) +end + +function Cone:say() + print("one say " .. self.name) +end + +return Cone diff --git a/source/tools/monitor/unity/test/lab/structCheck.lua b/source/tools/monitor/unity/test/lab/structCheck.lua new file mode 100644 index 0000000000000000000000000000000000000000..86e5b9e5f0f6985d590b3d4faee56a59f9db9b0e --- /dev/null +++ b/source/tools/monitor/unity/test/lab/structCheck.lua @@ -0,0 +1,12 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/8 01:01 +--- + +require("struct") + +local s = struct.pack(" +#include +#include +#include + + +void processPacket(u_char *arg, const struct pcap_pkthdr *pkthdr, const u_char *packet) +{ + int *count = (int *)arg; + + printf("Packet Count: %d\n", ++(*count)); + printf("Received Packet Size: %d\n", pkthdr->len); + printf("Payload:\n"); + + for(int i=0; i < pkthdr->len; ++i) + { + printf("%02x ", packet[i]); + if ((i + 1) % 16 == 0) + { + printf("\n"); + } + } + printf("\n\n"); + return; +} + +int main() +{ + char errBuf[PCAP_ERRBUF_SIZE], * devStr; + + devStr = pcap_lookupdev(errBuf); + if (devStr) + printf("success: device: %s\n", devStr); + else + { + printf("error: %s\n", errBuf); + exit(1); + } + + /* open a device, wait until a packet arrives */ + pcap_t * device = pcap_open_live(devStr, 64, 1, 0, errBuf); + if (!device) + { + printf("error: pcap_open_live(): %s\n", errBuf); + exit(1); + } + + int count = 0; + /*Loop forever & call processPacket() for every received packet.*/ + pcap_loop(device, 30, processPacket, (u_char *)&count); + + pcap_close(device); + return 0; +} diff --git a/source/tools/monitor/unity/test/pcap/capOne/Makefile b/source/tools/monitor/unity/test/pcap/capOne/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..76ac043e497464cefb9c6fe8202c1795f85f4a02 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/capOne/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g +LDFLAG := -g -lpcap +OBJS = capOne.o +EXEC = capOne + +all: $(EXEC) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(EXEC): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f *.o $(EXEC) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/pcap/capOne/capOne.c b/source/tools/monitor/unity/test/pcap/capOne/capOne.c new file mode 100644 index 0000000000000000000000000000000000000000..c1a59b4160443ee1778d15d730602a9f7afaf086 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/capOne/capOne.c @@ -0,0 +1,56 @@ +// +// Created by 廖肇燕 on 2023/5/4. +// + +#include +#include +#include +#include + +int main() +{ + char errBuf[PCAP_ERRBUF_SIZE], * devStr; + + devStr = pcap_lookupdev(errBuf); + if (devStr) + printf("success: device: %s\n", devStr); + else + { + printf("error: %s\n", errBuf); + exit(1); + } + + /* open a device, wait until a packet arrives */ + pcap_t * device = pcap_open_live(devStr, 65536, 1, 1000, errBuf); + if (!device) + { + printf("error: pcap_open_live(): %s\n", errBuf); + exit(1); + } + + /* wait a packet to arrive */ + struct pcap_pkthdr packet; + const u_char * pktStr = pcap_next(device, &packet); + + if (!pktStr) + { + printf("did not capture a packet!\n"); + exit(1); + } + + printf("Packet len:%d, Bytes:%d, Received time:%s\n", packet.len, + packet.caplen, ctime((const time_t *)&packet.ts.tv_sec)); + + for(int i=0; i < packet.len; ++i) + { + printf(" %02x", pktStr[i]); + if ((i + 1) % 16 == 0) + { + printf("\n"); + } + } + printf("\n"); + + pcap_close(device); + return 0; +} diff --git a/source/tools/monitor/unity/test/pcap/caps/Makefile b/source/tools/monitor/unity/test/pcap/caps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fc90df5ee5d6ff9ef61bfef085f622d157346f97 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/caps/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g +LDFLAG := -g -lpcap +OBJS = caps.o +EXEC = caps + +all: $(EXEC) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(EXEC): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f $(SO) $(EXEC) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/pcap/caps/caps.c b/source/tools/monitor/unity/test/pcap/caps/caps.c new file mode 100644 index 0000000000000000000000000000000000000000..6ffb23288ae27a3429174c67fbb8689127601579 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/caps/caps.c @@ -0,0 +1,257 @@ +// +// Created by 廖肇燕 on 2023/5/4. +// + +#include +#include +#include +#include + +#define u_char unsigned char +#define u_short unsigned short +#define u_int unsigned int +#define uint16_t unsigned short int + +/* 以太网帧头部 */ +struct sniff_ethernet { +#define ETHER_ADDR_LEN 6 + u_char ether_dhost[ETHER_ADDR_LEN]; /* 目的主机的地址 */ + u_char ether_shost[ETHER_ADDR_LEN]; /* 源主机的地址 */ + u_short ether_type; /* IP：0x0800;IPV6:0x86DD; ARP:0x0806;RARP:0x8035 */ +}; +#define ETHERTYPE_IPV4 (0x0800) +#define ETHERTYPE_IPV6 (0x86DD) +#define ETHERTYPE_ARP (0x0806) +#define ETHERTYPE_RARP (0x8035) + + +/* IP数据包的头部 */ +struct sniff_ip { +#if BYTE_ORDER == LITTLE_ENDIAN + u_int ip_hl:4, /* 头部长度 */ + ip_v:4; /* 版本号 */ +#if BYTE_ORDER == BIG_ENDIAN + u_int ip_v:4, /* 版本号 */ + ip_hl:4; /* 头部长度 */ +#endif +#endif /* not _IP_VHL */ + u_char ip_tos; /* 服务的类型 */ + u_short ip_len; /* 总长度 */ + u_short ip_id; /*包标志号 */ + u_short ip_off; /* 碎片偏移 */ +#define IP_RF 0x8000 /* 保留的碎片标志 */ +#define IP_DF 0x4000 /* dont fragment flag */ +#define IP_MF 0x2000 /* 多碎片标志*/ +#define IP_OFFMASK 0x1fff /*分段位 */ + u_char ip_ttl; /* 数据包的生存时间 */ + u_char ip_p; /* 所使用的协议:1 ICMP;2 IGMP;4 IP;6 TCP;17 UDP;89 OSPF */ + u_short ip_sum; /* 校验和 */ + struct in_addr ip_src,ip_dst; /* 源地址、目的地址*/ +}; +#define IPTYPE_ICMP (1) +#define IPTYPE_IGMP (2) +#define IPTYPE_IP (4) +#define IPTYPE_TCP (6) +#define IPTYPE_UDP (17) +#define IPTYPE_OSPF (89) + +typedef u_int tcp_seq; +/* TCP 数据包的头部 */ +struct sniff_tcp { + u_short th_sport; /* 源端口 */ + u_short th_dport; /* 目的端口 */ + tcp_seq th_seq; /* 包序号 */ + tcp_seq th_ack; /* 确认序号 */ +#if BYTE_ORDER == LITTLE_ENDIAN + u_int th_x2:4, /* 还没有用到 */ + th_off:4; /* 数据偏移 */ +#endif +#if BYTE_ORDER == BIG_ENDIAN + u_int th_off:4, /* 数据偏移*/ + th_x2:4; /*还没有用到 */ +#endif + u_char th_flags; +#define TH_FIN 0x01 +#define TH_SYN 0x02 +#define TH_RST 0x04 +#define TH_PUSH 0x08 +#define TH_ACK 0x10 +#define TH_URG 0x20 +#define TH_ECE 0x40 +#define TH_CWR 0x80 +#define TH_FLAGS (TH_FINTH_SYNTH_RSTTH_ACKTH_URGTH_ECETH_CWR) + u_short th_win; /* TCP滑动窗口 */ + u_short th_sum; /* 头部校验和 */ + u_short th_urp; /* 紧急服务位 */ +}; + + +/* UDP header */ +struct sniff_udp{ + uint16_t sport; /* source port */ + uint16_t dport; /* destination port */ + uint16_t udp_length; + uint16_t udp_sum; /* checksum */ +}; + +int pcap_protocal(const struct pcap_pkthdr *pkthdr, const u_char *packet,pcap_dumper_t* arg) +{ + pcap_dump((char *)arg, pkthdr, packet); + printf("packet size:%u, data len:%u\n", pkthdr->len, pkthdr->caplen); //数据包实际的长度, 抓到时的数据长度 + + struct sniff_ethernet *ethernet = (struct sniff_ethernet*)packet; + unsigned char* src_mac = ethernet->ether_shost; + unsigned char* dst_mac = ethernet->ether_dhost; + + printf("src_mac:%x:%x:%x:%x:%x:%x\n",src_mac[0],src_mac[1],src_mac[2],src_mac[3],src_mac[4],src_mac[5]); + printf("dst_mac:%x:%x:%x:%x:%x:%x\n",dst_mac[0],dst_mac[1],dst_mac[2],dst_mac[3],dst_mac[4],dst_mac[5]); + printf("ether_type:%u\n",ethernet->ether_type); + + int eth_len = sizeof(struct sniff_ethernet); //以太网头的长度 + int ip_len = sizeof(struct sniff_ip); //ip头的长度 + int tcp_len = sizeof(struct sniff_tcp); //tcp头的长度 + int udp_len = sizeof(struct sniff_udp); //udp头的长度 + printf("eth_len: %d\n",eth_len); + printf("ip_len: %d\n",ip_len); + printf("tcp_len: %d\n",tcp_len); + printf("udp_len: %d\n",udp_len); + printf("/************************************/\n"); + + /*解析网络层 IP头*/ + //ntohs()是一个函数名，作用是将一个16位数由网络字节顺序转换为主机字节顺序。 + if(ntohs(ethernet->ether_type) == ETHERTYPE_IPV4) + { //IPV4 + printf("/**********************************************************************/\n"); + printf("It's IPv4!\n"); + struct sniff_ip* ip = (struct sniff_ip*)(packet + eth_len); + printf("ip->ip_hl:%d\n",ip->ip_hl); + printf("ip->ip_hl & 0x0f:%x\n",ip->ip_hl & 0x0f); + ip_len = (ip->ip_hl & 0x0f)*4; //ip头的长度 + printf("ip->ip_v:%d\n",ip->ip_v); + // printf("ip->ip_tos:%s\n",ip->ip_tos); + printf("ip->ip_len:%d\n",ip->ip_len); + // struct in_addr + // { + // in_addr_t s_addr; + // }; + unsigned char *saddr = (unsigned char*)&ip->ip_src.s_addr; //网络字节序转换成主机字节序 + unsigned char *daddr = (unsigned char*)&ip->ip_dst.s_addr; + + //printf("eth_len:%u ip_len:%u tcp_len:%u udp_len:%u\n", eth_len, ip_len, tcp_len, udp_len); + printf("src_ip:%d.%d.%d.%d\n", saddr[0], saddr[1],saddr[2],saddr[3]/*InttoIpv4str(saddr)*/); //源IP地址 + printf("dst_ip:%d.%d.%d.%d\n", daddr[0],daddr[1],daddr[2],daddr[3]/*InttoIpv4str(daddr)*/); //目的IP地址 + + /*解析传输层 TCP、UDP、ICMP*/ + + if(ip->ip_p == IPTYPE_TCP) + { //TCP + printf("ip->proto:TCP\n"); //传输层用的哪一个协议 + struct sniff_tcp* tcp = (struct sniff_tcp*)(packet + eth_len + ip_len); + printf("tcp_sport = %u\n", tcp->th_sport); + printf("tcp_dport = %u\n", tcp->th_dport); + for(int i=0;*(packet + eth_len + ip_len+tcp_len+i)!='\0';i++) + { + printf("%02x ",*(packet + eth_len + ip_len+tcp_len+i)); + } + + /**********(pcaket + eth_len + ip_len + tcp_len)就是TCP协议传输的正文数据了***********/ + } + else if(ip->ip_p == IPTYPE_UDP) + { //UDP + printf("ip->proto:UDP\n"); //传输层用的哪一个协议 + struct sniff_udp* udp = (struct sniff_udp*)(packet + eth_len + ip_len); + printf("udp_sport = %u\n", udp->sport); + printf("udp_dport = %u\n", udp->dport); + /**********(pcaket + eth_len + ip_len + udp_len)就是UDP协议传输的正文数据了***********/ + } + else if(ip->ip_p == IPTYPE_ICMP) + { //ICMP + printf("ip->proto:CCMP\n"); //传输层用的哪一个协议 + } + + } + else if(ntohs(ethernet->ether_type) == ETHERTYPE_IPV6) + { //IPV6 + printf("It's IPv6!\n"); + } + else{ + printf("既不是IPV4也不是IPV6\n"); + } + printf("============================================\n"); + return 0; +} + +int main() +{ + int ret32 = -1; + pcap_t *handle = NULL; /* 会话的句柄 */ + char *dev = "lo"; /* 执行嗅探的设备 */ + char errbuf[PCAP_ERRBUF_SIZE]; /* 存储错误 信息的字符串 */ + struct bpf_program filter; /*已经编译好的过滤表达式*/ + char filter_app[] = "port 7890"; /* 过滤表达式*/ + bpf_u_int32 mask; /* 执行嗅探的设备的网络掩码 */ + bpf_u_int32 net; /* 执行嗅探的设备的IP地址 */ + const u_char *packet; /* 实际的包 */ + struct pcap_pkthdr header; /* 由pcap.h定义 */ + struct in_addr ip_addr; + int pcapnum=0; + + /*开启支持PCAP的设备嗅探*/ + // dev = pcap_lookupdev(errbuf); + ret32 = pcap_lookupnet(dev, &net, &mask, errbuf);//获取指定设备的网络号与掩码，如果出错，返回-1，errbuf存放错误信息 + printf("Device: %s\n", dev); + if(ret32 < 0) + { + printf("pcap_lookupnet return %d, errbuf:%s\n", ret32, errbuf); + } + printf("sizeof(mask) = %d, mask:%#x, net:%#x\n",sizeof(mask), mask, net); + ip_addr.s_addr= net; + printf("ipaddress is :%s\n",inet_ntoa(ip_addr)); + + + handle = pcap_open_live(dev, 10*1024, 1, 0, errbuf); + // printf("handle = %s\n",handle); + //捕获网络数据包的数据包捕获描述字，打开名为DEV的网络设备，最大捕获字节为1024字节， + //将网络接口设定为混杂模式，超时时间为0ms意味着一直嗅探直到捕获到数据 + if(handle == NULL) + { + printf("pcap_open_live return err,errbuf:%s...\n", errbuf); + return -1; + } + + pcap_dumper_t* out_pcap; + out_pcap = pcap_dump_open(handle,"protocal.pcap"); + + while(1) + { + pcapnum++; + if(pcapnum>100) + { + break; + } + /* 截获一个包 */ + packet = pcap_next(handle, &header); + if(packet) + { + /* 打印它的长度 */ + printf("Jacked a packet with length of [%d]\n", header.len); + //数据包协议解析 + pcap_protocal(&header, packet,out_pcap); + } + else + { + printf("pcap_next return err, errbuf:%s\n", errbuf); + break; + } + } + + /*flush buff*/ + pcap_dump_flush(out_pcap); + + pcap_dump_close(out_pcap); + /* 关闭会话 */ + pcap_close(handle); + + return 0; +} + diff --git a/source/tools/monitor/unity/test/pcap/dump/Makefile b/source/tools/monitor/unity/test/pcap/dump/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d29279e9f12e99e5dbaea5c53f2080108fa6c49a --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/dump/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g +LDFLAG := -g -lpcap +OBJS = dump.o +EXEC = dump + +all: $(EXEC) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(EXEC): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f *.o $(EXEC) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/pcap/dump/dump.c b/source/tools/monitor/unity/test/pcap/dump/dump.c new file mode 100644 index 0000000000000000000000000000000000000000..c6889d7be20a4e4746af182b865d454154490600 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/dump/dump.c @@ -0,0 +1,85 @@ +// +// Created by 廖肇燕 on 2023/5/5. +// + +#include +#include +#include +#include + +static int pack_cnt = 0; + +void processPacket(u_char *arg, const struct pcap_pkthdr *pkthdr, const u_char *packet) +{ + pcap_dump(arg, pkthdr, packet); + pack_cnt ++; + return; +} + +int main(int argc, char * argv[]) { + int ret; + time_t hope; + char errBuf[PCAP_ERRBUF_SIZE], *devStr; + + if (argc < 2) { + printf("argc %d is less than 2\n", argc); + exit(1); + } + + devStr = pcap_lookupdev(errBuf); + if (devStr) + printf("success: device: %s\n", devStr); + else + { + printf("error: %s\n", errBuf); + exit(1); + } + + /* open a device, wait until a packet arrives */ + pcap_t * device = pcap_open_live(NULL, 96, 1, 1000, errBuf); + if (!device) + { + fprintf(stderr, "error: pcap_open_live(): %s\n", errBuf); + exit(1); + } + + struct bpf_program filter; +// ret = pcap_compile(device, &filter, "tcp and not port 22", 1, 0); + printf("filter: %s\n", argv[1]); + ret = pcap_compile(device, &filter, argv[1], 1, 0); + if (ret < 0) { + fprintf(stderr, "pcap_compile: %s\n", pcap_geterr(device)); + exit(1); + } + ret = pcap_setfilter(device, &filter); + if (ret < 0) { + fprintf(stderr, "pcap_setfilter: %s\n", pcap_geterr(device)); + exit(1); + } + + /*open pcap write output file*/ + pcap_dumper_t* out_pcap; + out_pcap = pcap_dump_open(device,"pack.pcap"); + + hope = 30 + time(NULL); + while (time(NULL) <= hope) { + int ret = pcap_dispatch(device, -1, processPacket, (u_char *)out_pcap); + if (ret == -1) { + fprintf(stderr, "pcap_dispatch: %s\n", pcap_geterr(device)); + break; + } + + if (ret == -2) { + printf("pcap_breakloop called\n"); + break; + } + } + printf("cap %d\n", pack_cnt); + + /*flush buff*/ + pcap_dump_flush(out_pcap); + + pcap_dump_close(out_pcap); + pcap_close(device); + return 0; +} diff --git a/source/tools/monitor/unity/test/pcap/filter/Makefile b/source/tools/monitor/unity/test/pcap/filter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d781bc08cccc9350b102c77f577d1fb1766a6ff8 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/filter/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g +LDFLAG := -g -lpcap +OBJS = filter.o +EXEC = filter + +all: $(EXEC) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(EXEC): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f *.o $(EXEC) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/pcap/filter/filter.c b/source/tools/monitor/unity/test/pcap/filter/filter.c new file mode 100644 index 0000000000000000000000000000000000000000..98ede821b400357e7106d8c66674bbd8d661e877 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/filter/filter.c @@ -0,0 +1,63 @@ +// +// Created by 廖肇燕 on 2023/5/5. +// + +#include +#include +#include +#include + + +void processPacket(u_char *arg, const struct pcap_pkthdr *pkthdr, const u_char *packet) +{ + int *count = (int *)arg; + + printf("Packet Count: %d\n", ++(*count)); + printf("Received Packet Size: %d\n", pkthdr->len); + printf("Payload:\n"); + + for(int i=0; i < pkthdr->len; ++i) + { + printf("%02x ", packet[i]); + if ((i + 1) % 16 == 0) + { + printf("\n"); + } + } + printf("\n\n"); + return; +} + +int main() +{ + char errBuf[PCAP_ERRBUF_SIZE], * devStr; + + devStr = pcap_lookupdev(errBuf); + if (devStr) + printf("success: device: %s\n", devStr); + else + { + printf("error: %s\n", errBuf); + exit(1); + } + + /* open a device, wait until a packet arrives */ + pcap_t * device = pcap_open_live(devStr, 65535, 1, 0, errBuf); + if (!device) + { + printf("error: pcap_open_live(): %s\n", errBuf); + exit(1); + } + + /* construct a filter */ + struct bpf_program filter; + pcap_compile(device, &filter, "tcp", 1, 0); + pcap_setfilter(device, &filter); + + int count = 0; + /*Loop forever & call processPacket() for every received packet.*/ + pcap_loop(device, 30, processPacket, (u_char *)&count); + + pcap_close(device); + return 0; +} diff --git a/source/tools/monitor/unity/test/pcap/lookupdev/Makefile b/source/tools/monitor/unity/test/pcap/lookupdev/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9b8b54e7268faa1cbec578e31b2dcb2021375037 --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/lookupdev/Makefile @@ -0,0 +1,16 @@ +CC := gcc +CFLAG := -g +LDFLAG := -g -lpcap +OBJS = lookupdev.o +EXEC = lookupdev + +all: $(EXEC) + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(EXEC): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +clean: + rm -f $(SO) $(EXEC) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/pcap/lookupdev/lookupdev.c b/source/tools/monitor/unity/test/pcap/lookupdev/lookupdev.c new file mode 100644 index 0000000000000000000000000000000000000000..9a97b9a199b5e1b533ffa619fdac9da30ebb4bae --- /dev/null +++ b/source/tools/monitor/unity/test/pcap/lookupdev/lookupdev.c @@ -0,0 +1,23 @@ +// +// Created by 廖肇燕 on 2023/5/4. +// + +#include +#include +#include + +int main() +{ + char errBuf[PCAP_ERRBUF_SIZE], * devStr; + + devStr = pcap_lookupdev(errBuf); + if (devStr) + printf("success: device: %s\n", devStr); + else + { + printf("error: %s\n", errBuf); + exit(1); + } + + return 0; +} diff --git a/source/tools/monitor/unity/test/rbtree/event.lua b/source/tools/monitor/unity/test/rbtree/event.lua index 14d6f91b00e7cfdb511c2440b224b8013127f3a7..104f8e457c7a3e98467f46c4fa624644e5433096 100644 --- a/source/tools/monitor/unity/test/rbtree/event.lua +++ b/source/tools/monitor/unity/test/rbtree/event.lua @@ -7,12 +7,47 @@ package.cpath = package.cpath .. ";../../beeQ/lib/?.so" package.path = package.path .. ";../../?.lua;" -local CrbEvent = require("common/rbEvent") +local CrbEvent = require("beeQ.rbtree.rbEvent") + +require("common.class") +local tObj = class("tOjb") + +function tObj:_init_(title) + self._title = title +end + +function tObj:work(t) + print("obj " .. self._title .. " work") +end + +local tObjAdd = class("tObjAdd", tObj) + +function tObjAdd:_init_(title) + tObj._init_(self, title) + self._add = true + self._count = 0 +end + +function tObjAdd:work(t, tree) + if self._add then + local o3 = tObj.new("o3") + tree:addEvent("stop period 3", o3, 3, true, 10) + self._add = false + end + tObj.work(self, t) + self._count = self._count + 1 + if self._count > 5 then + print("add Obj leave.") + return -1 + end +end + +local o1 = tObj.new("o1") +local o2 = tObjAdd.new("o2") local e = CrbEvent.new() -e:addEvent("test period 2", 2) -e:addEvent("test period 3", 3) -e:addEvent("stop period 3", 3, true, 10) +e:addEvent("test period 2", o1, 2) +e:addEvent("test period 3", o2, 3) e:proc() \ No newline at end of file diff --git a/source/tools/monitor/unity/test/readlink.lua b/source/tools/monitor/unity/test/readlink.lua new file mode 100644 index 0000000000000000000000000000000000000000..d95f583875e98ef7cf49de93bdc8bafcaa5514af --- /dev/null +++ b/source/tools/monitor/unity/test/readlink.lua @@ -0,0 +1,14 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/5/5 11:33 +--- + +local stat = require("posix.sys.stat") + +local res, err, errno = stat.stat("/proc/1/fd/36") +print(err, errno) +print(stat.S_ISSOCK(res.st_mode)) +for a, b in pairs(res) do + print(a, b) +end diff --git a/source/tools/monitor/unity/test/snappy_test.lua b/source/tools/monitor/unity/test/snappy_test.lua new file mode 100644 index 0000000000000000000000000000000000000000..4bcc636f13e961c11f87a7fd12309d601e733f03 --- /dev/null +++ b/source/tools/monitor/unity/test/snappy_test.lua @@ -0,0 +1,11 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2022/12/18 12:58 AM +--- + +local snappy = require("snappy") + +local compressed = snappy.compress("hello world. you are here") +local decompressed = snappy.decompress(compressed) +print(decompressed) diff --git a/source/tools/monitor/unity/yamls/group.yaml b/source/tools/monitor/unity/yamls/group.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec7217b891a0ccc758e46b867cf0de87603984b9 --- /dev/null +++ b/source/tools/monitor/unity/yamls/group.yaml @@ -0,0 +1,73 @@ +config: + freq: 20 # unit second + daemon: true + port: 8400 # bind port + bind_addr: 127.0.0.1 # bind ip + backlog: 32 # listen backlog + identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg), env(need name arg) + mode: hostip + proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ + db: + rotate: 7 # tsdb file retention time, unit day + budget: 200 # max query buffer from tsdb. + limit: + cpu: 30 # unit % + mem: 50 # unit mb + tasks: 10 # monitor 10 pid max. + +outline: + - /var/sysom/outline + +pushTo: + to: "Influx" + host: "ld-8vb0s2ih252f53pv4-proxy-tsdb.lindorm.rds.aliyuncs.com" + port: 8242 + url: "/api/v2/write?db=sysom" + +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat", + "proc_uptime"] + +plugins: + - so: kmsg + description: "collect dmesg info." + - so: net_health + description: "tcp net health." + - so: net_retrans + description: "tcp retrans monitor." + - so: virtout + description: "virt status out put." + - so: sum_retrans + description: "summary retrans out put." + +metrics: + - title: sysak_proc_pkt_status + from: pkt_status + head: counter + help: "net status info from /proc/net/snmp and /proc/net/status." + type: "gauge" + - title: sysak_net_health_hist + from: net_health_hist + head: value + help: "net_health_hist" + type: "gauge" + - title: sysak_net_health_count + from: net_health_count + head: value + help: "net_health_count" + type: "gauge" + - title: sysak_net_retrans_count + from: net_retrans_count + head: value + help: "net_retrans_count" + type: "gauge" + - title: sysak_virtout_dist + from: virtout_dist + head: value + help: "sysak_virtout_dist" + type: "gauge" + - title: sysak_retrans + from: retrans + head: value + help: "sysak_retrans" + type: "gauge"