From dc7cb72f2a685e5680f70295091df9981fccc36f Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Mon, 20 Feb 2023 17:48:58 +0800 Subject: [PATCH 01/77] unity/plugin: chang the table name to adjust cload-moni Signed-off-by: Hailong Liu --- .../unity/collector/plugin/proc_loadavg/proc_loadavg.c | 2 +- .../unity/collector/plugin/proc_schedstat/proc_schedstat.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index 7494e325..ace88eaf 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -67,7 +67,7 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); - unity_set_table(line, "proc_loadavg"); + unity_set_table(line, "sched_moni"); full_line(line); return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index 41850778..b51d5d38 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -164,10 +164,10 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, nr_cpus+1); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); - unity_set_table(lines1[i], "proc_schedstat"); + unity_set_table(lines1[i], "sched_moni"); } line2 = unity_get_line(lines, nr_cpus); - unity_set_table(line2, "proc_schedstat"); + unity_set_table(line2, "sched_moni"); full_line(lines1, line2); return 0; } -- Gitee From d5c91506a977692830a33159f0787582b88a3731 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 21 Feb 2023 01:05:32 +0800 Subject: [PATCH 02/77] http can brower by markdown, html, text, report failure info for every bad call. --- source/tools/monitor/unity/beaver/beaver.c | 37 +++---- source/tools/monitor/unity/beaver/frame.lua | 32 ++++-- .../monitor/unity/beaver/guide/dev_proc.md | 2 +- .../tools/monitor/unity/beaver/guide/guide.md | 12 +-- .../monitor/unity/beaver/guide/hotplugin.md | 2 +- .../tools/monitor/unity/beaver/guide/oop.md | 2 +- .../monitor/unity/beaver/guide/proc_probe.md | 2 +- .../monitor/unity/beaver/guide/pystring.md | 3 +- .../monitor/unity/beaver/guide/webdevel.md | 4 +- source/tools/monitor/unity/beaver/index.lua | 2 +- .../monitor/unity/beaver/localBeaver.lua | 86 ++++++++++------ .../tools/monitor/unity/beaver/url_guide.lua | 48 ++------- source/tools/monitor/unity/beeQ/apps.c | 98 +++++++++++++------ .../monitor/unity/collector/outline/outline.c | 36 +++---- .../tools/monitor/unity/collector/plugin.yaml | 5 + .../unity/collector/plugin/proto_sender.c | 39 +++----- source/tools/monitor/unity/common/lmd.lua | 26 ++++- source/tools/monitor/unity/common/system.lua | 5 + .../tools/monitor/unity/httplib/httpBase.lua | 22 ++++- .../tools/monitor/unity/httplib/httpHtml.lua | 71 +++++++++++++- .../monitor/unity/test/beaver/walkDir.lua | 43 ++++++++ .../tools/monitor/unity/test/unix/pyunix.py | 28 ++++++ source/tools/monitor/unity/tsdb/foxTSDB.lua | 31 +++--- 23 files changed, 420 insertions(+), 216 deletions(-) create mode 100644 source/tools/monitor/unity/test/beaver/walkDir.lua create mode 100644 source/tools/monitor/unity/test/unix/pyunix.py diff --git a/source/tools/monitor/unity/beaver/beaver.c b/source/tools/monitor/unity/beaver/beaver.c index 293cbb4b..9885f056 100644 --- a/source/tools/monitor/unity/beaver/beaver.c +++ b/source/tools/monitor/unity/beaver/beaver.c @@ -12,22 +12,19 @@ #include #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, char *fYaml) { +static int call_init(lua_State *L, int err_func, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -67,6 +64,7 @@ void LuaAddPath(lua_State *L, char *name, char *value) { static lua_State * echos_init(char *fYaml) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -77,22 +75,15 @@ static lua_State * echos_init(char *fYaml) { /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); - LuaAddPath(L, "path", "../beaver/?.lua"); + err_func = lua_reg_errFunc(L); - ret = luaL_loadfile(L, "../beaver/beaver.lua"); - ret = lua_pcall(L, 0, LUA_MULTRET, 0); + ret = lua_load_do_file(L, "../beaver/beaver.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L, fYaml); + ret = call_init(L, err_func, fYaml); if (ret < 0) { goto endCall; } @@ -107,13 +98,14 @@ static lua_State * echos_init(char *fYaml) { static int echos(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "echo"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -147,7 +139,6 @@ int beaver_init(char *fYaml) { } ret = echos(L); lua_close(L); - sleep(5); // to release port } exit(1); } diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 0c243edd..09e95484 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -6,17 +6,16 @@ -- refer to https://blog.csdn.net/zx_emily/article/details/83024065 -local unistd = require("posix.unistd") -local poll = require("posix.poll") - require("common.class") local ChttpComm = require("httplib.httpComm") local pystring = require("common.pystring") +local system = require("common.system") local Cframe = class("frame", ChttpComm) function Cframe:_init_() ChttpComm._init_(self) self._objs = {} + self._obj_res = {} end local function waitDataRest(fread, rest, tReq) @@ -123,6 +122,14 @@ function Cframe:echo404() return pystring:join("\r\n", tHttp) end +function Cframe:findObjRes(path) + for k, v in pairs(self._obj_res) do + if string.find(path, k) then + return v + end + end +end + function Cframe:proc(fread) local stream = waitHttpHead(fread) if stream == nil then -- read return stream or error code or nil @@ -135,13 +142,15 @@ function Cframe:proc(fread) local obj = self._objs[tReq.path] local res, keep = obj:call(tReq) return res, keep - else - print("show all path.") - for k, _ in pairs(self._objs) do - print("path:", k) - end - return self:echo404(), false end + + local obj = self:findObjRes(tReq.path) + if obj then + local res, keep = obj:calls(tReq) + return res, keep + end + + return self:echo404(), false end end @@ -150,4 +159,9 @@ function Cframe:register(path, obj) self._objs[path] = obj end +function Cframe:registerRe(path, obj) + assert(self._obj_res[path] == nil, "the " .. path .. " is already registered.") + self._obj_res[path] = obj +end + return Cframe diff --git a/source/tools/monitor/unity/beaver/guide/dev_proc.md b/source/tools/monitor/unity/beaver/guide/dev_proc.md index 3c006d0d..b21b8863 100644 --- a/source/tools/monitor/unity/beaver/guide/dev_proc.md +++ b/source/tools/monitor/unity/beaver/guide/dev_proc.md @@ -304,4 +304,4 @@ return CkvProc -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index 0ac809e7..d6b61fa3 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,8 @@ # 目录 -1. [插件化与热更新](/guide/hotplugin) -2. [面向对象设计](/guide/oop) -3. [字符串处理](/guide/pystring) -4. [页面开发](/guide/webdevel) -5. [proc和probe记录表](/guide/proc_probe) -6. [采集proc 接口指标](/guide/dev_proc) \ No newline at end of file +1. [插件化与热更新](/guide/hotplugin.md) +2. [面向对象设计](/guide/oop.md) +3. [字符串处理](/guide/pystring.md) +4. [页面开发](/guide/webdevel.md) +5. [proc和probe记录表](/guide/proc_probe.md) +6. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/hotplugin.md b/source/tools/monitor/unity/beaver/guide/hotplugin.md index 5eb67541..5754909c 100644 --- a/source/tools/monitor/unity/beaver/guide/hotplugin.md +++ b/source/tools/monitor/unity/beaver/guide/hotplugin.md @@ -111,4 +111,4 @@ unity监控采用[yaml](http://yaml.org/)对插件进行管理，当前插件分 此时数据只是已经更新入库了，但是要在nodexport上面显示，需要配置beaver/export.yaml 文件，才能将查询从数据表中更新。 -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/oop.md b/source/tools/monitor/unity/beaver/guide/oop.md index 90c265bc..87c4a675 100644 --- a/source/tools/monitor/unity/beaver/guide/oop.md +++ b/source/tools/monitor/unity/beaver/guide/oop.md @@ -113,4 +113,4 @@ Ctwo 继承于Cone，这里重新实现并复用了父类的say方法。 function Cone:say() function Cone.say(self) -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/proc_probe.md b/source/tools/monitor/unity/beaver/guide/proc_probe.md index ac412ac6..3283c9bc 100644 --- a/source/tools/monitor/unity/beaver/guide/proc_probe.md +++ b/source/tools/monitor/unity/beaver/guide/proc_probe.md @@ -22,4 +22,4 @@ libbpf kprobe/kretprobe/trace\_event/perf event 等事件记录在这里 | ----- | --------- | | xxx | xxx | -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/pystring.md b/source/tools/monitor/unity/beaver/guide/pystring.md index 5e795f4b..c783d0b2 100644 --- a/source/tools/monitor/unity/beaver/guide/pystring.md +++ b/source/tools/monitor/unity/beaver/guide/pystring.md @@ -1,4 +1,5 @@ # 字符串处理 +![pystring](image/python.png) 同为脚本语言，lua 默认的字符串处理并不像python 那么完善。但只要通过拓展，也可以像python 一样对字符串进行处理。当前已经实现了split/strip 等高频使用函数。参考[Python字符串处理](https://www.jianshu.com/p/b758332c44bb) @@ -98,4 +99,4 @@ find 用于子串查找，成功返回首次开始的位置，如果不包含， assert(pystring:find("hello world.", "hello") == 1) ``` -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/webdevel.md b/source/tools/monitor/unity/beaver/guide/webdevel.md index 1fa5b829..b23efd6d 100644 --- a/source/tools/monitor/unity/beaver/guide/webdevel.md +++ b/source/tools/monitor/unity/beaver/guide/webdevel.md @@ -58,11 +58,11 @@ end return CurlGuide ``` -这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop) +这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop.md) ## 热更新 * 如果仅修改了markdown文件，直接更新文件刷新页面即可； * 如果修改了lua文件，给主进程发送1号信号，进程会重新装载，新页面也会立即生效； -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/index.lua b/source/tools/monitor/unity/beaver/index.lua index c3ce944c..b6884e0b 100644 --- a/source/tools/monitor/unity/beaver/index.lua +++ b/source/tools/monitor/unity/beaver/index.lua @@ -52,7 +52,7 @@ function CurlIndex:show(tReq) ### Tips - This page is rendered directly via markdown, for [guide](/guide) + This page is rendered directly via markdown, for [guide](/guide/guide.md) ]] local content2 = string.format("\n thread id is:%d\n", unistd.getpid()) local title = "welcome to visit SysAk Agent server." diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 5671ce87..1f253c64 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -34,6 +34,13 @@ function CLocalBeaver:_init_(frame, fYaml) end function CLocalBeaver:_del_() + for fd in pairs(self._cos) do + socket.shutdown(fd, socket.SHUT_RDWR) + local res = self._cffi.del_fd(self._efd, fd) + print("close fd: " .. fd) + assert(res >= 0) + end + if self._efd then self._cffi.deinit(self._efd) end @@ -42,11 +49,6 @@ function CLocalBeaver:_del_() end end -local function posixError(msg, err, errno) - local s = msg .. string.format(": %s, errno: %d", err, errno) - error(s) -end - function CLocalBeaver:_installTmo(fd) self._tmos[fd] = os.time() end @@ -57,7 +59,7 @@ function CLocalBeaver:_checkTmo() -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety local tmos = system:dictCopy(self._tmos) for fd, t in pairs(tmos) do - if now - t >= 60 then + if now - t >= 10 * 60 then local e = self._ffi.new("native_event_t") e.ev_close = 1 e.fd = fd @@ -81,25 +83,51 @@ function CLocalBeaver:_installFFI() return efd end +local function localBind(fd, tPort) + local try = 0 + local res, err, errno + + -- can reuse for time wait socket. + res, err, errno = socket.setsockopt(fd, socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); + if not res then + system:posixError("set sock opt failed."); + end + + while try < 120 do + res, err, errno = socket.bind(fd, tPort) + if res then + return 0 + elseif errno == 98 then -- port already in use? try 30s; + unistd.sleep(1) + try = try + 1 + else + break + end + end + system:posixError(string.format("bind port %d failed.", tPort.port), err, errno) +end + function CLocalBeaver:_install_fd(port, ip, backlog) local fd, res, err, errno fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if fd then -- for socket - res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) - if res then -- for bind + local tPort = {family=socket.AF_INET, addr=ip, port=port} + local r, msg = pcall(localBind, fd, tPort) + if r then res, err, errno = socket.listen(fd, backlog) if res then -- for listen return fd else - posixError("socket listen failed", err, errno) + unistd.close(fd) + system:posixError("socket listen failed", err, errno) end - else -- for bind failed + else + print(msg) unistd.close(fd) - posixError("socket bind failed", err, errno) os.exit(1) end else -- socket failed - posixError("create socket failed", err, errno) + system:posixError("create socket failed", err, errno) end end @@ -120,7 +148,7 @@ function CLocalBeaver:read(fd, maxLen) return nil end else - posixError("socket recv error", err, errno) + system:posixError("socket recv error", err, errno) end else print(system:dump(e)) @@ -137,7 +165,6 @@ function CLocalBeaver:write(fd, stream) sent, err, errno = socket.send(fd, stream) if sent then if sent < #stream then -- send buffer may full - print("need to send buffer for " .. (#stream - sent)) res = self._cffi.mod_fd(self._efd, fd, 1) -- epoll write ev assert(res == 0) @@ -149,7 +176,7 @@ function CLocalBeaver:write(fd, stream) stream = string.sub(stream, sent + 1) sent, err, errno = socket.send(fd, stream) if sent == nil then - posixError("socket send error.", err, errno) + system:posixError("socket send error.", err, errno) return nil end else -- need to read ? may something error or closed. @@ -161,7 +188,7 @@ function CLocalBeaver:write(fd, stream) end return 1 else - posixError("socket send error.", err, errno) + system:posixError("socket send error.", err, errno) return nil end end @@ -211,12 +238,12 @@ function CLocalBeaver:accept(fd, e) self:co_add(nfd) self:_installTmo(nfd) else - posixError("accept new socket failed", err, errno) + system:posixError("accept new socket failed", err, errno) end end end -function CLocalBeaver:_poll(bfd, nes) +function CLocalBeaver:_pollFd(bfd, nes) for i = 0, nes.num - 1 do local e = nes.evs[i]; local fd = e.fd @@ -233,10 +260,7 @@ function CLocalBeaver:_poll(bfd, nes) self:_checkTmo() end -function CLocalBeaver:poll() - assert(self._once, "poll loop only run once time.") - self._once = false - +function CLocalBeaver:_poll() local bfd = self._bfd local efd = self._efd while true do @@ -244,17 +268,21 @@ function CLocalBeaver:poll() local res = self._cffi.poll_fds(efd, 10, nes) if res < 0 then - break + return "end poll." end - self:_poll(bfd, nes) + self:_pollFd(bfd, nes) end +end + +function CLocalBeaver:poll() + assert(self._once, "poll loop only run once time.") + self._once = false + + local _, msg = pcall(self._poll, self) + print(msg) - for fd in pairs(self._cos) do - local res = self._cffi.del_fd(self._efd, fd) - assert(res >= 0) - end return 0 end -return CLocalBeaver \ No newline at end of file +return CLocalBeaver diff --git a/source/tools/monitor/unity/beaver/url_guide.lua b/source/tools/monitor/unity/beaver/url_guide.lua index 1362b01e..fcb6cd5c 100644 --- a/source/tools/monitor/unity/beaver/url_guide.lua +++ b/source/tools/monitor/unity/beaver/url_guide.lua @@ -5,56 +5,20 @@ --- require("common.class") +local pystring = require("common.pystring") local ChttpHtml = require("httplib.httpHtml") local CurlGuide = class("CurlIndex", ChttpHtml) function CurlGuide:_init_(frame) ChttpHtml._init_(self) - self._urlCb["/guide"] = function(tReq) return self:guide(tReq) end - self._urlCb["/guide/hotplugin"] = function(tReq) return self:hotplugin(tReq) end - self._urlCb["/guide/oop"] = function(tReq) return self:oop(tReq) end - self._urlCb["/guide/pystring"] = function(tReq) return self:pystring(tReq) end - self._urlCb["/guide/webdevel"] = function(tReq) return self:webdevel(tReq) end - self._urlCb["/guide/proc_probe"] = function(tReq) return self:proc_probe(tReq) end - self._urlCb["/guide/dev_proc"] = function(tReq) return self:dev_proc(tReq) end - self:_install(frame) + self:_installRe("^/guide*", frame) + self._head = "/" -- need to strip + self._filePath = "../beaver/" end -local function loadFile(fPpath) - local path = "../beaver/guide/" .. fPpath - local f = io.open(path,"r") - local s = f:read("*all") - f:close() - return s -end - -function CurlGuide:guide(tReq) - return {title="guide", content=self:markdown(loadFile("guide.md"))} -end - -function CurlGuide:hotplugin(tReq) - return {title="hotplugin", content=self:markdown(loadFile("hotplugin.md"))} -end - -function CurlGuide:oop(tReq) - return {title="oop", content=self:markdown(loadFile("oop.md"))} -end - -function CurlGuide:pystring(tReq) - return {title="pystring", content=self:markdown(loadFile("pystring.md"))} -end - -function CurlGuide:webdevel(tReq) - return {title="webdevel", content=self:markdown(loadFile("webdevel.md"))} -end - -function CurlGuide:proc_probe(tReq) - return {title="proc and probes", content=self:markdown(loadFile("proc_probe.md"))} -end - -function CurlGuide:dev_proc(tReq) - return {title="develop proc interface.", content=self:markdown(loadFile("dev_proc.md"))} +function CurlGuide:callRe(tReq, keep) + return self:reSource(tReq, keep, self._head, self._filePath) end return CurlGuide diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index a0cdc50a..2e522bc9 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -15,22 +15,63 @@ static int sample_period = 0; extern char *g_yaml_file; -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +static int lua_traceback(lua_State *L) +{ + const char *errmsg = lua_tostring(L, -1); + lua_getglobal(L, "debug"); + lua_getfield(L, -1, "traceback"); + lua_call(L, 0, 1); + printf("%s \n%s\n", errmsg, lua_tostring(L, -1)); + return 1; +} + +int lua_reg_errFunc(lua_State *L) { + lua_pushcfunction(L, lua_traceback); + return lua_gettop(L); +} -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); +int lua_check_ret(int ret) { + switch (ret) { + case 0: + break; + case LUA_ERRRUN: + printf("lua runtime error.\n"); + break; + case LUA_ERRMEM: + printf("lua memory error.\n"); + case LUA_ERRERR: + printf("lua exec error.\n"); + case LUA_ERRSYNTAX: + printf("file syntax error.\n"); + case LUA_ERRFILE: + printf("load lua file error.\n"); + default: + printf("bad res for %d\n", ret); + exit(1); + } + return ret; } -static int call_init(lua_State *L) { +int lua_load_do_file(lua_State *L, const char* path) { + int err_func = lua_gettop(L); + int ret; + + ret = luaL_loadfile(L, path); + if (ret) { + return lua_check_ret(ret); + } + ret = lua_pcall(L, 0, LUA_MULTRET, err_func); + return lua_check_ret(ret); +} + +static int call_init(lua_State *L, int err_func) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -56,7 +97,7 @@ static int call_init(lua_State *L) { static lua_State * app_recv_init(void) { int ret; - + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); if (L == NULL) { @@ -65,19 +106,14 @@ static lua_State * app_recv_init(void) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "bees.lua"); + ret = lua_load_do_file(L, "bees.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L); + ret = call_init(L, err_func); if (ret < 0) { goto endCall; } @@ -112,6 +148,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { int lret; lua_State *L = (lua_State *)(q->qarg); char *body; + int err_func; if (counter != sighup_counter) { // check counter for signal. lua_close(L); @@ -134,13 +171,13 @@ int app_recv_proc(void* msg, struct beeQ* q) { goto endMem; } memcpy(body, &pMsg->body[0], len); + err_func = lua_gettop(L); lua_getglobal(L, "proc"); lua_pushlstring(L, body, len); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); free(body); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -165,6 +202,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { endReturn: endCall: free(msg); + exit(1); return ret; } @@ -190,6 +228,7 @@ int collector_qout(lua_State *L) { static lua_State * app_collector_init(void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -199,15 +238,10 @@ static lua_State * app_collector_init(void* q, void* proto_q) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "collectors.lua"); + ret = lua_load_do_file(L, "collectors.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } @@ -218,10 +252,9 @@ static lua_State * app_collector_init(void* q, void* proto_q) { lua_pushlightuserdata(L, q); lua_pushlightuserdata(L, proto_q); lua_pushstring(L, g_yaml_file); - ret = lua_pcall(L, 3, 1, 0); + ret = lua_pcall(L, 3, 1, err_func); if (ret < 0) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -252,6 +285,7 @@ static lua_State * app_collector_init(void* q, void* proto_q) { static int app_collector_work(lua_State **pL, void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; static int counter = 0; @@ -268,12 +302,12 @@ static int app_collector_work(lua_State **pL, void* q, void* proto_q) { counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "work"); lua_pushinteger(L, sample_period); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 753049af..eae253a4 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -5,23 +5,19 @@ #include "outline.h" #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, void* q, char *fYaml) { +static int call_init(lua_State *L, int err_func, void* q, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, q); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -48,6 +44,7 @@ static int call_init(lua_State *L, void* q, char *fYaml) { extern int collector_qout(lua_State *L); static lua_State * pipe_init(void* q, char *fYaml) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -57,21 +54,17 @@ static lua_State * pipe_init(void* q, char *fYaml) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "outline.lua"); + ret = lua_load_do_file(L, "outline.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, q, fYaml); - if (ret < 0) { + ret = call_init(L, err_func, q, fYaml); + if (ret) { + lua_check_ret(ret); goto endCall; } return L; @@ -85,13 +78,14 @@ static lua_State * pipe_init(void* q, char *fYaml) { static int work(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "work"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 43eb74f9..a1f78315 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -120,3 +120,8 @@ metrics: head: value help: "loadavg of system from /proc/loadavg" type: "gauge" + - title: sysak_io_burst + from: io_burst + head: value + help: "io burst value." + type: "gauge" \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/proto_sender.c b/source/tools/monitor/unity/collector/plugin/proto_sender.c index bdf1397e..a05211c1 100644 --- a/source/tools/monitor/unity/collector/plugin/proto_sender.c +++ b/source/tools/monitor/unity/collector/plugin/proto_sender.c @@ -10,23 +10,19 @@ #define PROTO_QUEUE_SIZE 64 #define gettidv1() syscall(__NR_gettid) -LUALIB_API void luaL_traceback(lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, struct beeQ* pushQ) { +static int call_init(lua_State *L, int err_func, struct beeQ* pushQ) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, pushQ); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("proto_sender lua init func error"); - report_lua_failed(L); goto endCall; } @@ -52,6 +48,7 @@ static int call_init(lua_State *L, struct beeQ* pushQ) { extern int collector_qout(lua_State *L); lua_State * proto_sender_lua(struct beeQ* pushQ) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -61,20 +58,15 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "proto_send.lua"); + ret = lua_load_do_file(L, "proto_send.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, pushQ); + ret = call_init(L, err_func, pushQ); if (ret < 0) { goto endCall; } @@ -88,13 +80,13 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { struct beeQ* proto_que(lua_State *L) { int ret; + int err_func = lua_gettop(L); struct beeQ* que; lua_getglobal(L, "que"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("proto_que lua que func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } if (!lua_isuserdata(L, -1)) { // check @@ -120,6 +112,7 @@ struct beeQ* proto_que(lua_State *L) { extern volatile int sighup_counter; int proto_send_proc(void* msg, struct beeQ* q) { int ret = 0; + int err_func; struct unity_lines *lines = (struct unity_lines *)msg; int num = lines->num; struct unity_line * pLine = lines->line; @@ -139,14 +132,14 @@ int proto_send_proc(void* msg, struct beeQ* q) { q->qarg = L; counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "send"); lua_pushnumber(L, num); lua_pushlightuserdata(L, pLine); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/common/lmd.lua b/source/tools/monitor/unity/common/lmd.lua index 67a0e1bf..685c66d1 100644 --- a/source/tools/monitor/unity/common/lmd.lua +++ b/source/tools/monitor/unity/common/lmd.lua @@ -10,6 +10,7 @@ require("common.class") local pystring = require("common.pystring") local Clmd = class("lmd") +local srcPath = "" function Clmd:_init_() self._escs = '\\`*_{}[]()>#+-.!' @@ -98,9 +99,24 @@ local function pCode(s) end end +local function images(s) + local name, link = unpack(pystring:split(s, "](", 1)) + name = string.sub(name, 3) -- ![]() + link = string.sub(link, 1, -2) + + if string.sub(name, -1, -1) == "\\" then + return s + end + if string.sub(link, -1, -1) == "\\" then + return s + end + local path = srcPath .. link + return string.format('%s', path, name) +end + local function links(s) local name, link = unpack(pystring:split(s, "](", 1)) - name = string.sub(name, 2) + name = string.sub(name, 2) -- []() link = string.sub(link, 1, -2) if string.sub(name, -1, -1) == "\\" then return s @@ -111,6 +127,10 @@ local function links(s) return string.format('%s', link, name) end +local function pImages(s) + return string.gsub(s, "!%[.-%]%(.-%)", function(s) return images(s) end) +end + local function pLink(s) return string.gsub(s, "%[.-%]%(.-%)", function(s) return links(s) end) end @@ -353,6 +373,7 @@ function Clmd:seg(s) s = pItalic(s) s = pDelete(s) s = pCode(s) + s = pImages(s) s = pLink(s) return pEscape(s) end @@ -362,11 +383,12 @@ function Clmd:pSeg(s) return pystring:join("", {"

", pEnter(s), "

"}) end -function Clmd:toHtml(md) +function Clmd:toHtml(md, path) local mds = pystring:split(md, '\n') local res = {} local len = #mds local stop = 0 + srcPath = path or "" for i = 1, len do local line = mds[i] diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index 1c083430..68dfa7f5 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -125,4 +125,9 @@ function system:parseYaml(fYaml) return lyaml.load(s) end +function system:posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + return system \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 7ddbc24e..432b622e 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -5,6 +5,7 @@ --- require("common.class") +local system = require("common.system") local ChttpComm = require("httplib.httpComm") local ChttpBase = class("ChttpBase", ChttpComm) @@ -20,22 +21,35 @@ function ChttpBase:_install(frame) end end +function ChttpBase:_installRe(path, frame) + frame:registerRe(path, self) +end + function ChttpBase:echo(tRet, keep) error("ChttpBase:echo is a virtual function.") end local function checkKeep(tReq) local conn = tReq.header["connection"] - if conn and string.lower(conn) == "keep-alive" then - return true + if conn and string.lower(conn) == "close" then + return false end - return false + return true end function ChttpBase:call(tReq) + local keep = checkKeep(tReq) local tRet = self._urlCb[tReq.path](tReq) + local res = self:echo(tRet, keep) + + return res, keep +end + +function ChttpBase:calls(tReq) local keep = checkKeep(tReq) - return self:echo(tRet, keep), keep + local res = self:callRe(tReq, keep) + + return res, keep end return ChttpBase diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index 5205af65..34a9a9ea 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -5,13 +5,35 @@ --- require("common.class") +local unistd = require("posix.unistd") local pystring = require("common.pystring") +local system = require("common.system") local ChttpBase = require("httplib.httpBase") local ChttpHtml = class("ChttpHtml", ChttpBase) function ChttpHtml:_init_(frame) ChttpBase._init_(self) + + self._reCb = { + md = function(s, keep, suffix) return self:renderMd(s, keep, suffix) end, + html = function(s, keep, suffix) return self:renderHtml(s, keep, suffix) end, + jpg = function(s, keep, suffix) return self:renderImage(s, keep, "jpeg") end, + jpeg = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + gif = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + png = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + bmp = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + txt = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + text = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + log = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + } +end + +local function loadFile(fPpath) + local f = io.open(fPpath,"r") + local s = f:read("*all") + f:close() + return s end function ChttpHtml:markdown(text) @@ -19,6 +41,43 @@ function ChttpHtml:markdown(text) return md:toHtml(text) end +function ChttpHtml:renderMd(s, keep, suffix) + local tmd = self:markdown(s) + local tRet = { + title = "markdown document render from beaver.", + content = tmd, + cType = "text/html", + } + return self:echo(tRet, keep) +end + +function ChttpHtml:renderHtml(s, keep, suffix) + local cType = "text/html" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderText(s, keep, suffix) + local cType = "text/plain" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderImage(s, keep, suffix) + local cType = "image/" .. suffix + return self:pack(cType, keep, s) +end + +function ChttpHtml:reSource(tReq, keep, head, srcPath) + local path = tReq.path + path = srcPath .. pystring:lstrip(path, head) + if unistd.access(path) then + local s = loadFile(path) + local _, suffix = unpack(pystring:rsplit(path, ".", 1)) + if system:keyIsIn(self._reCb, suffix) then + return self._reCb[suffix](s, keep, suffix) + end + end +end + local function htmlPack(title, content) local h1 = [[ @@ -41,16 +100,22 @@ local function htmlPack(title, content) return pystring:join("", bodies) end -function ChttpHtml:echo(tRet, keep) +function ChttpHtml:pack(cType, keep, body) local stat = self:packStat(200) local tHead = { - ["Content-Type"] = "text/html", + ["Content-Type"] = cType, ["Connection"] = (keep and "keep-alive") or "close" } - local body = htmlPack(tRet.title, tRet.content) local headers = self:packHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end +function ChttpHtml:echo(tRet, keep) + local cType = tRet.type or "text/html" + local body = htmlPack(tRet.title, tRet.content) + + return self:pack(cType, keep, body) +end + return ChttpHtml diff --git a/source/tools/monitor/unity/test/beaver/walkDir.lua b/source/tools/monitor/unity/test/beaver/walkDir.lua new file mode 100644 index 00000000..27a0fd0d --- /dev/null +++ b/source/tools/monitor/unity/test/beaver/walkDir.lua @@ -0,0 +1,43 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/18 12:13 PM +--- + +package.path = package.path .. ";../../?.lua;" +local dirent = require("posix.dirent") +local sysStat = require("posix.sys.stat") +local system = require("common.system") + +local function join(dir, fName) + local paths = {dir, fName} + return table.concat(paths, "/") +end + +local function walk(orig, dir, tbl) + local ls = dirent.dir(dir) + local len = string.len(orig) + for _, l in ipairs(ls) do + if l == ".." or l == '.' then + goto continue + end + local path = join(dir, l) + local stat, err, errno = sysStat.stat(path) + if stat then + local mode = stat.st_mode + if sysStat.S_ISDIR(mode) > 0 then + walk(orig, path, tbl) + elseif sysStat.S_ISREG(mode) > 0 then + table.insert(tbl, string.sub(path, len + 2)) + end + else + system:posixError(string.format("bad access to file %s", path), err, errno) + end + ::continue:: + end +end + +local tbl = {} +local dir = "../../beaver/guide" +walk(dir, dir, tbl) +print(system:dump(tbl)) diff --git a/source/tools/monitor/unity/test/unix/pyunix.py b/source/tools/monitor/unity/test/unix/pyunix.py new file mode 100644 index 00000000..c215808a --- /dev/null +++ b/source/tools/monitor/unity/test/unix/pyunix.py @@ -0,0 +1,28 @@ +import os +import time +import socket + +PIPE_PATH = "/tmp/sysom" +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = PIPE_PATH + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check Netinfo is running.") + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut() + i = 10 + while True: + nf.puts('io_burst,disk=/dev/vda1 limit=10.0,max=%d,log="io log burst"' % i) + i += 1 + time.sleep(5) \ No newline at end of file diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index b808489b..d5efb6fd 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -268,28 +268,31 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday end function CfoxTSDB:qlast(last, ms) + assert(last < 24 * 60 * 60) + local now = self:get_us() local date = self:getDateFrom_us(now) local beg = now - last * 1e6; - if self._man then -- has setup - if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day - return self:query(beg, now, ms) - else - self:_del_() -- destroy old manager - if self:_setupRead(now) ~= 0 then -- try to create new - return ms - else - return self:query(beg, now, ms) - end - end - else + if not self._man then -- check _man is already installed. if self:_setupRead(now) ~= 0 then -- try to create new return ms - else - return self:query(beg, now, ms) end end + + if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day + return self:query(beg, now, ms) + else + local dStop = self:getDateFrom_us(now) + + local beg1 = beg + local beg2 = self.cffi.make_stamp(dStop) + local now1 = beg2 - 1 + local now2 = now + + ms = self:query(beg1, now1, ms) + return self:query(beg2, now2, ms) + end end function CfoxTSDB:qDay(start, stop, ms, tbls, budget) -- Gitee From e0a88f23d5246673ff64db8d7086181b50bd4ab5 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 21 Feb 2023 01:08:07 +0800 Subject: [PATCH 03/77] http can brower by markdown, html, text, report failure info for every bad call. --- source/tools/monitor/unity/beaver/url_export_raw.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/source/tools/monitor/unity/beaver/url_export_raw.lua b/source/tools/monitor/unity/beaver/url_export_raw.lua index b6c904bf..2629fba8 100644 --- a/source/tools/monitor/unity/beaver/url_export_raw.lua +++ b/source/tools/monitor/unity/beaver/url_export_raw.lua @@ -14,6 +14,7 @@ function CurlExportRaw:_init_(frame, export) self._export = export self._urlCb["/export/metrics"] = function(tReq) return self:show(tReq) end + self._urlCb["/metrics"] = function(tReq) return self:show(tReq) end self:_install(frame) end -- Gitee From 938b52d774064bf8eebf3c3da80229d5e9ddac05 Mon Sep 17 00:00:00 2001 From: zhilan Date: Tue, 21 Feb 2023 10:29:37 +0800 Subject: [PATCH 04/77] unity: use local var in proc_buddyinfo --- source/tools/monitor/unity/collector/proc_buddyinfo.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index 8d85696e..f2fc8c9b 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -17,10 +17,10 @@ end function CprocBuddyinfo:proc(elapsed, lines) CvProc.proc(self) - buddyinfo = {} + local buddyinfo = {} for line in io.lines(self.pFile) do if string.find(line,"Normal") then - subline = pystring:split(line,"Normal",1)[2] + local subline = pystring:split(line,"Normal",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end @@ -31,7 +31,7 @@ function CprocBuddyinfo:proc(elapsed, lines) if not buddyinfo then for line in io.lines(self.pFile) do if string.find(line,"DMA32") then - subline = pystring:split(line,"DMA32",1)[2] + local subline = pystring:split(line,"DMA32",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end -- Gitee From cf6aef8156f5e20908d78071e78640374e86f05b Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 22 Feb 2023 10:54:57 +0800 Subject: [PATCH 05/77] ioMonitor: Add ioMonitor to unity ioMonitor dependon unity, You must start ioMonitor after the unity is started and just as follow: sysak ioMonitor -y [plugin.yaml of unity] Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/Makefile | 4 + source/tools/monitor/ioMonitor/README.md | 2 + .../tools/monitor/ioMonitor/ioMon/__init__.py | 4 + .../monitor/ioMonitor/ioMon/displayClass.py | 510 ++++++++++++++++++ .../ioMonitor/ioMon/exceptCheckClass.py | 187 +++++++ .../ioMonitor/ioMon/exceptDiagnoseClass.py | 267 +++++++++ .../monitor/ioMonitor/ioMon/ioMonCfgClass.py | 137 +++++ .../monitor/ioMonitor/ioMon/ioMonitorClass.py | 369 +++++++++++++ .../monitor/ioMonitor/ioMon/ioMonitorMain.py | 80 +++ source/tools/monitor/ioMonitor/ioMon/nfPut.py | 37 ++ .../monitor/ioMonitor/ioMon/tools/__init__.py | 4 + .../ioMon/tools/iofstool/__init__.py | 4 + .../ioMonitor/ioMon/tools/iofstool/common.py | 129 +++++ .../ioMon/tools/iofstool/diskstatClass.py | 219 ++++++++ .../ioMon/tools/iofstool/fsstatClass.py | 418 ++++++++++++++ .../ioMon/tools/iofstool/iofsstat.py | 107 ++++ .../ioMon/tools/iofstool/iostatClass.py | 244 +++++++++ .../ioMon/tools/iofstool/promiscClass.py | 215 ++++++++ .../ioMon/tools/iowaitstat/__init__.py | 4 + .../ioMon/tools/iowaitstat/iowaitstat.py | 354 ++++++++++++ source/tools/monitor/ioMonitor/ioMonitor.sh | 11 + .../tools/monitor/unity/collector/plugin.yaml | 15 + 22 files changed, 3321 insertions(+) create mode 100755 source/tools/monitor/ioMonitor/Makefile create mode 100644 source/tools/monitor/ioMonitor/README.md create mode 100644 source/tools/monitor/ioMonitor/ioMon/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/displayClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/nfPut.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/__init__.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py create mode 100755 source/tools/monitor/ioMonitor/ioMonitor.sh diff --git a/source/tools/monitor/ioMonitor/Makefile b/source/tools/monitor/ioMonitor/Makefile new file mode 100755 index 00000000..1345670e --- /dev/null +++ b/source/tools/monitor/ioMonitor/Makefile @@ -0,0 +1,4 @@ +mods = ioMon +target := ioMonitor + +include $(SRC)/mk/sh.mk diff --git a/source/tools/monitor/ioMonitor/README.md b/source/tools/monitor/ioMonitor/README.md new file mode 100644 index 00000000..4856ff90 --- /dev/null +++ b/source/tools/monitor/ioMonitor/README.md @@ -0,0 +1,2 @@ +# 功能说明 +监控服务主程序，收集系统监控指标，支持查看历史监控数据 diff --git a/source/tools/monitor/ioMonitor/ioMon/__init__.py b/source/tools/monitor/ioMonitor/ioMon/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/displayClass.py b/source/tools/monitor/ioMonitor/ioMon/displayClass.py new file mode 100755 index 00000000..cc51352e --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/displayClass.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import string +import time +import re +import json +import threading +from collections import OrderedDict +from nfPut import CnfPut + + +def bwToValue(bw): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + if str(bw) == '0': + return 0 + for i in range(5, -1, -1): + if units[i] in bw: + return float(bw.split(units[i])[0]) * pow(1024, i) + + +def humConvert(value): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + for i in range(len(units)): + if (value / size) < 1: + return "%.1f%s/s" % (value, units[i]) + value = value / size + + +def iolatencyResultReport(*argvs): + result = [] + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + ioburst = False + nfPrefix = [] + iolatStartT = statusReportDicts['iolatency']['startT'] + iolatEndT = statusReportDicts['iolatency']['endT'] + ioutilStartT = statusReportDicts['ioutil']['startT'] + ioutilEndT = statusReportDicts['ioutil']['endT'] + lastIOburstT = statusReportDicts['iolatency']['lastIOburstT'] + + # If IO burst occurs in the short term(first 180 secs or next 60secs) or + # during IO delay diagnosis, it should be considered as one of + # the delay factors + if iolatStartT - lastIOburstT < 300 or \ + (ioutilStartT >= (iolatStartT - 300) and ioutilEndT <= (iolatEndT + 60)): + statusReportDicts['iolatency']['lastIOburstT'] = iolatStartT + ioburst = True + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 | '\ + 'xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log.stat'): + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + else: + return + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + delays = sorted(ds['delays'], + key=lambda e: (float(e['percent'].strip('%'))), + reverse=True) + maxDelayComp = delays[0]['component'] + maxDelayPercent = float(delays[0]['percent'].strip('%')) + avgLat = format(sum([d['avg'] for d in delays])/1000.0, '.3f') + diagret = 'diagret=\"IO delay(AVG %sms) detected in %s\"' % ( + avgLat, str(ds['diskname'])) + nfPrefix.append(',diag_type=IO-Delay,devname='+str(ds['diskname'])) + + if ioburst and {maxDelayComp, delays[1]['component']}.issubset( + ['disk', 'os(block)']): + if (delays[0]['avg'] / delays[1]['avg']) < 10: + suggest = 'solution=\"reduce IO pressure. Refer to the '\ + 'diagnosis of IO-Burst and optimize some tasks\"' + diskIdx = 0 + if maxDelayComp == 'os(block)': + diskIdx = 1 + reason = ( + 'reason=\"IO burst occurs, too mang IO backlogs'\ + '(disk avg/max lat:%s/%s ms, lat percent:%s,'\ + ' OS dispatch avg/max lat:%s/%s ms, lat percent:%s\"' % + (str(delays[diskIdx]['avg'] / 1000.000), + str(delays[diskIdx]['max'] / 1000.000), + str(delays[diskIdx]['percent']), + str(delays[1 - diskIdx]['avg'] / 1000.000), + str(delays[1 - diskIdx]['max'] / 1000.000), + str(delays[1 - diskIdx]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + else: + statusReportDicts['iolatency']['lastIOburstT'] = lastIOburstT + + suggest = 'solution=\"Please ask the OS kernel expert\"' + maxDelayLog = 'avg/max lat:%s/%s ms, lat percent:%s' %( + str(delays[0]['avg']/1000.000), + str(delays[0]['max']/1000.000), + str(delays[0]['percent'])) + if maxDelayComp == 'disk': + reason = ( + 'reason=\"Disk delay(processing IO slowly, %s)\"' %(maxDelayLog)) + suggest = 'solution=\"Please confirm whether the disk is normal\"' + elif maxDelayComp == 'os(block)': + if delays[1]['component'] == 'disk' and \ + float(delays[1]['percent'].strip('%')) > 20: + with open(argvs[0]+'/resultCons.log') as logF: + data = filter(lambda l : 'F' in l, logF.readlines()) + flushIO = False + if len(data) > 0: + for d in data: + if 'F' in d.split()[-6]: + flushIO = True + break + if flushIO: + suggest = ( + 'Disable flush IO dispatch(echo \"write through\" > '\ + '/sys/class/block/%s/queue/write_cache;'\ + 'echo 0 > /sys/class/block/%s/queue/fua)}' % ( + str(ds['diskname']), str(ds['diskname']))) + suggest += '; Notes: Flush IO is a special instruction to '\ + 'ensure that data is stored persistently on the disk '\ + 'in time, and not saved in the internal cache of the disk.'\ + ' Before disabling, please confirm with the disk FAE '\ + '\"Whether it is necessary to rely on the software to issue'\ + ' flush instructions to ensure data persistent storage\",'\ + ' And avoid data loss due to crash or disk power down' + suggest = 'solution=\"'+suggest+'\"' + else: + suggest = 'solution=\"Please confirm whether the disk is normal\"' + reason = ( + 'reason=\"Disk delay(processing %s slowly, avg/max lat:'\ + '%s/%s ms, lat percent:%s)\"' %( + 'Flush IO' if flushIO else 'IO', + str(delays[1]['avg']/1000.000), + str(delays[1]['max']/1000.000), + str(delays[1]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + reason = ( + 'reason=\"OS delay(Issuing IO slowly at os(block), %s)\"' %( + maxDelayLog)) + else: + reason = ( + 'reason=\"OS delay(processing IO slowly at %s, %s)\"' %( + str(maxDelayComp), maxDelayLog)) + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + # print(e+'\n') + #nf.put(nfPutPrefix, p+' '+e) + nf.puts(nfPutPrefix+p+' '+e) + statusReportDicts['iolatency']['valid'] = True + + +def iohangResultReport(*argvs): + abnormalDicts={} + firstioDicts={} + result=[] + nf=argvs[1] + nfPutPrefix=str(argvs[2]) + statusReportDicts = argvs[3] + nfPrefix=[] + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log'): + with open(argvs[0]+'/result.log') as logF: + data=logF.readline() + else: + return + try: + stat=json.loads(data, object_pairs_hook = OrderedDict) + except Exception: + return + + for ds in stat['summary']: + maxDelay = 0 + hungIO = None + if ds['diskname'] not in abnormalDicts.keys(): + abnormalDicts.setdefault(ds['diskname'], {}) + firstioDicts.setdefault( + ds['diskname'], + {'time':0, 'iotype':0, 'sector':0}) + for hi in ds['hung ios']: + key=hi['abnormal'].split('hang')[0] + delay = float(hi['abnormal'].split('hang')[1].split()[0]) + if delay > maxDelay: + maxDelay = delay + hungIO = hi + if key not in abnormalDicts[ds['diskname']].keys(): + abnormalDicts[ds['diskname']].setdefault(key, 0) + abnormalDicts[ds['diskname']][key] += 1 + t = hungIO['time'].split('.')[0] + tStamp = float(time.mktime(time.strptime(t,'%Y-%m-%d %H:%M:%S'))) + tStamp -= maxDelay + firstioDicts[ds['diskname']]['time'] = \ + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(tStamp+8*3600)) + firstioDicts[ds['diskname']]['iotype'] = hungIO['iotype'] + firstioDicts[ds['diskname']]['sector'] = hungIO['sector'] + for diskname, val in abnormalDicts.items(): + abnormalDicts[diskname] = OrderedDict( + sorted(val.items(), key=lambda e: e[1], reverse=True)) + + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + hungIOS = sorted(ds['hung ios'], + key = lambda e: (float(e['percent'].strip('%'))), + reverse = True) + maxDelayComp=hungIOS[0]['component'] + maxDelayPercent=float(hungIOS[0]['percent'].strip('%')) + maxDelay=format(hungIOS[0]['max']/1000.0, '.3f') + diagret='diagret=\"IO hang %sms detected in %s' % ( + maxDelay, ds['diskname'])+'\"' + nfPrefix.append(',diag_type=IO-Hang,devname='+str(ds['diskname'])) + for key in abnormalDicts[ds['diskname']].keys(): + if maxDelayComp in key: + detail = str( + ''.join(re.findall(re.compile(r'[(](.*?)[)]', re.S), key))) + break + reason = ('reason=\"%s hang(%s, avg/max delay:%s/%s ms), first hang['\ + 'time:%s, iotype:%s, sector:%d]\"' %( + maxDelayComp, detail, + str(hungIOS[0]['avg']/1000.000), + str(hungIOS[0]['max']/1000.000), + firstioDicts[ds['diskname']]['time'], + firstioDicts[ds['diskname']]['iotype'], + firstioDicts[ds['diskname']]['sector'])) + if maxDelayComp == 'Disk' or maxDelayComp == 'OS': + suggest = 'solution=\"Please confirm whether the disk is normal\"' + if maxDelayComp == 'OS': + suggest = 'solution=\"Please ask the OS kernel expert\"' + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + nf.puts(nfPutPrefix+p+' '+e) + #nf.put(nfPutPrefix, p+' '+e) + statusReportDicts['iohang']['valid'] = True + + +def ioutilDataParse(data, resultInfo): + tUnit = None + totalBw = totalIops = 0 + for ds in data['mstats']: + iops = ds['iops_rd'] + ds['iops_wr'] + bps = bwToValue(ds['bps_wr']) + bwToValue(ds['bps_rd']) + totalBw += bps + totalIops += iops + key = ds['comm']+':'+ds['pid']+':'+ds['cid'][0:20]+':'+ds['device'] + if not tUnit: + if ds['bps_wr'] != '0': + tUnit = ds['bps_wr'].split('/')[1] + else: + tUnit = ds['bps_rd'].split('/')[1] + if key not in resultInfo.keys(): + resultInfo.setdefault(key, + {'disk':ds['device'], 'maxIops':0, 'maxBps':0, 'file':ds['file']}) + resultInfo[key]['maxBps'] = max(bps, resultInfo[key]['maxBps']) + resultInfo[key]['maxIops'] = max(iops, resultInfo[key]['maxIops']) + if resultInfo[key]['maxBps'] != bps or resultInfo[key]['maxIops'] != iops: + resultInfo[key]['file'] = ds['file'] + if 'bufferio' in resultInfo.keys(): + del resultInfo[key]['bufferio'] + if 'bufferio' in ds.keys() and 'bufferio' not in resultInfo[key].keys(): + resultInfo[key].setdefault('bufferio', ds['bufferio']) + return totalIops,totalBw,tUnit + + +def ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret): + top = 1 + suggestPS = reason = '' + resultInfo = \ + sorted(resultInfo.items(), key=lambda e: e[1]['maxBps'], reverse=True) + for key, val in resultInfo: + if val['maxIops'] < 50 or val['maxBps'] < 1024 * 1024 * 5: + continue + file = ', target file:'+str(val['file']) if val['file'] != '-' else '' + if 'kworker' in str(key): + kTasklist = [] + if 'bufferio' in val.keys(): + for i in val["bufferio"]: + if 'KB' in i["Wrbw"]: + continue + kTasklist.append(i['task']) + file += ('%s Wrbw %s disk %s file %s;' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + if len(kTasklist): + file = '(Write bio from: '+file+')' + if top == 1: + suggestPS = '(Found \'kworker\' flush dirty pages, Try to reduce'\ + ' the buffer-IO write?%s or check the config /proc/sys/vm/'\ + '{dirty_ratio,dirty_background_ratio} too small?)' %( + '('+';'.join(kTasklist)+')' if len(kTasklist) else '') + maxBps = humConvert(val['maxBps']).replace('s', tUnit) + reason += ('%d. task[%s], access disk %s with iops:%s, bps:%s%s; ' %( + top, str(key.rsplit(':',1)[0]), str(val['disk']), + str(val['maxIops']), maxBps, file)) + if top == 1 and suggestPS == '': + suggestPS = '(Found task \'%s\')' %(str(key.rsplit(':',1)[0])) + top += 1 + suggest = \ + 'Optimize the tasks that contributes the most IO flow%s' % suggestPS + putIdx = ',diag_type=IO-Burst ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + if reason != '': + nf.puts(nfPutPrefix+putIdx+putField) + # print(prefix+reason+suggest+'\n') + + +def ioutilResultReport(*argvs): + resultInfo= {} + nf= argvs[1] + nfPutPrefix= str(argvs[2]) + statusReportDicts = argvs[3] + totalBw = 0 + maxIops = maxBw = 0 + minIops = minBw = sys.maxsize + tUnit = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook =OrderedDict) + except Exception: + return + iops,bw,tUnit = ioutilDataParse(stat, resultInfo) + maxIops = max(maxIops, iops) + minIops = min(minIops, iops) + maxBw = max(maxBw, bw) + minBw = min(minBw, bw) + totalBw += bw + if totalBw < 1024 * 1024 * 10: + return + + if resultInfo: + content = 'Iops:'+str(minIops)+'~'+str(maxIops)+\ + ', Bps:'+humConvert(minBw).replace('s', tUnit)+\ + '~'+humConvert(maxBw).replace('s', tUnit) + diagret = 'IO-Burst('+content+') detected' + ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret) + statusReportDicts['ioutil']['valid'] = True + + +def iowaitDataParse(data, resultInfo): + unkownDisable = False + for io in data['iowait']: + if 'many dirty' in io['reason'] or 'queue full' in io['reason']: + unkownDisable = True + if 'Unkown' in io['reason'] and unkownDisable == True: + continue + key = io['comm']+':'+io['tgid']+':'+io['pid'] + if key not in resultInfo.keys(): + resultInfo.setdefault( + key, {'timeout': 0, 'maxIowait': 0, 'reason': ''}) + if float(io['iowait']) > float(resultInfo[key]['maxIowait']): + resultInfo[key]['maxIowait'] = io['iowait'] + resultInfo[key]['timeout'] = io['timeout'] + resultInfo[key]['reason'] = io['reason'] + return data['global iowait'],unkownDisable + + +def iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret): + top = 0 + reason = '' + resDicts = { + 'Too many dirty pages':False, + 'Device queue full':False, + 'Ioscheduler queue full':False} + + for key, val in resultInfo.items(): + if unkownDisable == True and 'Unkown' in val['reason']: + del resultInfo[key] + + resultInfo = OrderedDict( + sorted(resultInfo.items(), key=lambda e: float(e[1]['maxIowait']), + reverse=True)[:3]) + for key, val in resultInfo.items(): + if unkownDisable == True: + resDicts[val['reason']] = True + top += 1 + reason += ( + '%d. task[%s], wait %sms, contribute iowait %s due to \'%s\'; ' %( + top, str(key), str(val['timeout']), str(val['maxIowait'])+'%', + str(val['reason']))) + + if unkownDisable == True: + if resDicts['Too many dirty pages'] == True: + suggest = 'Reduce io-write pressure or Adjust /proc/sys/vm/'\ + '{dirty_ratio,dirty_bytes} larger carefully' + else: + if resDicts['Device queue full'] and resDicts['Ioscheduler queue full']: + suggest = \ + 'Device queue full -> Disk busy due to disk queue full, '\ + 'Please reduce io pressure;'\ + 'Ioscheduler queue full -> Io scheduler busy due to '\ + 'scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + elif resDicts['Device queue full']: + suggest = 'Disk busy due to disk queue full, '\ + 'Please reduce io pressure' + elif resDicts['Ioscheduler queue full']: + suggest = 'Io scheduler busy due to scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + else: + suggest = 'Report stacktrace to OS kernel specialist' + + putIdx = ',diag_type=IOwait-high ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + nf.puts(nfPutPrefix+putIdx+putField) + + +def iowaitResultReport(*argvs): + resultInfo = {} + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + maxGiowait = 0 + minGiowait = sys.maxsize + unkownDisable = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + gIowait,disable = iowaitDataParse(stat, resultInfo) + if not unkownDisable: + unkownDisable = disable + maxGiowait = max(maxGiowait, gIowait) + minGiowait = min(minGiowait, gIowait) + + if resultInfo: + content = str(minGiowait)+'%~'+str(maxGiowait)+'%' + diagret = 'IOwait high('+content+') detected' + iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret) + statusReportDicts['iowait']['valid'] = True + # print(diagret+reason+solution+'\n') + + +class displayClass(object): + def __init__(self, sender): + self.funcResultReportDicts = { + 'iohang': iohangResultReport, + 'ioutil': ioutilResultReport, + 'iolatency': iolatencyResultReport, + 'iowait': iowaitResultReport} + self.statusReportDicts = { + 'iohang': {'startT': 0, 'endT': 0, 'valid': False}, + 'ioutil': {'startT': 0, 'endT': 0, 'valid': False, + 'iopsThresh': 0, 'bpsThresh': 0}, + 'iolatency': {'startT': 0, 'endT': 0, 'valid': False, + 'lastIOburstT': 0}, + 'iowait': {'startT': 0, 'endT': 0, 'valid': False}, + } + self._sender = sender + self._nfPutPrefix = 'IOMonDiagLog' + + def markIoburst(self, now): + self.statusReportDicts['iolatency']['lastIOburstT'] = now + + def setIoburstThresh(self, iopsThresh, bpsThresh): + self.statusReportDicts['ioutil']['iopsThresh'] = iopsThresh + self.statusReportDicts['ioutil']['bpsThresh'] = bpsThresh + + def diagnoseValid(self, diagType): + return self.statusReportDicts[diagType]['valid'] + + def start(self, timeout, diagType, filepath, startTime, endTime): + self.statusReportDicts[diagType]['startT'] = startTime + self.statusReportDicts[diagType]['endT'] = endTime + self.statusReportDicts[diagType]['valid'] = False + argvs = [ + filepath, self._sender, self._nfPutPrefix, self.statusReportDicts] + timer = threading.Timer(timeout, + self.funcResultReportDicts[diagType], + argvs) + timer.start() diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py new file mode 100755 index 00000000..23b2a9ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- + +import sys +import string + + +class exceptCheckClass(): + def __init__(self, window): + self.window = int(window) if window is not None else 100 + self._exceptChkDicts = {} + + def addItem(self, key): + exceptChkItem = { + 'baseThresh': { + 'nrSample': 0, + 'moveWinData': [], + 'curWinMinVal': sys.maxsize, + 'curWinMaxVal': 0, + 'moveAvg': 0, + 'thresh': 0}, + 'compensation': { + 'thresh': 0, + 'shouldUpdThreshComp': True, + 'decRangeThreshAvg': 0, + 'decRangeCnt': 0, + 'minStableThresh': sys.maxsize, + 'maxStableThresh': 0, + 'stableThreshAvg': 0, + 'nrStableThreshSample': 0}, + 'dynTresh': sys.maxsize, + 'usedWin': 0} + self._exceptChkDicts.setdefault(key, exceptChkItem) + + # The sliding window calculates the basic threshold, through which the spikes + # and burrs in the IO indicators can be screened. The calculation idea is as + # follows: + # 1. take 100 data as a group for calculation (calculate 1 ~ 100 data for the + # first time, 2 ~ 101 for the second time, 3 ~ 102 for the third time, and + # so on), and calculate the average value mavg of 100 data in the current + # window + # 2. obtain the maximum value Max and minimum value min of 100 data, then record + # the thresh (MAX((max-mavg),(mavg-min))) each time, and calculate the average + # value(threshavg) of all thresh at this time each time, taking threshavg as + # the basic threshold for this time + # 3. The next basic threshold follows steps 1, 2, and so on + def _calcBaseThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + thresh = None + + bt['nrSample'] += 1 + if bt['nrSample'] >= self.window: + if len(bt['moveWinData']) < self.window: + bt['moveWinData'].append(e) + else: + bt['moveWinData'][exceptChkDict['usedWin'] % self.window] = e + moveAvg = float( + format(sum(bt['moveWinData']) / float(self.window), '.1f')) + + # Find the min and max values of this window so far + maxVal = max(bt['curWinMaxVal'], e) + minVal = min(bt['curWinMinVal'], e) + nrThreshSample = bt['nrSample'] + 1 - self.window + thresh = float( + format(max(maxVal - moveAvg, moveAvg - minVal), '.1f')) + # Calculate base threshold + threshAvg = float(format( + (bt['thresh'] * (nrThreshSample - 1) + thresh) / nrThreshSample, + '.3f')) + bt['thresh'] = threshAvg + bt['moveAvg'] = moveAvg + bt['curWinMaxVal'] = maxVal + bt['curWinMinVal'] = minVal + + exceptChkDict['usedWin'] += 1 + if exceptChkDict['usedWin'] >= self.window: + # the next window, set min and Max to 0 + bt['curWinMaxVal'] = 0 + bt['curWinMinVal'] = sys.maxsize + exceptChkDict['usedWin'] = 0 + else: + # Here, only the first window will enter to ensure that + # the data in one window is accumulated + bt['moveWinData'].append(e) + bt['curWinMaxVal'] = max(bt['curWinMaxVal'], e) + bt['curWinMinVal'] = min(bt['curWinMinVal'], e) + exceptChkDict['usedWin'] += 1 + return thresh + + # Called by _calcCompThresh to calculate the compensation value + # under normal steady state + def _calcStableThresh(self, ct, curBaseThresh, curThresh): + # Discard points exceeding (base-threshold / 10) + avg = ct['decRangeThreshAvg'] + if (curThresh - avg) < ((curBaseThresh - avg) / 10.0): + tSum = ct['stableThreshAvg'] * \ + ct['nrStableThreshSample'] + curThresh + ct['nrStableThreshSample'] += 1 + ct['stableThreshAvg'] = tSum / ct['nrStableThreshSample'] + ct['minStableThresh'] = min(ct['minStableThresh'], curThresh) + ct['maxStableThresh'] = max(ct['maxStableThresh'], curThresh) + # 1.5 windows of stable data have been counted, + # which can be used as normal threshold compensation value + if ct['nrStableThreshSample'] >= (self.window * 1.5): + ct['thresh'] = \ + max(ct['stableThreshAvg'] - ct['minStableThresh'], + ct['maxStableThresh'] - ct['stableThreshAvg']) + ct['shouldUpdThreshComp'] = False + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Calculate the threshold compensation value and superimpose this value + # on the basic threshold to eliminate false alarms + def _calcCompThresh(self, key, lastBaseThresh, curThresh): + exceptChkDict = self._exceptChkDicts[key] + curBaseThresh = exceptChkDict['baseThresh']['thresh'] + ct = exceptChkDict['compensation'] + + # It is not confirmed whether the current state is constant + # (constant state is defined as IO index fluctuation, which is stable) + # 1. the max basic threshold of this window is the compensation value + # 2. enter a new window to reset to the current basic threshold + if ct['shouldUpdThreshComp'] == True and \ + (ct['thresh'] < curBaseThresh or exceptChkDict['usedWin'] == 0): + ct['thresh'] = curBaseThresh + + # Continuous monotonic decreasing, constant steady state, + # constant compensation threshold inferred + if curBaseThresh < lastBaseThresh: + tSum = ct['decRangeThreshAvg'] * ct['decRangeCnt'] + curThresh + ct['decRangeCnt'] += 1 + ct['decRangeThreshAvg'] = tSum / ct['decRangeCnt'] + # The monotonic decline has continued for 1.5 windows, + # indicating that IO pressure may return to normality + if ct['decRangeCnt'] >= (self.window * 1.5): + self._calcStableThresh(ct, curBaseThresh, curThresh) + else: + # As long as the basic threshold curve is not + # continuously monotonically decreasing, + # reset to 0 and make statistics again + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Update the dynamic threshold of the corresponding indicator type + # and call it after collecting the IO indicators. The key is await, + # util, IOPs, BPS, etc + def updateDynThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + ct = exceptChkDict['compensation'] + lastBaseThresh = bt['thresh'] + + curThresh = self._calcBaseThresh(key, e) + if curThresh is not None: + self._calcCompThresh(key, lastBaseThresh, curThresh) + exceptChkDict['dynTresh'] = \ + bt['thresh'] + bt['moveAvg'] + ct['thresh'] + + # Turn off the threshold compensation of the corresponding indicators. + # Generally, when it is detected that the IO util exceeds 20%, + # it will be disabled according to the situation of each indicator + def disableThreshComp(self, key): + exceptChkDict = self._exceptChkDicts[key] + ct = exceptChkDict['compensation'] + bt = exceptChkDict['baseThresh'] + + #if exceptChkDict['dynTresh'] == sys.maxsize: + # return + + if ct['shouldUpdThreshComp'] == True: + ct['shouldUpdThreshComp'] = False + exceptChkDict['dynTresh'] = bt['thresh'] + bt['moveAvg'] + ct['thresh'] = 0.000001 + + + def getNrDataSample(self, key): + return self._exceptChkDicts[key]['baseThresh']['nrSample'] + + # Get the dynamic threshold of the corresponding indicator type, + # call it after collecting the IO indicators, and judge whether + # the indicators are abnormal. The key is await, util, IOPs, BPS, etc + def getDynThresh(self, key): + return self._exceptChkDicts[key]['dynTresh'] diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py new file mode 100755 index 00000000..98f912ce --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +import os +import string +import time +from collections import OrderedDict +import threading +from tools.iofstool.iofsstat import iofsstatStart +from tools.iowaitstat.iowaitstat import iowaitstatStart +from displayClass import displayClass + + +class runDiag(object): + def __init__(self, logRootPath, sender): + self.funcDicts = { + 'iohang': self.startIohangDiagnose, + 'ioutil': self.startIoutilDiagnose, + 'iolatency': self.startIolatencyDiagnose, + 'iowait': self.startIowaitDiagnose} + self.lastDiagTimeDicts = \ + {'iohang': 0, 'ioutil': 0, 'iolatency': 0, 'iowait': 0} + self.display = displayClass(sender) + self.sysakPath = 'sysak' + self.logRootPath = logRootPath + + + def _recentDiagnoseValid(self, diagType): + return self.display.diagnoseValid(diagType) + + + def startIohangDiagnose(self, *argv): + devname = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iohang'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/hangdetect/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iohang'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' > '+outlog+' &') + self.display.start(20, 'iohang', logdir, now, now+60) + + + def startIolatencyDiagnose(self, *argv): + devname = argv[0] + thresh = argv[1] + ioburst = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['iolatency'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/latency/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iolatency'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' > '+outlog+' &') + if ioburst: + self.display.markIoburst(now) + self.display.start(60, 'iolatency', logdir, now, now+60) + + + def startIoutilDiagnose(self, *argv): + devname = argv[0] + bwThresh = argv[1] + iopsThresh = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['ioutil'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iofsstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['ioutil'] = now + #self.display.setIoburstThresh(iopsThresh, bwThresh) + argvs = ['-j',outlog,'-n','-m','-c','1','-t','5','-T','40', + '-i',str(iopsThresh),'-b',str(bwThresh)] + threading.Thread(target=iofsstatStart, args=(argvs,)).start() + self.display.start(55, 'ioutil', outlog, now, now+60) + + + def startIowaitDiagnose(self, *argv): + iowaitThresh = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iowait'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iowaitstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iowait'] = now + argvs = ['-j', outlog, '-t', '5', '-w', str(iowaitThresh), '-T', '45'] + threading.Thread(target=iowaitstatStart, args=(argvs,)).start() + self.display.start(55, 'iowait', outlog, now, now+60) + + + def runDiagnose(self, diagType, argv): + self.funcDicts[diagType](*list(argv)) + + +class diagnoseClass(runDiag): + def __init__(self, window, logRootPath, sender): + super(diagnoseClass, self).__init__(logRootPath, sender) + self.window = window + self.diagnoseDicts = OrderedDict() + self._diagStat = OrderedDict( + {'iohang': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'ioutil': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iowait': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iolatency': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}}) + + + def addItem(self, devname, key, reportInterval, triggerInterval): + diagRecord = { + 'statWindow': self.window, + 'trigger': False, + 'lastReport': 0, + 'reportInterval': reportInterval, + 'reportCnt': 0, + 'lastDiag': 0, + 'triggerInterval': triggerInterval, + 'diagArgs': [0, 0, 0, 0, 0, 0, 0, 0]} + if devname not in self.diagnoseDicts.keys(): + self.diagnoseDicts.setdefault(devname, {key: diagRecord}) + else: + self.diagnoseDicts[devname].setdefault(key, diagRecord) + + + def setUpDiagnose(self, devname, key, nrSample, *argv): + diagnoseDicts = self.diagnoseDicts[devname][key] + lastDiag = diagnoseDicts['lastDiag'] + lastReport = diagnoseDicts['lastReport'] + statWindow = diagnoseDicts['statWindow'] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if lastReport == 0 or (nrSample-lastReport) > statWindow: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 1 + else: + diagnoseDicts['reportCnt'] += 1 + if diagnoseDicts['reportCnt'] > reportInterval: + if lastDiag == 0 or (nrSample-lastDiag) > triggerInterval: + diagnoseDicts['trigger'] = True + diagnoseDicts['reportCnt'] = 0 + diagnoseDicts['lastDiag'] = nrSample + else: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 0 + elif triggerInterval != 0: + if lastDiag == 0 or (nrSample-lastDiag) >= triggerInterval: + diagnoseDicts['lastDiag'] = nrSample + diagnoseDicts['trigger'] = True + else: + diagnoseDicts['trigger'] = True + + for idx, val in enumerate(argv): + diagnoseDicts['diagArgs'][idx] = val + + + def isException(self, devname, key): + diagnoseDicts = self.diagnoseDicts[devname][key] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if (diagnoseDicts['reportCnt'] + 1) >= reportInterval: + return True + elif triggerInterval != 0: + return True + else: + return True + return False + + + def clearDiagStat(self): + for diagType, stat in self._diagStat.items(): + stat['run'] = False + stat['argv'][0:] = [0, 0, 0, 0, 0, 0, 0, 0] + + + def checkDiagnose(self): + diagnoseDicts = self.diagnoseDicts + diagInfo = {'iohang': [], 'iolatency': [], 'ioutil': []} + diagStat = self._diagStat + ioburst = False + + for devname, diagDict in diagnoseDicts.items(): + if devname == 'system': + if diagDict['iowait']['trigger'] == True: + diagStat['iowait']['run'] = True + diagStat['iowait']['argv'][0] = \ + diagDict['iowait']['diagArgs'][0] + diagDict['iowait']['trigger'] = False + continue + + for diagType in ['iohang', 'iolatency', 'ioutil']: + if diagDict[diagType]['trigger'] == True: + if diagType == 'iolatency': + ioburst = diagDict['iolatency']['diagArgs'][1] + diagInfo[diagType].append(devname) + diagDict[diagType]['trigger'] = False + + for diagType, value in diagInfo.items(): + diagStat[diagType]['run'] = True + if len(value) > 1: + diagStat[diagType]['argv'][0] = None + elif len(value) == 1: + diagStat[diagType]['argv'][0] = value[0] + else: + diagStat[diagType]['run'] = False + + if diagStat['ioutil']['run'] == True: + for idx in [1,2]: + val = sorted( + [diagnoseDicts[dev]['ioutil']['diagArgs'][idx-1] + for dev in diagInfo['ioutil']], + reverse=True) + diagStat['ioutil']['argv'][idx] = val[-1] + + if diagStat['iolatency']['run'] == True: + diagStat['iolatency']['argv'][1] = sorted( + [diagnoseDicts[dev]['iolatency']['diagArgs'][0] + for dev in diagInfo['iolatency']], + reverse=True)[-1] + diagStat['iolatency']['argv'][2] = ioburst + + for diagType, stat in diagStat.items(): + if stat['run'] == True: + self.runDiagnose(diagType, stat['argv']) + stat['run'] = False + + + # In displayClass, after the diagnostic log is reported to the remote end, + # it will be marked as a valid diagnosis, and In exceptDiagnoseClass, + # clear the valid mark before each diagnosis + def recentDiagnoseValid(self, diagType): + return self._recentDiagnoseValid(diagType) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py new file mode 100755 index 00000000..0610135c --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import json +from collections import OrderedDict + +globalCfgDicts = {} +globalCfgPath = '' +def loadCfg(cfgPath): + with open(cfgPath) as f: + data = f.read() + return json.loads(data) + + +def loadCfgHander(signum, frame): + global globalCfgDicts + globalCfgDicts = loadCfg(globalCfgPath) + + +class ioMonCfgClass(object): + def __init__(self, cfgArg, resetCfg, logRootPath): + global globalCfgPath + self.cfgPath = logRootPath+'/ioMon/ioMonCfg.json' + globalCfgPath = self.cfgPath + cfg = self._paserCfg(cfgArg) + hasArgs = any(list(cfg.values())) + if not os.path.exists(self.cfgPath) or resetCfg: + cfg['iowait'] = int(cfg['iowait']) if cfg['iowait'] else 5 + cfg['await'] = int(cfg['await']) if cfg['await'] else 10 + cfg['util'] = int(cfg['util']) if cfg['util'] else 20 + cfg['iops'] = int(cfg['iops']) if cfg['iops'] else 150 + cfg['bps'] = int(cfg['bps']) if cfg['bps'] else 31457280 + cfg['cycle'] = int(cfg['cycle']) if cfg['cycle'] else 1000 + cfg['diagIowait'] = cfg['diagIowait'] if cfg['diagIowait'] else 'on' + cfg['diagIoburst'] = cfg['diagIoburst'] if cfg['diagIoburst'] else 'on' + cfg['diagIolat'] = cfg['diagIolat'] if cfg['diagIolat'] else 'on' + cfg['diagIohang'] = cfg['diagIohang'] if cfg['diagIohang'] else 'off' + self._updateCfg(cfg) + return + else: + self._loadCfg() + if hasArgs: + self._updateCfg(cfg) + + + def _paserCfg(self, cfgArg): + cfgDicts = { + 'iowait':None, 'await':None, 'util':None, 'iops':None, 'bps':None, + 'cycle':None, 'diagIowait':None, 'diagIoburst':None, + 'diagIolat':None, 'diagIohang':None} + try: + cfgList = \ + cfgArg.split(',') if cfgArg is not None and len(cfgArg) > 0 else [] + for cfg in cfgList: + errstr = None + c = cfg.split('=') + if c[0] not in cfgDicts.keys() or len(c[1]) == 0: + errstr = "bad cfg item: %s, must be in %s" %( + cfg, str(cfgDicts.keys())) + elif 'diag' not in c[0] and not c[1].isdigit(): + errstr = "monitor cfg argv must be digit: %s." %cfg + elif 'diag' in c[0] and c[1] not in ['on', 'off']: + errstr = \ + "diagnose cfg argv must be [\'on\', \'off\']: %s." %cfg + if errstr: + print(errstr) + sys.exit(0) + cfgDicts[c[0]] = c[1] + except Exception: + print "bad cfg: %s." %cfg + sys.exit(0) + return cfgDicts + + + def _setGlobalCfgDicts(self, CfgDicts): + global globalCfgDicts + globalCfgDicts = CfgDicts + + + def _getGlobalCfgDicts(self): + global globalCfgDicts + return globalCfgDicts + + + def _updateCfg(self, cfgDicts): + oldCfg = {} + if not os.path.exists(self.cfgPath): + if not os.path.exists(os.path.dirname(self.cfgPath)): + os.mkdir(os.path.dirname(self.cfgPath)) + else: + oldCfg = loadCfg(self.cfgPath) + f = open(self.cfgPath, 'w+') + newCfg = json.loads(json.dumps(cfgDicts)) + if oldCfg: + for key,val in newCfg.items(): + if val is not None: + oldCfg[key] = val + newCfg = oldCfg + s = json.dumps(newCfg, indent=4) + f.write(s) + f.close() + self._setGlobalCfgDicts(newCfg) + + + def _loadCfg(self): + self._setGlobalCfgDicts(loadCfg(self.cfgPath)) + + + def createCfgFlagFile(self): + f = open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag', 'w+') + f.write(str(os.getpid())) + f.close() + signal.signal(signal.SIGUSR2, loadCfgHander) + + + def notifyIoMon(self): + try: + with open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag') as f: + pid = f.read() + with open('/proc/'+str(pid)+'/cmdline') as f: + cmdline = f.read().strip() + except Exception: + sys.exit(0) + if 'ioMonEntry' in cmdline: + os.system('kill -USR2 '+str(pid)) + + + def getCfgItem(self, key): + val = str(self._getGlobalCfgDicts()[key]) + if val.isdigit(): + val = int(val) + return val diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py new file mode 100755 index 00000000..0521a7ff --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +from exceptDiagnoseClass import diagnoseClass +from exceptCheckClass import exceptCheckClass +from ioMonCfgClass import ioMonCfgClass +from collections import OrderedDict +from nfPut import CnfPut + +class ioMonitorClass(object): + def __init__(self, logRootPath, cfg, pipeFile): + self.window = 60 + self.cfg = cfg + self.cfg.createCfgFlagFile() + self.diagSwitch = { + 'diagIowait': {'sw': self.cfg.getCfgItem('diagIowait'), + 'esi':'IOwait-High'}, + 'diagIoburst': {'sw': self.cfg.getCfgItem('diagIoburst'), + 'esi':'IO-Delay'}, + 'diagIolat': {'sw': self.cfg.getCfgItem('diagIolat'), + 'esi':'IO-Burst'}, + 'diagIohang': {'sw': self.cfg.getCfgItem('diagIohang'), + 'esi':'IO-Hang'} + } + self._sender = CnfPut(pipeFile) + self._nfPutTlb = 'IOMonIndForDisksIO' + self._nfPutTlb4System = 'IOMonIndForSystemIO' + self.fieldDicts = OrderedDict() + self.exceptChkDicts = {'system': exceptCheckClass(self.window)} + self.exceptChkDicts['system'].addItem('iowait') + self.diagnose = diagnoseClass(self.window, logRootPath, self._sender) + self.diagnose.addItem('system', 'iowait', 0, 60) + self.fDiskStats = open("/proc/diskstats") + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + self.uploadInter = 0 + self.exceptionStat = {'system': {'IOwait-High': {'cur':0,'max':0}}} + self.dataStat = {'system': {'iowait': 0}} + + + def _addMonitorAttrForDisk(self, disk): + dataStat = self.dataStat + exceptChkDicts = self.exceptChkDicts + diagnose = self.diagnose + exceptionStat = self.exceptionStat + + # used for reporting per-index to database + dataStat.setdefault( + disk, {'await': 0, 'util': 0, 'iops': 0, 'bps': 0, 'qusize': 0}) + + # add exception-check attr for per-index on per-disk + exceptChkDicts.setdefault(disk, exceptCheckClass(self.window)) + for key in ['util', 'await', 'iops', 'bps']: + exceptChkDicts[disk].addItem(key) + + # add diagnose attr for per-index on per-disk + diagnoseDict = { + 'iohang': {'triggerInterval': self.window * 5, + 'reportInterval': 10}, + 'ioutil': {'triggerInterval': 60, 'reportInterval': 0}, + 'iolatency': {'triggerInterval': 60, 'reportInterval': 0} + } + for key, item in diagnoseDict.items(): + diagnose.addItem( + disk, key, item['reportInterval'], item['triggerInterval']) + # used for reporting exception to database + exceptionStat.setdefault( + disk, + {'IO-Delay':{'cur':0,'max':0}, 'IO-Burst':{'cur':0,'max':0}, + 'IO-Hang':{'cur':0,'max':0}}) + + + def _removeDiskMonitor(self, disk): + del self.fieldDicts[disk] + del self.dataStat[disk] + del self.exceptChkDicts[disk] + del self.diagnose[disk] + del self.exceptionStat[disk] + + + def _disableThreshComp(self, devname, qusize): + exceptChkDicts = self.exceptChkDicts + exceptChkDicts[devname].disableThreshComp('util') + exceptChkDicts[devname].disableThreshComp('iops') + exceptChkDicts[devname].disableThreshComp('bps') + if qusize > 1: + exceptChkDicts[devname].disableThreshComp('await') + + + def _calcIowait(self): + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + iowait = float(format( + (statList[4] - self.cpuStatIowait['iowait']) * 100.0 / + (sum(statList) - self.cpuStatIowait['sum']), '.2f')) + return iowait + + + def _calcIoIndex(self, devname, field, secs): + ds = self.dataStat[devname] + uploadInter = self.uploadInter + + rws = field['1'][1] + field['5'][1] - field['1'][0] - field['5'][0] + iops = round(rws / secs, 1) + ds['iops'] = (ds['iops'] * (uploadInter - 1) + iops) / uploadInter + + rwSecs = field['3'][1] + field['7'][1] - field['3'][0] - field['7'][0] + bps = rwSecs / secs * 512 + ds['bps'] = (ds['bps'] * (uploadInter - 1) + bps) / uploadInter + + qusize = round((field['11'][1] - field['11'][0]) / secs / 1000, 2) + ds['qusize'] = (ds['qusize'] * (uploadInter - 1) + qusize) / uploadInter + + rwTiks = field['4'][1] + field['8'][1] - field['4'][0] - field['8'][0] + wait = round(rwTiks / (iops * secs), 2) if iops else 0 + ds['await'] = (ds['await'] * (uploadInter - 1) + wait) / uploadInter + + times = field['10'][1] - field['10'][0] + util = round(times * 100.0 / (secs * 1000), 2) + util = util if util <= 100 else 100.0 + ds['util'] = (ds['util'] * (uploadInter - 1) + util) / uploadInter + + return {'iops': iops*secs, 'bps': bps*secs, 'qusize': qusize*secs, + 'wait': wait, 'util': util} + + + def _checkDiagSwitchChange(self, t): + s = self.diagSwitch[t] + newSW = self.cfg.getCfgItem(t) + if newSW != s['sw'] and newSW == 'on': + for k,v in self.exceptionStat.items(): + if s['esi'] in v.keys(): + v[s['esi']]['max'] = 0 + s['sw'] = newSW + return newSW + + + def _checkIOwaitException(self, iowait): + exceptChk = self.exceptChkDicts['system'] + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + diagnose = self.diagnose + uploadInter = self.uploadInter + + if iowait >= self.cfg.getCfgItem('iowait'): + exceptChk.disableThreshComp('iowait') + + dataStat['iowait'] = \ + (dataStat['iowait'] * (uploadInter - 1) + iowait) / uploadInter + + # Detect iowait exception + minThresh = self.cfg.getCfgItem('iowait') + iowaitThresh = max(exceptChk.getDynThresh('iowait'), minThresh) + if iowait >= iowaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIowait') + rDiagValid = diagnose.recentDiagnoseValid('iowait') + # Configure iowait diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('iowait') + iowaitArg = max(int(iowait * 0.25), minThresh) + diagnose.setUpDiagnose('system', 'iowait', nrSample, iowaitArg) + + exceptChk.updateDynThresh('iowait', iowait) + + def _checkIoburstException(self, devname, es, bps, iops, exceptChk): + bpsLowW = self.cfg.getCfgItem('bps') + bpsHighW = max(exceptChk.getDynThresh('bps'), bpsLowW) + bpsMiddW = max(bpsLowW, bpsHighW / 2) + iopsLowW = self.cfg.getCfgItem('iops') + iopsHighW = max(exceptChk.getDynThresh('iops'), iopsLowW) + iopsMiddW = max(iopsLowW, iopsHighW / 2) + ioburst = exception = False + + if iops >= iopsMiddW or bps >= bpsMiddW: + ioburst = True + bpsOver = True if bps >= bpsHighW else False + iopsOver = True if iops >= iopsHighW else False + + if iopsOver or bpsOver: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIoburst') + rDiagValid = self.diagnose.recentDiagnoseValid('ioutil') + # Configure IO load diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + bpsArg = iopsArg = 0 + if bpsOver == True: + bpsArg = max(int(bps * 0.25), bpsLowW) + if iopsOver == True: + iopsArg = max(int(iops * 0.7), iopsLowW) + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose( + devname, 'ioutil', nrSample, bpsArg, iopsArg) + return ioburst + + def _checkIohangException( + self, devname, es, util, qusize, iops, exceptChk): + # Detect IO hang + if util >= 100 and qusize >= 1 and iops < 50: + # Configure IO hang diagnosis + if self.diagnose.isException(devname, 'iohang') == True: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIohang') + rDiagValid = self.diagnose.recentDiagnoseValid('iohang') + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose(devname, 'iohang', nrSample) + + def _checkUtilException(self, devname, util, iops, bps, qusize): + exceptChk = self.exceptChkDicts[devname] + exceptionStat = self.exceptionStat[devname] + diagnose = self.diagnose + ioburst = False + + utilMinThresh = self.cfg.getCfgItem('util') + utilThresh = max(exceptChk.getDynThresh('util'), utilMinThresh) + if util >= utilThresh: + es = exceptionStat['IO-Burst'] + ioburst = \ + self._checkIoburstException(devname, es, bps, iops, exceptChk) + if not ioburst: + es = exceptionStat['IO-Hang'] + self._checkIohangException( + devname, es, util, qusize, iops, exceptChk) + exceptChk.updateDynThresh('util', util) + exceptChk.updateDynThresh('iops', iops) + exceptChk.updateDynThresh('bps', bps) + return ioburst + + + def _checkAwaitException(self, devname, wait, ioburst): + exceptChk = self.exceptChkDicts[devname] + es = self.exceptionStat[devname]['IO-Delay'] + diagnose = self.diagnose + + awaitMinThresh = self.cfg.getCfgItem('await') + awaitThresh = max(exceptChk.getDynThresh('await'), awaitMinThresh) + if wait >= awaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIolat') + rDiagValid = diagnose.recentDiagnoseValid('iolatency') + # Configuring IO delay diagnostics + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('await') + waitArg = max(int(wait * 0.4), awaitMinThresh) + diagnose.setUpDiagnose( + devname, 'iolatency', nrSample, waitArg, ioburst) + exceptChk.updateDynThresh('await', wait) + + + def _reportDataToRemote(self, devList): + # report datastat&&exception to database + nCycle = 1000.0 / float(self.cfg.getCfgItem('cycle')) + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + + putIdx = ',idx_type=system_Indicator,devname=system ' + putField = 'iowait=%f,iowaithighCnt=%f' %( + dataStat['iowait'], es['cur'] / nCycle) + self._sender.puts(self._nfPutTlb4System + putIdx + putField) + + es['max'] = max(es['max'] if es['cur'] else 0, es['cur']) + es['cur'] = 0 + cur = {'IO-Delay':0, 'IO-Burst':0, 'IO-Hang':0} + for devname in devList: + dataStat = self.dataStat[devname] + es = self.exceptionStat[devname] + for type in cur.keys(): + cur[type] = int(es[type]['cur']) / nCycle + es[type]['max'] = \ + max(es[type]['max'] if cur[type] else 0, cur[type]) + es[type]['cur'] = 0 + + putIdx = ',idx_type=iostat_Indicator,devname=%s ' % devname + putField = 'await=%f,util=%f,iops=%f,bps=%f,qusize=%f' %( + dataStat['await'], dataStat['util'], dataStat['iops'], + dataStat['bps'] / 1024.0, dataStat['qusize'] + ) + putField += ',iodelayCnt=%f,ioburstCnt=%f,iohangCnt=%f' %( + cur['IO-Delay'], cur['IO-Burst'], cur['IO-Hang']) + self._sender.puts(self._nfPutTlb + putIdx + putField) + + + def _collectBegin(self): + fieldDicts = self.fieldDicts + + # collect iowait begin + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + # collect iostat begin + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + + if stat[2] in fieldDicts.keys(): + field = fieldDicts[stat[2]] + else: + field = { + '1': [0, 0], '3': [0, 0], '4': [0, 0], + '5': [0, 0], '7': [0, 0], '8': [0, 0], + '10': [0, 0], '11': [0, 0]} + # add data staticsis for per-disk + fieldDicts.setdefault(stat[2], field) + self._addMonitorAttrForDisk(stat[2]) + + for idx, value in field.items(): + value[0] = long(stat[int(idx) + 2]) + + + def _collectEnd(self, secs): + fieldDicts = self.fieldDicts + exceptChkDicts = self.exceptChkDicts + uploadInter = self.uploadInter + + self.uploadInter = \ + 1 if ((uploadInter * secs) % 60) == 0 else (uploadInter + 1) + + # Calculate iowait + iowait = self._calcIowait() + # Detect iowait exception + self._checkIOwaitException(iowait) + + # collect iostat end + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + for idx, value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx) + 2]) + + for devname, field in fieldDicts.items(): + io = self._calcIoIndex(devname, field, secs) + if io['util'] >= self.cfg.getCfgItem('util'): + # There is IO Burst at present, turn off threshold compensation + self._disableThreshComp(devname, io['qusize']) + # Detect util exception + ioburst = self._checkUtilException( + devname, io['util'], io['iops'], io['bps'], io['qusize']) + # Detect await exception + self._checkAwaitException(devname, io['wait'], ioburst) + + if ((self.uploadInter * secs) % 60) == 0: + self._reportDataToRemote(fieldDicts.keys()) + + + def monitor(self): + while True: + secs = self.cfg.getCfgItem('cycle') / 1000.0 + self._collectBegin() + time.sleep(secs) + self._collectEnd(secs) + # Check if it is necessary to start the diagnosis + self.diagnose.checkDiagnose() + self.fDiskStats.close() + diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py new file mode 100755 index 00000000..921a3435 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import argparse +import signal +from ioMonCfgClass import ioMonCfgClass +from ioMonitorClass import ioMonitorClass + +setcfg_descripton = """set monitor cfg, like -s \'xxxx=xxx,xxx=xxx\' +iowait, The min cpu-iowait not report exceptions(1~100, default 5). +await, The min partition-await not report exceptions(N ms, default 10). +util, The min partition-util not report exceptions(1~100, default 20). +bps, The min partition-bps not report exceptions(bytes, default 30MB). +iops, The min partition-iops not report exceptions(default 150). +cycle, The cycle of Monitoring data collection(default 500ms). +diagIowait, Disable or enable diagnose while reporting iowait exceptions(default on). +diagIolat, Disable or enable diagnose while reporting latency exceptions(default on). +diagIoburst, Disable or enable diagnose while reporting ioburst exceptions(default on). +diagIohang, Disable or enable diagnose while reporting iohang exceptions(default on). +""" + +def getRunArgsFromYaml(yamlF): + logRootPath = '' + pipeFile = '' + with open(yamlF) as f: + lines = f.readlines() + for l in lines: + if not l.startswith('#'): + if 'proc_path:' in l: + logRootPath = l.split()[1].strip('\n') + elif 'outline:' in l: + pipeFile = lines[lines.index(l) + 1].split()[1].strip('\n') + if logRootPath and pipeFile: + break + if not logRootPath or not pipeFile: + raise ValueError( + 'Unable to get labels \"proc_path\" and \"outline\" in %s' % yamlF) + return logRootPath+'/run',pipeFile + + +def main(): + examples = """e.g. + ./ioMonitorMain.py -y [yaml_file] + Start ioMonitor + ./ioMonitorMain.py -y [yaml_file] --reset_cfg --only_set_cfg + Only reset cfg to default + ./ioMonitorMain.py -y [yaml_file] -s 'iowait=10,iops=200,diagIowait=on' --only_set_cfg + Only set min-iowait&&min-iops and disable iowait diagnose to config. + """ + parser = argparse.ArgumentParser( + description="start ioMonitor.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-y','--yaml_file', + help='Specify the socket pipe for data upload'\ + ' and exception log path') + parser.add_argument('-s','--set_cfg', help=setcfg_descripton) + parser.add_argument('-r','--reset_cfg', action='store_true', + help='Reset cfg to default') + parser.add_argument('-o','--only_set_cfg', action='store_true', + help='Only set cfg') + args = parser.parse_args() + + signal.signal(signal.SIGCHLD, signal.SIG_IGN) + logRootPath,pipeFile = getRunArgsFromYaml(args.yaml_file) + if args.only_set_cfg: + if not os.path.exists(logRootPath+'/ioMon/ioMonCfg.json'): + print("%s" % ("config fail, not found ioMonCfg.json")) + return + if args.set_cfg is None and not args.reset_cfg: + print("%s" % ("--set_cfg or --reset_cfg not found.")) + return + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMonCfg.notifyIoMon() + return + + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMon = ioMonitorClass(logRootPath, ioMonCfg, pipeFile) + ioMon.monitor() + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/nfPut.py b/source/tools/monitor/ioMonitor/ioMon/nfPut.py new file mode 100755 index 00000000..5fcaea96 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/nfPut.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： nfPut + Description : + Author : liaozhaoyan + date： 2022/4/28 +------------------------------------------------- + Change Activity: + 2022/4/28: +------------------------------------------------- +""" +__author__ = 'liaozhaoyan' + +import os +import socket +import requests +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self, pipeFile): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = pipeFile + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check unity is running.") + + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut("/tmp/sysom") + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py new file mode 100755 index 00000000..f01dbf27 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import string +import re +from subprocess import PIPE, Popen + + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def echoFile(filename, txt): + execCmd("echo \'"+txt+"\' > "+filename) + + +def echoFileAppend(filename, txt): + execCmd("echo \'"+txt+"\' >> "+filename) + + +def humConvert(value, withUnit): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + + for i in range(len(units)): + if (value / size) < 1: + if withUnit: + return "%.1f%s/s" % (value, units[i]) + else: + return "%.1f" % (value) + value = value / size + + +def getDevt(devname): + try: + with open('/sys/class/block/' + devname + '/dev') as f: + dev = f.read().split(':') + return ((int(dev[0]) << 20) + int(dev[1])) + except Exception: + return -1 + + +def getDevtRegion(devname): + if os.path.exists('/sys/block/'+devname): + isPart = False + elif os.path.exists('/sys/class/block/'+devname): + isPart = True + else: + return [-1, -1] + + master = devname if not isPart else \ + os.readlink('/sys/class/block/'+devname).split('/')[-2] + partList = list( + filter(lambda x: master in x, + os.listdir('/sys/class/block/'+master))) + if not partList: + partList = [] + partList.append(master) + return [getDevt(p) for p in partList] + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def getContainerId(pid): + try: + piddir = "/proc/"+str(pid) + with open(piddir+"/cgroup") as f: + # ... + # cpuset,cpu,cpuacct:/docker/e2afa607d8f13e5b1f89d38ee86d86.... + # memory:/docker/e2afa607d8f13e5b1f89d38ee86..... + # ... + cid = f.read().split('memory:')[1].split('\n')[0].rsplit('/',1)[1] + if not cid: + cid = '-' + except Exception: + cid = '-' + return cid + + +def getFullNameFromProcPid(pid, ino): + try: + piddir = "/proc/"+str(pid) + # list the open files of the task + fdList = os.listdir(piddir+"/fd") + for f in fdList: + try: + path = os.readlink(piddir+"/fd/"+f) + if '/dev/' in path or '/proc/' in path or '/sys/' in path: + continue + + if os.path.isfile(path) and os.stat(path).st_ino == int(ino): + return path + except (IOError, EOFError) as e: + continue + except Exception: + pass + return "-" + + +def supportKprobe(name): + file = '/sys/kernel/debug/tracing/available_filter_functions' + with open(file) as f: + ss = f.read() + if ss.find(name) > 0: + return True + return False diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py new file mode 100755 index 00000000..c9755bfe --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py @@ -0,0 +1,219 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import time +import json +import string +from collections import OrderedDict +from common import humConvert + + +class diskstatClass(object): + def __init__(self, devname, utilThresh, json, nodiskStat, Pattern): + self.devname = devname + self.json = json + self.cycle = 1 + self.started = False + self.Pattern = Pattern + self.nodiskStat = nodiskStat + self.utilThresh = int(utilThresh) if utilThresh is not None else 0 + self.fieldDicts = OrderedDict() + self.diskInfoDicts = {} + self.deviceStatDicts = {} + self.f = open("/proc/diskstats") + if json: + self.fJson = open(json, 'w+') + + + def getDevNameByDevt(self, devt): + try: + return self.diskInfoDicts[str(devt)]['partName'] + except Exception: + return '-' + + def getMasterDev(self, devt): + try: + return self.diskInfoDicts[str(devt)]['master'] + except Exception: + return '-' + + def getDiskStatInd(self, disk, key): + return self.deviceStatDicts[disk][key] + + def start(self): + fieldDicts = self.fieldDicts + diskInfoDicts = self.diskInfoDicts + deviceStatDicts = self.deviceStatDicts + + if self.started: + return + self.started = True + self.cycle = time.time() + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + + field = {\ + '1':[0,0], '2':[0,0], '3':[0,0], '4':[0,0],\ + '5':[0,0], '6':[0,0], '7':[0,0], '8':[0,0],\ + '10':[0,0], '11':[0,0]} + for idx,value in field.items(): + value[0] = long(stat[int(idx)+2]) + if stat[2] not in fieldDicts.keys(): + fieldDicts.setdefault(stat[2], field) + path = os.readlink('/sys/class/block/'+stat[2]).split('/') + master = path[-2] + if master not in path[-1]: + master = path[-1] + diskInfoDicts.setdefault( + str((int(stat[0])<<20)+int(stat[1])), + {'partName': stat[2], 'master': master}) + deviceStat = { + 'r_rqm':0, 'w_rqm':0, 'r_iops':0, 'w_iops':0, 'r_bps':0, + 'w_bps':0, 'wait':0, 'r_wait':0, 'w_wait':0, 'util%':-1} + deviceStatDicts.setdefault(stat[2], deviceStat) + else: + deviceStatDicts[stat[2]]['util%'] = -1 + fieldDicts[stat[2]].update(field) + + def stop(self): + fieldDicts = self.fieldDicts + deviceStatDicts = self.deviceStatDicts + self.cycle = max(int(time.time()-self.cycle), 1) + + if not self.started: + return + self.started = False + + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + for idx,value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx)+2]) + + for devname,field in fieldDicts.items(): + if self.devname is not None and devname not in self.devname and \ + self.devname not in devname: + continue + util = round((field['10'][1]-field['10'][0])*100.0/(self.cycle*1000),2) + util = util if util <= 100 else 100.0 + if util < self.utilThresh: + continue + deviceStatDicts[devname]['util%'] = util + r_iops = field['1'][1]-field['1'][0] + deviceStatDicts[devname]['r_iops'] = r_iops + r_rqm = field['2'][1]-field['2'][0] + deviceStatDicts[devname]['r_rqm'] = r_rqm + w_iops = field['5'][1]-field['5'][0] + deviceStatDicts[devname]['w_iops'] = w_iops + w_rqm = field['6'][1]-field['6'][0] + deviceStatDicts[devname]['w_rqm'] = w_rqm + r_bps = (field['3'][1]-field['3'][0]) * 512 + deviceStatDicts[devname]['r_bps'] = r_bps + w_bps = (field['7'][1]-field['7'][0]) * 512 + deviceStatDicts[devname]['w_bps'] = w_bps + r_ticks = field['4'][1]-field['4'][0] + w_ticks = field['8'][1]-field['8'][0] + wait = round((r_ticks+w_ticks)/(r_iops+w_iops), 2) if (r_iops+w_iops) else 0 + deviceStatDicts[devname]['wait'] = wait + r_wait = round(r_ticks / r_iops, 2) if r_iops else 0 + deviceStatDicts[devname]['r_wait'] = r_wait + w_wait = round(w_ticks / w_iops, 2) if w_iops else 0 + deviceStatDicts[devname]['w_wait'] = w_wait + + + def __showJson(self): + deviceStatDicts = self.deviceStatDicts + + statJsonStr = '{\ + "time":"",\ + "diskstats":[]}' + dstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + dstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + for devname,stat in deviceStatDicts.items(): + if stat['util%'] < 0: + continue + dstatJsonStr = '{\ + "diskname":"","r_rqm":0,"w_rqm":0,"r_iops":0,"w_iops":0,\ + "r_bps":0,"w_bps":0,"wait":0,"r_wait":0,"w_wait":0,"util%":0}' + dstatDict = json.loads(dstatJsonStr, object_pairs_hook=OrderedDict) + dstatDict["diskname"] = devname + for key,val in stat.items(): + dstatDict[key] = val + dstatDicts["diskstats"].append(dstatDict) + if len(dstatDicts["diskstats"]) > 0: + data = json.dumps(dstatDicts) + self.writeDataToJson(data) + return + + def show(self): + secs = self.cycle + deviceStatDicts = self.deviceStatDicts + if self.nodiskStat: + return + + if self.enableJsonShow() == True: + self.__showJson() + return + + if self.Pattern: + WrTotalIops = 0 + RdTotalIops = 0 + WrTotalBw = 0 + RdTotalBw = 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (("device-stat:"),"r_rqm","w_rqm","r_iops","w_iops","r_bps",\ + "w_bps","wait","r_wait","w_wait","util%")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for devname,stat in deviceStatDicts.items(): + if (not self.devname and not os.path.exists('/sys/block/'+devname)) or \ + stat['util%'] < 0: + continue + if self.Pattern: + WrTotalIops += stat['w_iops'] + RdTotalIops += stat['r_iops'] + WrTotalBw += stat['w_bps'] + RdTotalBw += stat['r_bps'] + stWbps = humConvert(stat['w_bps'], True).replace('s', stSecs) if stat['w_bps'] else 0 + stRbps = humConvert(stat['r_bps'], True).replace('s', stSecs) if stat['r_bps'] else 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (devname, str(stat['r_rqm']), str(stat['w_rqm']), str(stat['r_iops']), + str(stat['w_iops']), stRbps, stWbps, str(stat['wait']), str(stat['r_wait']), + str(stat['w_wait']), str(stat['util%']))) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + ((WrTotalIops+RdTotalIops), RdTotalIops, WrTotalIops, + (humConvert((WrTotalBw+RdTotalBw), True).replace('s', stSecs) if (WrTotalBw+RdTotalBw > 0) else 0), + (humConvert(RdTotalBw, True).replace('s', stSecs) if RdTotalBw else 0), + (humConvert(WrTotalBw, True).replace('s', stSecs) if WrTotalBw else 0))) + print("") + + def clear(self): + self.f.close() + if self.enableJsonShow(): + self.fJson.close() + + def notCareDevice(self, devname): + if not self.nodiskStat and self.deviceStatDicts[devname]['util%'] < 0: + return True + return False + + def disableShow(self): + deviceStatDicts = self.deviceStatDicts + for devname,stat in deviceStatDicts.items(): + if self.deviceStatDicts[devname]['util%'] >= 0: + return False + return True + + def enableJsonShow(self): + return True if self.json else False + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py new file mode 100755 index 00000000..763de302 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py @@ -0,0 +1,418 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevt,getDevtRegion +from common import humConvert,supportKprobe +from common import execCmd,echoFile,echoFileAppend +from common import getTgid,fixComm,getContainerId,getFullNameFromProcPid +from mmap import PAGESIZE + + +def getMntPath(fileInfoDict): + mntfname = fileInfoDict['mntfname'] + fsmountInfo = fileInfoDict['fsmountinfo'] + + if len(fsmountInfo) <= 0: + return '-' + + if mntfname.isspace() or len(mntfname) == 0: + return fsmountInfo[0].split()[1] + try: + for l in fsmountInfo: + if l.find(mntfname) > -1: + return l.split()[1] + return '-' + except IndexError: + return fsmountInfo[0].split()[1] + + +def getFullName(fileInfoDict): + fileSuffix = '' + + mntdir = getMntPath(fileInfoDict) + if mntdir == '/': + mntdir = '' + + for f in [ + fileInfoDict['d3fname'], fileInfoDict['d2fname'], + fileInfoDict['d1fname'], fileInfoDict['bfname']]: + if f != '/' and f != '(fault)': + fileSuffix += ('/' + f) + if fileInfoDict['d3fname'] != '/' and fileInfoDict['d3fname'] != '(fault)': + fileSuffix = '/...' + fileSuffix + filename = mntdir + fileSuffix + + if '...' in filename: + f = getFullNameFromProcPid( + fileInfoDict['pid'], fileInfoDict['ino']) + if f != '-': + filename = f + return filename + + +class fsstatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, bwThresh, top, + json, nodiskStat, miscStat, Pattern): + super(fsstatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.expression = [] + self.pid = pid + self.miscStat = miscStat + self.devname = devname + self.top = int(top) if top is not None else 99999999 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = getDevtRegion(devname) if devname is not None else [-1, -1] + tracingBaseDir = "/sys/kernel/debug/tracing" + self.kprobeEvent = tracingBaseDir+"/kprobe_events" + self.tracingDir = tracingBaseDir+'/instances/iofsstat4fs' + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.kprobe = [] + self.kprobeArgsFormat = 'dev=+0x10(+0x28(+0x20(%s))):u32 '\ + 'inode_num=+0x40(+0x20(%s)):u64 len=%s:u64 '\ + 'mntfname=+0x0(+0x28(+0x0(+0x10(%s)))):string '\ + 'bfname=+0x0(+0x28(+0x18(%s))):string '\ + 'd1fname=+0x0(+0x28(+0x18(+0x18(%s)))):string '\ + 'd2fname=+0x0(+0x28(+0x18(+0x18(+0x18(%s))))):string '\ + 'd3fname=+0x0(+0x28(+0x18(+0x18(+0x18(+0x18(%s)))))):string %s' + + kprobeArgs = self._getKprobeArgs('None') + self.ftracePaserCommArgs = ' comm=(.*)' if 'comm=' in kprobeArgs else '' + mmapKprobeArgs = self._getKprobeArgs('mmap') + self.fsmountInfo = self._getFsMountInfo() + for entry in self.fsmountInfo: + fstype = entry.split()[2] + self._kprobeReadWrite(fstype, kprobeArgs) + self._kprobeMmap(fstype, mmapKprobeArgs) + if len(self.kprobe) <= 0: + print("%s" % ("error: not available kprobe")) + sys.exit(-1) + self.outlogFormatBase = 10 + + def _kprobeReadWrite(self, fstype, kprobeArgs): + for op in ['write', 'read']: + kPoints = [ + fstype+"_file_"+op+"_iter", fstype+"_file_"+op, + fstype+"_file_aio_"+op, "generic_file_aio_"+op] + if list(set(self.kprobe) & set(kPoints)): + continue + kprobe = None + for k in kPoints: + if supportKprobe(k): + kprobe = k + break + if not kprobe: + if self.enableJsonShow() == False: + print("warnning: not available %s kprobe" % op) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _kprobeMmap(self, fstype, kprobeArgs): + for kprobe in [fstype+"_page_mkwrite", 'filemap_fault']: + if kprobe in self.kprobe: + continue + if not supportKprobe(kprobe): + if self.enableJsonShow() == False: + print("not support kprobe %s" % kprobe) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _getKprobeArgs(self, type): + commArgs = '' + vinfo = execCmd('uname -r') + version = vinfo.split('.') + + if type == 'mmap': + offFile = '0xa0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0xa0%s' + offLen = '0x0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0x0%s' + else: + offLen = '0x10' + offFile = '0x0' if int(version[0]) > 3 or ( + int(version[0]) == 3 and int(version[1]) > 10) else '0x8' + if int(version[0]) <= 3: + offLen = '0x8' if int(version[1]) < 13 else '0x18' + + if int(version[0]) > 3: + commArgs = 'comm=$comm' + + re= execCmd('lscpu | grep -E \"Architecture|架构\" | sed \"s/：/:/g\"') + arch = re.split(':')[1].strip() + regs = { + "arm":['(%r0)','(%r1)'], + "x86":['(%di)', '(%si)'], + "aarch64":['(%x0)','(%x1)']} + argv0 = argv1 = '' + for key,val in regs.items(): + if arch.startswith(key): + if type == 'mmap': + argv0 = '+' + (offFile % val[0]) + argv1 = '+' + (offLen % val[0]) + argv2 = argv1 + else: + argv2 = argv0 = '+' + offFile + val[0] + argv1 = '+' + offLen + val[1] + break + if argv0 == '': + raise ValueError('arch %s not support' % arch) + + kprobeArgs = self.kprobeArgsFormat % ( + argv0, argv0, argv1, argv0, argv0, argv0, argv0, argv2, commArgs) + return kprobeArgs + + def _getFsMountInfo(self): + devList = [] + if self.devname is not None: + devList.append('/dev/'+self.devname) + else: + sysfsBlockDirList = os.listdir("/sys/block") + for dev in sysfsBlockDirList: + devList.append('/dev/'+dev) + with open("/proc/mounts") as f: + fsmountInfo = list(filter(lambda x: any( + e in x for e in devList), f.readlines())) + return fsmountInfo + + def config(self): + devt = self.devt + + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + if devt[0] > 0: + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + if devt[0] > 0: + dev = getDevt(self.devname) + if dev == min(devt): + echoFile(filterKprobe, + "dev>="+str(min(devt))+"&&dev<="+str(max(devt))) + else: + echoFile(filterKprobe, "dev=="+str(dev)) + echoFile(enableKprobe, "1") + fmt = execCmd("grep print "+self.kprobeDir+"/"+probe+"/format") + matchObj = re.match(r'(.*) dev=(.*) inode_num=(.*)', fmt) + if 'x' in matchObj.group(2): + self.outlogFormatBase = 16 + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(fsstatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(fsstatClass, self).stop() + + def clear(self): + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if not os.path.exists(enableKprobe): + continue + echoFile(enableKprobe, "0") + if self.devt[0] > 0: + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + super(fsstatClass, self).clear() + + def _paserTraceToStat(self, traceText): + bwTotal = 0 + stat = {} + mStat = {} + fileInfoDict = { + 'device': 0, 'mntfname': '', 'bfname': '', 'd1fname': '', + 'd2fname': '', 'd3fname': '', 'fsmountinfo': '', 'ino': 0, + 'pid': 0} + commArgs = self.ftracePaserCommArgs + hasCommArgs = True if len(commArgs) else False + + # pool-1-thread-2-5029 [002] .... 5293018.252338: p_ext4_file_write_iter_0:\ + # (ext4_file_write_iter+0x0/0x6d0 [ext4]) dev=265289729 inode_num=530392 len=38 + # ... + for entry in traceText: + if ('dev=' not in entry) or ('.so' in entry and 'lib' in entry) or ( + '=\"etc\"' in entry) or ('=\"usr\"' in entry and ( + '=\"bin\"' in entry or '=\"sbin\"' in entry)): + continue + + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) dev=(.*) inode_num=(.*) len=(.*)'+ + ' mntfname=(.*) bfname=(.*) d1fname=(.*) d2fname=(.*)'+ + ' d3fname=(.*)'+commArgs, entry) + if matchObj is None: + continue + + pid = (matchObj.group(1).rsplit('-', 1))[1].strip() + dev = int(matchObj.group(4), self.outlogFormatBase) + if (self.pid is not None and int(pid) != self.pid) or \ + str(dev) == '0': + continue + + if hasCommArgs: + comm = matchObj.group(12).strip("\"") + else: + comm = (matchObj.group(1).rsplit('-', 1))[0].strip() + comm = fixComm(comm, pid) + if '..' in comm: + continue + + device = self.getDevNameByDevt(dev) + if device == '-': + continue + if self.miscStat is not None: + disk = self.getMasterDev(dev) + if not mStat.has_key(disk): + mStat.setdefault(disk, {}) + stat = mStat[disk] + + ino = int(matchObj.group(5), self.outlogFormatBase) + inoTask = str(ino)+':'+str(comm)+':'+device + if not stat.has_key(inoTask): + fsmountinfo = [f for f in self.fsmountInfo if ('/dev/'+device) in f] + fileInfoDict['device'] = device + fileInfoDict['mntfname'] = matchObj.group(7).strip("\"") + fileInfoDict['bfname'] = matchObj.group(8).strip("\"") + fileInfoDict['d1fname'] = matchObj.group(9).strip("\"") + fileInfoDict['d2fname'] = matchObj.group(10).strip("\"") + fileInfoDict['d3fname'] = matchObj.group(11).strip("\"") + fileInfoDict['fsmountinfo'] = fsmountinfo + fileInfoDict['ino'] = ino + fileInfoDict['pid'] = pid + stat.setdefault(inoTask, + {"inode":str(ino), "comm": comm, "tgid": getTgid(pid), "pid": pid, + "cnt_wr": 0, "bw_wr": 0, "cnt_rd": 0, "bw_rd": 0, "device": device, + "cid":getContainerId(pid), "file": getFullName(fileInfoDict)}) + + size = int(matchObj.group(6), self.outlogFormatBase) + if 'filemap_fault' in entry or 'page_mkwrite' in entry: + size = PAGESIZE + if 'write' in entry or 'page_mkwrite' in entry: + stat[inoTask]["cnt_wr"] += 1 + stat[inoTask]["bw_wr"] += int(size) + if 'read' in entry or 'filemap_fault' in entry: + stat[inoTask]["cnt_rd"] += 1 + stat[inoTask]["bw_rd"] += int(size) + if pid != stat[inoTask]["pid"]: + stat[inoTask]["pid"] = pid + stat[inoTask]["tgid"] = getTgid(pid) + if stat[inoTask]["cid"] == '-': + stat[inoTask]["cid"] = getContainerId(pid) + bwTotal += int(size) + return bwTotal,stat,mStat + + def _joinMiscStat(self, mStat): + for d,val in self.miscStat: + if d not in mStat.keys(): + mStat.setdefault(d, {}) + mStat[d].update(dict(val)) + tmpStat = [] + for d,val in mStat.items(): + idxSort = 'bw_wr' + if self.getDiskStatInd(d, 'w_iops') < self.getDiskStatInd(d, 'r_iops'): + idxSort = 'bw_rd' + s = sorted( + val.items(), key=lambda e: (e[1][idxSort]), reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return 0 + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","fsstats":[]}' + fstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + fstatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + fsstatJsonStr = '{\ + "inode":0,"comm":"","tgid":0,"pid":0,"cnt_rd":0,\ + "bw_rd":0,"cnt_wr":0,"bw_wr":0,"device":0,"cid":0,"file":0}' + fsstatDict = json.loads( + fsstatJsonStr, object_pairs_hook=OrderedDict) + for key, val in item.items(): + fsstatDict[key] = val + fstatDicts["fsstats"].append(fsstatDict) + if len(fstatDicts["fsstats"]) > 0: + self.writeDataToJson(json.dumps(fstatDicts)) + + def printStat(self, stat): + secs = self.cycle + print("%-20s%-8s%-8s%-24s%-8s%-12s%-8s%-12s%-12s%-12s%-32s%s" + % ("comm", "tgid", "pid", "cid", "cnt_rd", "bw_rd", "cnt_wr", + "bw_wr", "inode", "device", "filepath")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + print("%-20s%-8s%-8s%-24s%-8d%-12s%-8d%-12s%-12s%-12s%s" + % (item["comm"], item["tgid"], item["pid"], item["cid"][0:20], + item["cnt_rd"], item["bw_rd"], item["cnt_wr"], item["bw_wr"], + item["inode"], item["device"], item["file"])) + print("") + + def show(self): + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = f.read().split('\n') + #traceText = f.readlines() + #traceText = \ + # list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + bwTotal,stat,mStat = self._paserTraceToStat(traceText) + + if self.miscStat is not None: + return self._joinMiscStat(mStat) + elif (self.bwThresh and (bwTotal/secs) < self.bwThresh): + return + + stat = sorted(stat.items(), key=lambda e: ( + e[1]["bw_wr"]+e[1]["bw_rd"]), reverse=True)[:self.top] + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + if self.disableShow() == False: + super(fsstatClass, self).show() + + if self.enableJsonShow() == True: + self.showJson(stat) + else: + self.printStat(stat) + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py new file mode 100755 index 00000000..18290315 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import threading +from collections import OrderedDict +from iostatClass import iostatClass +from fsstatClass import fsstatClass +from promiscClass import promiscClass +import time + +global_iofsstat_stop = False +def signal_exit_handler(signum, frame): + global global_iofsstat_stop + global_iofsstat_stop = True + +def exit_handler(): + global global_iofsstat_stop + global_iofsstat_stop = True + +def iofsstatStart(argv): + global global_iofsstat_stop + global_iofsstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iofsstat.py -d vda -c 1 + Report iops and bps of process for vda per 1secs + ./iofsstat.py -d vda1 --fs -c 1 + Report fs IO-BW and file of process for vda1(must be parttion mounted by filesystem) per 1secs + ./iofsstat.py -m -c 5 -t 5 + Report top5 iops&&bps&&file of process with misc mode per 5secs + ./iofsstat.py -d vda -c 1 -b 1048576 -i 350 + Report process that iops over 350 or bps over 1048576 for vda per 1secs + ./iofsstat.py -u 90 + Report disk that io-util over %90 + """ + parser = argparse.ArgumentParser( + description="Report IO statistic for partitions.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-T','--Timeout', help='Specify the timeout for program exit(secs).') + parser.add_argument('-t','--top', help='Report the TopN with the largest IO resources.') + parser.add_argument('-u','--util_thresh', help='Specify the util-thresh to report.') + parser.add_argument('-b','--bw_thresh', help='Specify the BW-thresh to report.') + parser.add_argument('-i','--iops_thresh', help='Specify the IOPS-thresh to report.') + parser.add_argument('-c','--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-d','--device', help='Specify the disk name.') + parser.add_argument('-p','--pid', help='Specify the process id.') + parser.add_argument('-j','--json', help='Specify the json-format output.') + parser.add_argument('-f','--fs', action='store_true', + help='Report filesystem statistic for partitions.') + parser.add_argument('-P','--Pattern', action='store_true', + help='Report IO pattern(--fs not support).') + parser.add_argument('-n','--nodiskStat', action='store_true', + help='Not report disk stat.') + parser.add_argument('-m','--misc', action='store_true', + help='Promiscuous mode.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + secs = float(args.cycle) if args.cycle is not None else 0 + devname = args.device + pid = int(args.pid) if args.pid else None + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + interval = secs if loop == True else 1 + if args.misc: + c = promiscClass(devname, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, + args.Pattern) + else: + if args.fs: + c = fsstatClass(devname, pid, args.util_thresh, args.bw_thresh, + args.top, args.json, args.nodiskStat, None, args.Pattern) + else: + c = iostatClass(devname, pid, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, None, + args.Pattern) + c.config() + while global_iofsstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iofsstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py new file mode 100755 index 00000000..e7d03e85 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py @@ -0,0 +1,244 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevtRegion +from common import humConvert,echoFile +from common import getContainerId + + +class iostatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, iopsThresh, bwThresh, + top, json, nodiskStat, miscStat, Pattern): + super(iostatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.pid = pid + self.miscStat = miscStat + self.top = int(top) if top is not None else 99999999 + self.iopsThresh = int(iopsThresh) if iopsThresh is not None else 0 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = min(getDevtRegion(devname)) if devname is not None else -1 + self.tracingDir = "/sys/kernel/debug/tracing/instances/iofsstat4io" + self.blkTraceDir = self.tracingDir+"/events/block" + + def config(self): + devt = self.devt + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + if devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "dev=="+str(devt)) + else: + echoFile(self.blkTraceDir+"/block_getrq/filter", "") + echoFile(self.blkTraceDir+"/block_getrq/enable", "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(iostatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(iostatClass, self).stop() + + def clear(self): + echoFile(self.blkTraceDir+"/block_getrq/enable", "0") + if self.devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "0") + super(iostatClass, self).clear() + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","iostats":[]}' + iostatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iostatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + iostatJsonStr = '{\ + "comm":"","pid":0,"bps_rd":0,"iops_rd":0,"iops_wr":0,"bps_wr":0,"device":0}' + iostatDict = json.loads(iostatJsonStr, object_pairs_hook=OrderedDict) + for key in ['comm', 'pid', 'bps_rd', 'iops_rd', 'iops_wr', 'bps_wr', 'device']: + iostatDict[key] = item[key] + iostatDicts["iostats"].append(iostatDict) + if len(iostatDicts["iostats"]) > 0: + self.writeDataToJson(json.dumps(iostatDicts)) + + def patternIdx(self, size): + dp = [ + ("pat_W4K", (4*1024)), ("pat_W16K", (16*1024)), + ("pat_W32K", (32*1024)), ("pat_W64K",(64*1024)), + ("pat_W128K", (128*1024)), ("pat_W256K", (256*1024)), + ("pat_W512K", (512*1024))] + for d in dp: + if size <= d[1]: + return d[0] + return 'pat_Wlarge' + + def patternPercent(self, pat, total): + if total == 0 or pat == 0: + return '0' + return format(pat / (total * 1.0) * 100, '.2f') + '%' + + def show(self): + iopsTotal = 0 + WrIopsTotal = 0 + RdIopsTotal = 0 + bwTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + stat = {} + mStat = {} + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = list( + filter(lambda x: 'block_getrq' in x, f.readlines())) + # jbd2/vda1-8-358 ... : block_getrq: 253,0 WS 59098136 + 120 [jbd2/vda1-8] + for entry in traceText: + oneIO = entry.split() + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) \[([^\[\]]*)\]\n', entry) + comm = matchObj.group(4) + pid = matchObj.group(1).rsplit('-', 1)[1].strip() + if self.pid is not None and int(pid) != self.pid: + continue + devinfo = oneIO[-6-comm.count(' ')].split(',') + dev = ((int(devinfo[0]) << 20) + int(devinfo[1])) + if str(dev) == '0': + continue + device = self.getDevNameByDevt(dev) + if device == '-' or self.notCareDevice(device) == True: + continue + if self.miscStat is not None: + if not mStat.has_key(device): + mStat.setdefault(device, {}) + stat = mStat[device] + iotype = oneIO[-5-comm.count(' ')] + sectors = oneIO[-2-comm.count(' ')] + task = str(pid)+':'+device + if bool(stat.has_key(task)) != True: + stat.setdefault(task, + {"comm":"", "pid": pid, "iops_rd": 0, + "iops_wr": 0, "bps_rd": 0, "bps_wr": 0, + "flushIO": 0, "device": device, + "cid":getContainerId(pid), + "pat_W4K":0, "pat_W16K":0, "pat_W32K":0, + "pat_W64K":0, "pat_W128K":0, "pat_W256K":0, + "pat_W512K":0, "pat_Wlarge":0}) + size = int(sectors) * 512 + if len(comm) > 0: + stat[task]["comm"] = comm + if 'R' in iotype: + stat[task]["iops_rd"] += 1 + stat[task]["bps_rd"] += size + bwTotal += size + iopsTotal += 1 + if 'W' in iotype: + stat[task]["iops_wr"] += 1 + stat[task]["bps_wr"] += size + bwTotal += size + iopsTotal += 1 + if self.Pattern and size > 0 and size < 1024 * 1024 * 100: + stat[task][self.patternIdx(size)] += 1 + if 'F' in iotype: + stat[task]["flushIO"] += 1 + + if self.iopsThresh or self.bwThresh: + if (self.bwThresh and bwTotal >= self.bwThresh) or \ + (self.iopsThresh and iopsTotal >= self.iopsThresh): + pass + else: + return + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + super(iostatClass, self).show() + + if self.miscStat is not None: + tmpStat = [] + for d,val in mStat.items(): + s = sorted(val.items(), + key=lambda e: (e[1]["bps_wr"]+e[1]["bps_rd"]), + reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return + + stat = sorted(stat.items(), + key=lambda e: (e[1]["iops_rd"] + e[1]["iops_wr"]), + reverse=True)[:self.top] + + if self.enableJsonShow() == True: + self.showJson(stat) + return + + tPattern = '' + if self.Pattern: + WrIopsTotal = 0 + RdIopsTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + tPattern = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + "pat_W4K", "pat_W16K", "pat_W32K", "pat_W64K", "pat_W128K", + "pat_W256K", "pat_W512K", "pat_Wlarge" + )) + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", tPattern)) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + patPercent = '' + if self.Pattern: + WrIopsTotal += item["iops_wr"] + RdIopsTotal += item["iops_rd"] + WrBwTotal += item["bps_wr"] + RdBwTotal += item["bps_rd"] + patPercent = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + self.patternPercent(item["pat_W4K"], item["iops_wr"]), + self.patternPercent(item["pat_W16K"], item["iops_wr"]), + self.patternPercent(item["pat_W32K"], item["iops_wr"]), + self.patternPercent(item["pat_W64K"], item["iops_wr"]), + self.patternPercent(item["pat_W128K"], item["iops_wr"]), + self.patternPercent(item["pat_W256K"], item["iops_wr"]), + self.patternPercent(item["pat_W512K"], item["iops_wr"]), + self.patternPercent(item["pat_Wlarge"], item["iops_wr"]) + )) + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + patPercent += item["cid"] + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % (item["comm"], + str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], + item["device"], patPercent)) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + (iopsTotal, RdIopsTotal, WrIopsTotal, + (humConvert(bwTotal, True).replace('s', stSecs) if bwTotal else 0), + (humConvert(RdBwTotal, True).replace('s', stSecs) if RdBwTotal else 0), + (humConvert(WrBwTotal, True).replace('s', stSecs) if WrBwTotal else 0))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py new file mode 100755 index 00000000..b3f90fc5 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py @@ -0,0 +1,215 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from common import humConvert +from iostatClass import iostatClass +from fsstatClass import fsstatClass + + +class promiscClass(): + def __init__( + self, devname, utilThresh, iopsThresh, bwThresh, top, json, + nodiskStat, Pattern): + self._iostat = [] + self._fsstat = [] + self.fs = fsstatClass(devname, None, utilThresh, bwThresh, + top, json, nodiskStat, self._fsstat, Pattern) + self.io = iostatClass(devname, None, utilThresh, iopsThresh, + bwThresh, top, json, nodiskStat, self._iostat, + Pattern) + + + def _selectKworker(self, iostat, fsItem, kworker): + select = None + largeFound = False + diff = sys.maxsize + for k in kworker: + fsRestBw = fsItem["bw_wr"] + if 'restBW' in fsItem.keys(): + fsRestBw = fsItem["restBW"] + if fsRestBw > (iostat[k]["bps_wr"] * 15) or \ + (fsRestBw * 50) < iostat[k]["bps_wr"]: + continue + d = abs(fsItem["bw_wr"] - iostat[k]["bps_wr"]) + diff = min(d, diff) + if iostat[k]["bps_wr"] > fsItem["bw_wr"]: + if not largeFound or diff == d: + select = k + largeFound = True + continue + if not largeFound and diff == d: + select = k + return select + + + def _addBioToKworker(self, iostat, kworker, fsItem): + repeated = False + k = self._selectKworker(iostat, fsItem, kworker) + if not k: + return False, 0 + if 'bufferio' not in iostat[k].keys(): + iostat[k].setdefault('bufferio', []) + task = fsItem["comm"]+':'+str(fsItem["tgid"])+':'+str(fsItem["pid"])+\ + ':'+fsItem["cid"][0:20] + bio = {'task': task, 'Wrbw': fsItem["bw_wr"], 'file': fsItem["file"], + 'device': fsItem["device"]} + for d in iostat[k]["bufferio"]: + if task == d['task'] and d['file'] == bio['file'] and \ + d['device'] == bio['device']: + d['Wrbw'] = max(d['Wrbw'], bio["Wrbw"]) + repeated = True + break + if not repeated: + iostat[k]["bufferio"].append(bio) + return True, iostat[k]["bps_wr"] + + + def _checkDeleteItem(self, addOK, costBW, item): + now = time.time() + # After 10 secs without adding to any kworker, we will delete the fsItem + agingTime = 10 + if 'restBW' not in item.keys(): + item.setdefault('restBW', item["bw_wr"]) + item.setdefault('startAging', now) + if addOK: + item["startAging"] = time.time() + item["restBW"] = item["restBW"] - costBW if addOK else item["restBW"] + if item["restBW"] <= 0 or (item["restBW"] < item["bw_wr"] and \ + (now - item["startAging"]) >= agingTime): + return True + return False + + + def _miscIostatFromFsstat(self): + fsstats = self._fsstat + iostats = dict(self._iostat) + for disk, fsItems in fsstats: + if disk not in iostats.keys(): + continue + rmList = [] + iostat = dict(iostats[disk]) + kworker = [key for key,val in iostat.items() if 'kworker' in val['comm']] + for key, item in fsItems: + taskI = item["pid"]+':'+disk + if taskI in iostat.keys(): + if 'file' not in iostat[taskI].keys(): + iostat[taskI].setdefault('file', []) + iostat[taskI]['cid'] = item['cid'] + iostat[taskI]["file"].append(item["file"]) + if item["bw_wr"] <= (iostat[taskI]["bps_wr"] * 15): + rmList.append((key, item)) + continue + if kworker: + if item["bw_wr"] < item["bw_rd"]: + rmList.append((key, item)) + continue + addOK,cost = self._addBioToKworker(iostat, kworker, item) + deleted = self._checkDeleteItem(addOK, cost, item) + if deleted: + rmList.append((key, item)) + for key, item in rmList: + fsItems.remove((key, item)) + iostats[disk] = iostat + return iostats + + + def _miscShowJson(self, iostats): + secs = self.io.cycle + statJsonStr = '{"time":"","mstats":[]}' + mstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + mstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + if 'file' not in item.keys(): + item.setdefault('file', '-') + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + mstatDicts["mstats"].append(item) + if len(mstatDicts["mstats"]) > 0: + self.io.writeDataToJson(json.dumps(mstatDicts)) + + + def miscShow(self): + secs = self.io.cycle + if not self._fsstat and not self._iostat: + return + + iostats = self._miscIostatFromFsstat() + if not iostats: + return + tmp = {} + for d in iostats.values(): + tmp.update(dict(d)) + iostats = sorted( + tmp.items(), + key=lambda e: (int(e[1]["bps_rd"])+int(e[1]["bps_wr"])), + reverse=True) + if self.io.enableJsonShow() == True: + self._miscShowJson(iostats) + return + + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", "file")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + file = str(item["file"]) if 'file' in item.keys() else '-' + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + (item["comm"], str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], item["device"], file)) + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + print(' |----%-32s WrBw:%-12s Device:%-8s File:%s' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + print("") + + + def config(self): + self.fs.config() + self.io.config() + + def start(self): + self.clear() + self.fs.start() + self.io.start() + + def stop(self): + self.fs.stop() + self.io.stop() + + def clear(self): + del self._iostat[:] + + def show(self): + self.fs.show() + self.io.show() + self.miscShow() + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py new file mode 100755 index 00000000..69f473c8 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py @@ -0,0 +1,354 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +import re +import json +import threading +from collections import OrderedDict +from subprocess import PIPE, Popen +import shlex + +global_iowaitstat_stop = False + + +def signal_exit_handler(signum, frame): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def exit_handler(): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def echoFile(filename, txt): + os.system("echo \""+txt+"\" > "+filename) + + +def echoFileAppend(filename, txt): + os.system("echo \""+txt+"\" >> "+filename) + + +def supportKprobe(name): + cmd = "cat /sys/kernel/debug/tracing/available_filter_functions | grep " + name + ss = execCmd(cmd).strip() + for res in ss.split('\n'): + if ':' in res: + res = res.split(":", 1)[1] + if ' [' in res: # for ko symbol + res = res.split(" [", 1)[0] + if res == name: + return True + return False + +class iowaitClass(): + def __init__(self, pid, cycle, top, json, iowait_thresh): + self.pid = pid + self.top = int(top) if top is not None else 99999999 + self.json = json + self.cycle = cycle + self.iowait_thresh = int(iowait_thresh) if iowait_thresh is not None else 0 + self.kprobeEvent = "/sys/kernel/debug/tracing/kprobe_events" + self.tracingDir = "/sys/kernel/debug/tracing/instances/iowait" + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.expression = [] + self.kprobe = [] + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + if json: + self.fJson = open(json, 'w+') + + for kprobe,retProbe in {'io_schedule_timeout':True, 'io_schedule':True}.items(): + if supportKprobe(kprobe) == False: + print("not available %s kprobe" % kprobe) + continue + self.expression.append('p:p_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('p_%s_0' % kprobe) + if retProbe == True: + self.expression.append('r:r_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('r_%s_0' % kprobe) + if len(self.kprobe) == 0: + print "not available kprobe" + sys.exit(0) + + def config(self): + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + echoFile(enableKprobe, "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + + def clear(self): + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + if self.json: + self.fJson.close() + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') + + def showJson(self, stat, totalTimeout, gloabIowait): + top = 0 + statJsonStr = '{"time":"", "global iowait":0,"iowait":[]}' + iowaitStatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + iowaitStatDicts['global iowait'] = gloabIowait + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + iowaitStatJsonStr = '{"comm":"","pid":0,"tgid":0,"timeout":0,"iowait":0,"reason":0}' + iowaitStatDict = json.loads( + iowaitStatJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDict["comm"] = item["comm"] + iowaitStatDict["pid"] = pid + iowaitStatDict["tgid"] = item["tgid"] + iowaitStatDict["timeout"] = item["timeout"] + iowaitStatDict["iowait"] = iowait + iowaitStatDict["reason"] = reason + iowaitStatDicts["iowait"].append(iowaitStatDict) + if len(iowaitStatDicts["iowait"]) > 0: + self.writeDataToJson(json.dumps(iowaitStatDicts)) + + def show(self): + top = 0 + totalTimeout = 0 + stat = {} + secs = self.cycle + traceText = [] + + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + gloabIowait = float(format( + (statList[4]-self.cpuStatIowait['iowait'])*100.0 / + (sum(statList)-self.cpuStatIowait['sum']), '.2f')) + if gloabIowait < self.iowait_thresh: + return + + with open(self.tracingDir+"/trace") as f: + traceLoglist = list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + traceText = traceLoglist + + # jbd2/vda2-8-605 [001] .... 38890020.539912: p_io_schedule_0: (io_schedule+0x0/0x40) + # jbd2/vda2-8-605 [002] d... 38890020.540633: r_io_schedule_0: (bit_wait_io+0xd/0x50 <- io_schedule) + # <...>-130620 [002] .... 38891029.116442: p_io_schedule_timeout_0: (io_schedule_timeout+0x0/0x40) + # <...>-130620 [002] d... 38891029.123657: r_io_schedule_timeout_0: (balance_dirty_pages+0x270/0xc60 <- io_schedule_timeout) + for entry in traceText: + matchObj = re.match(r'(.*) \[([^\[\]]*)\] (.*) (.*): (.*): (.*)\n', entry) + if matchObj is None: + continue + commInfo = matchObj.group(1).rsplit('-', 1) + pid = commInfo[1].strip() + if self.pid is not None and pid != self.pid: + continue + if bool(stat.has_key(pid)) != True: + comm = fixComm(commInfo[0].lstrip(), pid) + if '..' in comm: + continue + stat.setdefault(pid, + {"comm": comm, "tgid": getTgid(pid), + "timeout": 0, "reason": {}, "entry": []}) + stat[pid]["entry"].append({ + 'time':matchObj.group(4), + 'point':matchObj.group(5), + 'trace':matchObj.group(6)}) + + if stat: + for key,item in stat.items(): + item["entry"] = sorted(item["entry"], key=lambda e: float(e["time"]), reverse=False) + count = 0 + startT = 0 + for entry in item["entry"]: + count += 1 + if (count % 2 != 0 and 'p_' not in entry['point']) or \ + (count % 2 == 0 and 'r_' not in entry['point']): + count = 0 + startT = 0 + continue + + if count % 2 != 0: + startT = float(entry['time']) + continue + + if startT > 0 and float(entry['time']) > startT: + if re.split('[(,+]', entry['trace'])[1] in re.split('[-,)]', entry['trace'])[1]: + count = 0 + startT = 0 + continue + item['timeout'] += (float(entry['time']) - startT) + totalTimeout += (float(entry['time']) - startT) + startT = 0 + if entry['trace'] not in item['reason'].keys(): + item['reason'].setdefault(entry['trace'], 0) + item['reason'][entry['trace']] += 1 + + if stat: + stat = OrderedDict(sorted(stat.items(), key=lambda e: e[1]["timeout"], reverse=True)) + if self.json: + self.showJson(stat, totalTimeout, gloabIowait) + return + else: + head = str(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()))+' -> global iowait%: '+str(gloabIowait) + print head + + print("%-32s%-8s%-8s%-16s%-12s%s" % ("comm", "tgid", "pid", "waitio(ms)", "iowait(%)", "reasons")) + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + print("%-32s%-8s%-8s%-16s%-12s%s" + % (item["comm"], item["tgid"], pid, item["timeout"], iowait, str(reason))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() + +def iowaitstatStart(argv): + global global_iowaitstat_stop + global_iowaitstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iowaitstat.py + Report iowait for tasks + ./iowaitstat.py -c 1 + Report iowait for tasks per secs + ./iowaitstat.py -p [PID] -c 1 + Report iowait for task with [PID] per 1secs + """ + parser = argparse.ArgumentParser( + description="Report iowait for tasks.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-p', '--pid', help='Specify the process id.') + parser.add_argument('-T', '--Timeout', + help='Specify the timeout for program exit(secs).') + parser.add_argument( + '-t', '--top', help='Report the TopN with the largest iowait.') + parser.add_argument('-c', '--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-j', '--json', help='Specify the json-format output.') + parser.add_argument('-w','--iowait_thresh', help='Specify the iowait-thresh to report.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + pid = int(args.pid) if args.pid else None + secs = float(args.cycle) if args.cycle is not None else 0 + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + c = iowaitClass(pid, secs, args.top, args.json, args.iowait_thresh) + c.config() + interval = secs if loop == True else 1 + while global_iowaitstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iowaitstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMonitor.sh b/source/tools/monitor/ioMonitor/ioMonitor.sh new file mode 100755 index 00000000..dfd3d468 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMonitor.sh @@ -0,0 +1,11 @@ +#!/bin/sh +#****************************************************************# +# ScriptName: ioMonitor.sh +# Author: $SHTERM_REAL_USER@alibaba-inc.com +# Create Date: 2021-06-06 16:53 +# Modify Author: $SHTERM_REAL_USER@alibaba-inc.com +# Modify Date: 2021-06-06 16:53 +# Function: +#***************************************************************# +TOOLS_ROOT="$SYSAK_WORK_PATH/tools" +python $TOOLS_ROOT/ioMon/ioMonitorMain.py $* diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index cc3f9066..7a64bdcc 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -125,4 +125,19 @@ metrics: head: value help: "buddyinfo of system from /proc/loadavg" type: "gauge" + - title: sysak_IOMonIndForDisksIO + from: IOMonIndForDisksIO + head: value + help: "Disk IO indicators and abnormal events" + type: "gauge" + - title: sysak_IOMonIndForSystemIO + from: IOMonIndForSystemIO + head: value + help: "System indicators and abnormal events about IO" + type: "gauge" + - title: sysak_IOMonDiagLog + from: IOMonDiagLog + head: value + help: "Diagnose log for IO exception" + type: "gauge" -- Gitee From b62cef584bba1fc3558081e757cc758ab8945c82 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Wed, 22 Feb 2023 10:59:55 +0800 Subject: [PATCH 06/77] unity: bpfsample: add bpfsample which is period sampling and add corresponded doc Signed-off-by: Shuyi Cheng --- .../tools/monitor/unity/beaver/guide/bpf.md | 71 ++++++++----- .../monitor/unity/beaver/guide/bpf_perf.md | 100 ++++++++++++++++++ .../monitor/unity/collector/plugin/Makefile | 2 +- .../unity/collector/plugin/bpfsample/Makefile | 8 ++ .../plugin/bpfsample/bpfsample.bpf.c | 23 ++++ .../collector/plugin/bpfsample/bpfsample.c | 41 +++++++ .../collector/plugin/bpfsample/bpfsample.h | 52 +++++++++ 7 files changed, 270 insertions(+), 27 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/guide/bpf_perf.md create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h diff --git a/source/tools/monitor/unity/beaver/guide/bpf.md b/source/tools/monitor/unity/beaver/guide/bpf.md index 81b67b62..9edada61 100644 --- a/source/tools/monitor/unity/beaver/guide/bpf.md +++ b/source/tools/monitor/unity/beaver/guide/bpf.md @@ -1,13 +1,13 @@ -## 基于 eBPF 的监控开发手册 +## 基于 eBPF 的周期性采样监控开发手册 -我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample` 路径提供了一个基于 eBPF 的周期性采样监控开发样例。其主要包含三个部分： 1. Makefile: 用于编译该工具； -2. bpfsample2.bpf.c: 此处编写 eBPF 程序 -3. bpfsmaple2.c: 此处编写用户态程序 +2. bpfsample.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple.c: 此处编写用户态程序 接下分别介绍这三个部分。 @@ -16,9 +16,9 @@ ```Makefile newdirs := $(shell find ./ -type d) -bpfsrcs := bpfsample2.bpf.c -csrcs := bpfsample2.c -so := libbpfsample2.so +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so include ../bpfso.mk ``` @@ -30,36 +30,32 @@ include ../bpfso.mk 开发者只需要关注上述三个变量的修改即可。 -### bpfsample2.bpf.c: eBPF 程序的编写 +### bpfsample.bpf.c: eBPF 程序的编写 ```c #include #include -#include "bpfsample2.h" +#include "bpfsample.h" -BPF_PERF_OUTPUT(perf, 1024); +BPF_ARRAY(count, u64, 200); SEC("kprobe/netstat_seq_show") int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) { - struct event e = {}; - - e.ns = ns(); - e.cpu = cpu(); - e.pid = pid(); - comm(e.comm); - - bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } return 0; } - ``` -1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 -2. `bpfsample2.h` 是开发者自定义的头文件 +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_ARRAY` 的helper函数，以及内核结构体的定义 +2. `bpfsample.h` 是开发者自定义的头文件 -### bpfsample2.c: 用户态程序的编写 +### bpfsample.c: 用户态程序的编写 unity 监控框架提供了三个函数，分别是： @@ -79,22 +75,45 @@ void deinit(void) } ``` -在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： +在 `init` 函数里，需要去 load, attach eBPF程序。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name);` 。因此，一般 `init` 函数具体形式如下： ```c int init(void *arg) { - return LOAD_SKEL_OBJECT(bpf_sample2, perf);; + return LOAD_SKEL_OBJECT(bpf_sample); +} +``` + +对于 `call` 函数，我们需要周期性去读取 `map` 数据。本样例，在 `call` 函数读取 `count` map里面的数据，去统计事件触发的频次。 + + +```c +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; } ``` -对于 `call` 函数，我们保持不变，即直接 `return 0`。 对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： ```c int deinit(void *arg) { - return DESTORY_SKEL_BOJECT(bpf_sample2); + return DESTORY_SKEL_BOJECT(bpf_sample); } ``` \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/bpf_perf.md b/source/tools/monitor/unity/beaver/guide/bpf_perf.md new file mode 100644 index 00000000..2614c60c --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/bpf_perf.md @@ -0,0 +1,100 @@ + + +## 基于 eBPF 的事件监控开发手册 + + +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 94f8322c..52d02c14 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 bpfsample all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile new file mode 100644 index 00000000..0b8e79a2 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c new file mode 100644 index 00000000..12556f2a --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c @@ -0,0 +1,23 @@ + + +#include +#include +#include "bpfsample.h" + + + +BPF_ARRAY(count, u64, 200); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } + return 0; +} + + + diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c new file mode 100644 index 00000000..18eff585 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c @@ -0,0 +1,41 @@ + + +#include "bpfsample.h" +#include "bpfsample.skel.h" + +#include +#include +#include "../../../../unity/beeQ/beeQ.h" + +DEFINE_SEKL_OBJECT(bpfsample); + +int init(void *arg) +{ + printf("bpfsample plugin install.\n"); + return LOAD_SKEL_OBJECT(bpfsample); +} + +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; +} + +void deinit(void) +{ + printf("bpfsample plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(bpfsample); +} diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h new file mode 100644 index 00000000..9bd981fc --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h @@ -0,0 +1,52 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif -- Gitee From 92fc70eac38ace9287b14de8f7114e909d3a9285 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 06:17:08 +0000 Subject: [PATCH 07/77] unity: Add ebpf nosched for unity Signed-off-by: Hailong Liu --- .../tools/monitor/unity/collector/plugin.yaml | 9 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../collector/plugin/unity_nosched/Makefile | 8 + .../plugin/unity_nosched/unity_nosched.bpf.c | 206 ++++++++++++++++++ .../plugin/unity_nosched/unity_nosched.c | 153 +++++++++++++ .../plugin/unity_nosched/unity_nosched.h | 92 ++++++++ 6 files changed, 468 insertions(+), 2 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 7a64bdcc..6d687b2f 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -32,7 +32,9 @@ plugins: - so: proc_loadavg description: "collect load avg" - + - + so: unity_nosched + description: "nosched:sys hold cpu and didn't scheduling" metrics: - title: sysak_proc_cpu_total @@ -140,4 +142,9 @@ metrics: head: value help: "Diagnose log for IO exception" type: "gauge" + - title: sched_moni_jitter + from: sched_moni_jitter + head: value + help: "nosched:sys hold cpu and didn't scheduling" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 94f8322c..b29bc4e8 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile new file mode 100644 index 00000000..3f88651e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_nosched.bpf.c +csrcs := unity_nosched.c +so := libunity_nosched.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c new file mode 100644 index 00000000..85011166 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c @@ -0,0 +1,206 @@ +#include +#include +#include "sched_jit.h" +#include "unity_nosched.h" + +BPF_PERF_OUTPUT(perf, 1024); + +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BITS_PER_LONG 64 +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +struct bpf_map_def SEC("maps") args_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(int), + .value_size = sizeof(struct args), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 1000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(max_entries, MAX_MONI_NR); + __type(key, u64); + __type(value, struct latinfo); +} info_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +static inline int test_bit(int nr, const volatile unsigned long *addr) +{ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +static inline int test_ti_thread_flag(struct thread_info *ti, int nr) +{ + int result; + unsigned long *addr; + unsigned long tmp = _(ti->flags); + + addr = &tmp; + result = 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); + return result; +} + +static inline int test_tsk_thread_flag_low(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)(BPF_CORE_READ(tsk, stack)); + return test_ti_thread_flag(tfp, flag); +} + +/* + * Note: This is based on + * 1) ->thread_info is always be the first element of task_struct if CONFIG_THREAD_INFO_IN_TASK=y + * 2) ->state now is the most nearly begin of task_struct except ->thread_info if it has. + * return ture if struct thread_info is in task_struct */ +static bool test_THREAD_INFO_IN_TASK(struct task_struct *p) +{ + volatile long *pstate; + size_t len; + + pstate = &(p->state); + + len = (u64)pstate - (u64)p; + return (len == sizeof(struct thread_info)); +} + +static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)tsk; + return test_ti_thread_flag(tfp, flag); +} + +static inline int test_tsk_need_resched(struct task_struct *tsk, int flag) +{ + if (test_THREAD_INFO_IN_TASK(tsk)) + return test_tsk_thread_flag(tsk, flag); + else + return test_tsk_thread_flag_low(tsk, flag); +} + +SEC("kprobe/account_process_tick") +int BPF_KPROBE(account_process_tick, struct task_struct *p, int user_tick) +{ + int args_key; + u64 cpuid; + u64 resched_latency, now; + struct latinfo lati, *latp; + struct args args, *argsp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argsp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argsp) + return 0; + + if (_(p->pid) == 0) + return 0; + + if(!test_tsk_need_resched(p, _(argsp->flag))) + return 0; + + now = bpf_ktime_get_ns(); + __builtin_memset(&cpuid, 0, sizeof(u64)); + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + if (!latp->last_seen_need_resched_ns) { + latp->last_seen_need_resched_ns = now; + latp->ticks_without_resched = 0; + latp->last_perf_event = now; + } else { + latp->ticks_without_resched++; + resched_latency = now - latp->last_perf_event; + if (resched_latency > _(argsp->thresh)) { + struct event event = {0}; + event.stamp = latp->last_seen_need_resched_ns; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + latp->last_perf_event = now; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&lati, 0, sizeof(struct latinfo)); + lati.last_seen_need_resched_ns = now; + lati.last_perf_event = now; + bpf_map_update_elem(&info_map, &cpuid, &lati, BPF_ANY); + } + + return 0; +} + +/* +struct trace_event_raw_sched_switch { + struct trace_entry ent; + char prev_comm[16]; + pid_t prev_pid; + int prev_prio; + long int prev_state; + char next_comm[16]; + pid_t next_pid; + int next_prio; + char __data[0]; +}; + */ +SEC("tp/sched/sched_switch") +int handle_switch(struct trace_event_raw_sched_switch *ctx) +{ + int args_key; + u64 cpuid; + struct latinfo lati, *latp; + struct args *argp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argp) + return 0; + + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + u64 now; + struct event event = {0}; + + now = bpf_ktime_get_ns(); + event.enter = latp->last_seen_need_resched_ns; + if (argp->thresh && event.enter && + (now - event.enter > argp->thresh)) { + event.stamp = now; + event.exit = now; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + latp->last_perf_event = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + latp->last_seen_need_resched_ns = 0; + } + + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c new file mode 100644 index 00000000..2ab26875 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "unity_nosched.h" +#include "sched_jit.h" +#include "unity_nosched.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +#ifdef __x86_64__ +#define TIF_NEED_RESCHED 3 +#elif defined (__aarch64__) +#define TIF_NEED_RESCHED 1 +#endif + +unsigned int nr_cpus; +struct sched_jit_summary summary; + +static void update_summary(struct sched_jit_summary* summary, const struct event *e) +{ + summary->num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event *e = (struct event *)data; + + e->delay = e->delay/(1000*1000); + if (e->cpu > nr_cpus - 1) + return; + if (e->exit != 0) + update_summary(&summary, e); +} + + +DEFINE_SEKL_OBJECT(unity_nosched); + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err, argfd, args_key; + struct args args; + + bump_memlock_rlimit1(); + unity_nosched = unity_nosched_bpf__open(); + if (!unity_nosched) { + err = errno; + printf("failed to open BPF object\n"); + return -err; + } + + err = unity_nosched_bpf__load(unity_nosched); + if (err) { + fprintf(stderr, "Failed to load BPF skeleton\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -err; + } + + argfd = bpf_map__fd(unity_nosched->maps.args_map); + args_key = 0; + args.flag = TIF_NEED_RESCHED; + args.thresh = 50*1000*1000; /* 50ms */ + + err = bpf_map_update_elem(argfd, &args_key, &args, 0); + if (err) { + fprintf(stderr, "Failed to update flag map\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + nr_cpus = libbpf_num_possible_cpus(); + memset(&summary, 0, sizeof(summary)); + { + struct perf_thread_arguments *perf_args = + malloc(sizeof(struct perf_thread_arguments)); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -ENOMEM; + } + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); + perf_args->mapfd = bpf_map__fd(unity_nosched->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + } + err = unity_nosched_bpf__attach(unity_nosched); + if (err) { + printf("failed to attach BPF programs: %s\n", strerror(err)); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + printf("unity_nosched plugin install.\n"); + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "noschd"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + printf("unity_nosched plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_nosched); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h new file mode 100644 index 00000000..f7280f4b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h @@ -0,0 +1,92 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#define MAX_MONI_NR 1024 + +#define PERF_MAX_STACK_DEPTH 32 +struct args { + int flag; + unsigned long long thresh; +}; + +struct latinfo { + unsigned long long last_seen_need_resched_ns; + unsigned long long last_perf_event; + int ticks_without_resched; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + if (!perf_args) \ + { \ + __ret = -ENOMEM; \ + printf("failed to allocate memory: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); \ + perf_args->mapfd = bpf_map__fd(skel_name->maps.perf); \ + perf_args->sample_cb = handle_event; \ + perf_args->lost_cb = handle_lost_events; \ + perf_args->ctx = arg; \ + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif -- Gitee From 4e8b29372534a1b040058e013e72dfa03e5ab6e8 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 07:31:52 +0000 Subject: [PATCH 08/77] unity/load_avg: Taking use the dynamic proc path Signed-off-by: Hailong Liu --- .../collector/plugin/proc_loadavg/proc_loadavg.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index ace88eaf..d06dd792 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -4,6 +4,7 @@ #include "proc_loadavg.h" #define LOADAVG_PATH "/proc/loadavg" +char *real_proc_path; struct stats_load { unsigned long nr_running; @@ -15,6 +16,14 @@ struct stats_load { int init(void * arg) { + int i, lenth; + char *mntpath = get_unity_proc(); + + lenth = strlen(mntpath)+strlen(LOADAVG_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, LOADAVG_PATH); printf("proc_loadavg plugin install.\n"); return 0; } @@ -28,7 +37,7 @@ int full_line(struct unity_line *uline) fp = NULL; errno = 0; - if ((fp = fopen(LOADAVG_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_loadavg install FAIL fopen\n"); return ret; @@ -74,5 +83,7 @@ int call(int t, struct unity_lines* lines) { void deinit(void) { + if (real_proc_path) + free(real_proc_path); printf("proc_loadavg plugin uninstall\n"); } -- Gitee From cb2dcb50b04289f497a4f3cf61d4b62fd49578d3 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 08:27:26 +0000 Subject: [PATCH 09/77] unity/schedstat: Taking use the dynamic proc path Signed-off-by: Hailong Liu --- .../collector/plugin/proc_schedstat/proc_schedstat.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index b51d5d38..f4691264 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -19,13 +19,22 @@ struct sched_stats { }; long nr_cpus; +char *real_proc_path; struct unity_line **lines1; struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; int init(void * arg) { int ret; + int i, lenth; + char *mntpath = get_unity_proc(); + lenth = strlen(mntpath)+strlen(SCHEDSTAT_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, SCHEDSTAT_PATH); + printf("path=%s\n", real_proc_path); errno = 0; lines1 = NULL; @@ -87,7 +96,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) fp = NULL; errno = 0; idx = 0; - if ((fp = fopen(SCHEDSTAT_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_schedstat install FAIL fopen\n"); return ret; -- Gitee From 48eeefa32059fe5be46dc45e3019cfae8758b3ea Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 22 Feb 2023 16:34:54 +0800 Subject: [PATCH 10/77] ioMonitor: Fix potential keyError in fieldDicts Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py index 0521a7ff..2a9dec0c 100755 --- a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py @@ -339,8 +339,11 @@ class ioMonitorClass(object): if stat[2] in fieldDicts.keys(): self._removeDiskMonitor(stat[2]) continue - for idx, value in fieldDicts[stat[2]].items(): - value[1] = long(stat[int(idx) + 2]) + try: + for idx, value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx) + 2]) + except Exception: + continue for devname, field in fieldDicts.items(): io = self._calcIoIndex(devname, field, secs) -- Gitee From 7a805a572079352ed97601153a23e08f0fc33d4d Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 22 Feb 2023 18:03:16 +0800 Subject: [PATCH 11/77] add develop documents for unity. --- .../monitor/unity/beaver/guide/develop.md | 677 ++++++++++++++++++ .../unity/beaver/guide/image/frame.png | Bin 0 -> 38680 bytes .../unity/beaver/guide/image/queue.png | Bin 0 -> 14649 bytes source/tools/monitor/unity/beeQ/pack.sh | 2 + source/tools/monitor/unity/beeQ/run.sh | 6 +- .../tools/monitor/unity/collector/plugin.yaml | 2 +- .../collector/plugin/threads/sample_threads.c | 6 +- 7 files changed, 690 insertions(+), 3 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/guide/develop.md create mode 100644 source/tools/monitor/unity/beaver/guide/image/frame.png create mode 100644 source/tools/monitor/unity/beaver/guide/image/queue.png diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md new file mode 100644 index 00000000..4e78d9e3 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -0,0 +1,677 @@ +# 1、unity 监控框架概述 + +unity 监控框架以插件化开发为主，支持coolbpf 应用，以及多种数据发布方式。具有配置灵活，资源占用率低等优点，适合在服务器监控等领域部署。 + +![frame](image/frame.png) + +# 2、开发构建流程 + +## 2.1 clone 代码 + +开发机可以访问gitee和registry.cn-hangzhou.aliyuncs.com，并且已经安装了docker 和 git。 + +``` +git clone -b unity https://gitee.com/anolis/sysak.git +``` + +## 2.2 拉起容器 + +``` +docker run -v /root/1ext/code/:/root/code -v /:/mnt/host:ro -v /var/run/docker.sock:/var/run/docker.sock --net=host --name unity --privileged -itd registry.cn-hangzhou.aliyuncs.com/sysom/sysom:v1.0 /bin/sh +``` + +docker 参数说明： + +* /root/1ext/code/:/root/code -> 将代码目录挂载到容器目录下，方便代码同步 +* /:/mnt/host:ro/:/mnt/host:ro ->将host根目录的以只读方式挂载进来 +* /var/run/docker.sock:/var/run/docker.sock -> 挂载host 侧的docker 接口，可以根据自己开发机的实际情况进行选择 +* --name unity docker 名，可以自定义命名 +* --privileged 特权容器模式，如果要在容器里面进行调试，该选项不能省略 + +启动编译 + +``` +./configure --enable-libbpf --enable-target-unity + make +``` + +编译后在 sysak/out/.sysak_components/tools/dist 目录下会生成目标包文件。 + +## 2.3 准备 plugin.yaml 配置文件 + +unity 监控启动脚本默认会从 /etc/sysak/plugin.yaml 读取配置。典型的配置表说明： + +``` +config: + freq: 20 # 采集间隔 + port: 8400 # 监听端口 + bind_addr: 0.0.0.0 # 监听ip + backlog: 32 # 服务监听对队列长度， + identity: # 实例id配置模式，当前支持以下五种模式 + # hostip: 获取主机IP + # curl: 通过网络请求获取，需要指定url 参数，适合ECS场景 + # file: 从文件读取，需要指定path 参数 + # specify: 指定id，需要指定name参数 + mode: specify + name: test_specify + proc_path: /mnt/host/ # proc 文件路径，在host侧，为 / 在容器侧，如配置 -v /:/mnt/host 则配置为 /mnt/host + +outline: # 外部数据入口，适合接入外部数据场景 + - /tmp/sysom # 外部unix socket 路径，可以指定多个 + +plugins: # 插件列表 对应 /collector/plugin 路径下编译出来的c库文件。 + - so: kmsg # 库名 + description: "collect dmesg info." # 描述符 + …… + +metrics: # export 导出的 metrics 列表 + - + title: sysak_proc_cpu_total # 显示的表名 + from: cpu_total # 数据源头，对应collector生成的数据表 + head: mode # 字段名，在prometheus 中以label 方式呈现 + help: "cpu usage info for total." # help 说明 + type: "gauge" # 数据类型 + …… +``` + +## 2.4 启动监控 + +进入 sysak/out/.sysak_components/tools/dist/app/beeQ 目录下， 执行run.sh 脚本，启动监控 +执行 curl 即可以查询到实时数据 + +``` +curl 127.0.0.1:8400/metrics +``` + +# 3、监控开发 + +## 3.1、监控指标采集 by lua + +本节将描述讲解如何基于lua 开发proc 数据采集。 + +### 3.1.1、纯pystring 处理方法 + +预备知识，lua + +* [pystring](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/pystring.md) 库，处理字符串 +* [面向对象设计](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/oop.md) + +以提取 /proc/net/sockstat 数据为例，原始的信息如下： + +``` +#cat /proc/net/sockstat +sockets: used 83 +TCP: inuse 6 orphan 0 tw 0 alloc 33 mem 2 +UDP: inuse 6 mem 12 +UDPLITE: inuse 0 +RAW: inuse 0 +FRAG: inuse 0 memory 0 +``` + +#### 3.1.1.1、数据处理策略 +sockstat 接口导出的数据非常有规律，基本上是 + +``` +[大标题]: [小标题] [值] …… +[大标题]: [小标题] [值] …… +``` + +这种方法进行组合，可以针对以上方式进行处理。 + +#### 3.1.1.2、数据格式 + +监控使用 [protobuf](https://www.jianshu.com/p/a24c88c0526a) 来序列化和存取数据，标准数据.proto 文件描述如下： + +``` + message labels { + required string name = 1; + required string index = 2; + } + message values { + required string name = 1; + required double value = 2; + } + message logs { + required string name = 1; + required string log = 2; + } + message dataLine{ + required string line = 1; + repeated labels ls = 2; + repeated values vs = 3; + repeated logs log = 4; + } + message dataLines{ + repeated dataLine lines = 1; + } + } +``` + +想了解监控 对 protobuf的处理，可以参考 [这个通用库](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/common/protoData.lua) + +#### 3.1.1.3、 vproc 虚基础类 +vproc 是所有 proc 接口数据采集的基础类，提供了通用的数据封装函数。根据前面的proto 文件描述，存储数据实质就是一堆数据表行组成的，在[vproc](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/vproc.lua) 声明如下： + +``` +function CvProc:_packProto(head, labels, vs, log) + return {line = head, ls = labels, vs = vs, log = log} +end +``` + +添加数据行： + +``` +function CvProc:appendLine(line) + table.insert(self._lines["lines"], line) +end +``` + +将生成好的数据往外部table 中推送并清空本地数据： + +``` +function CvProc:push(lines) + for _, v in ipairs(self._lines["lines"]) do + table.insert(lines["lines"], v) + end + self._lines = nil + return lines +end +``` + +#### 3.1.1.4、整体代码实现 +了解了vproc 类后，就可以从vproc 实现一个 /proc/net/sockstat 数据采集接口。代码 实现和注释如下： + +``` +require("class") -- 面向对象 class 声明 +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local CprocSockStat = class("procsockstat", CvProc) -- 从vproc 继承 + +function CprocSockStat:_init_(proto, pffi, pFile) -- 调用构造函数 + CvProc._init_(self, proto, pffi, pFile or "/proc/net/sockstat") +end + +function CprocSockStat:proc(elapsed, lines) -- 在主循环中会周期性调用proc 函数进行收集数据 + CvProc.proc(self) -- 新建本地表 + local vs = {} -- 用于暂存有效数据 + for line in io.lines(self.pFile) do -- 读取文件内容 + local cells = pystring:split(line, ":", 1) -- 按: 分割标题和内容 + if #cells > 1 then -- 防止 空行产生无效数据 + local head, body = cells[1], cells[2] + head = string.lower(head) -- 标题统一小写 + body = pystring:lstrip(body, " ") -- 去除开头的空格 + local bodies = pystring:split(body, " ") -- 按空格分割内容 + local len = #bodies / 2 + for i = 1, len do + local title = string.format("%s_%s", head, bodies[2 * i - 1]) -- 组合数值标题 + local v = { + name=title, + value=tonumber(bodies[2 * i]) + } + table.insert(vs, v) -- 添加到暂存表中 + end + end + end + self:appendLine(self:_packProto("sock_stat", nil, vs)) -- 保存到本地表中 + return self:push(lines) --推送到全局表，并发送出去 +end + +return CprocSockStat -- 这一行不能少 +``` + +#### 3.1.1.5、注册到主循环中 + +[loop.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/loop.lua) 是周期性采样所有数据的循环实现。首先将文件引入： + +``` +local CprocSockStat = require("collector.proc_sockstat") +``` + +然后添加到collector 表中 + +``` +CprocSockStat.new(self._proto, procffi), +``` + +此时数据已经保存在本地 + +#### 3.1.1.6、导出到export + +要将采集到的指标采集到export，只需要在 [plugin.yaml](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/plugin.yaml) 中添加以下行做配置即可： + +``` + - title: sysak_sock_stat + from: sock_stat # 代码中声明的表行 + head: value + help: "sock stat counters from /proc/net/sockstat" + type: "gauge" +``` + +#### 3.1.1.7、 数据呈现 +用浏览器打开本地8400端口，到指标链接中，就可以提取到以下新增数据 + +``` +# HELP sysak_sock_stat sock stat counters. +# TYPE sysak_sock_stat gauge +sysak_sock_stat{value="frag_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udplite_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udp_mem",instance="12345abdc"} 8.0 +sysak_sock_stat{value="tcp_mem",instance="12345abdc"} 1.0 +sysak_sock_stat{value="tcp_alloc",instance="12345abdc"} 32.0 +sysak_sock_stat{value="frag_memory",instance="12345abdc"} 0.0 +sysak_sock_stat{value="sockets_used",instance="12345abdc"} 80.0 +sysak_sock_stat{value="raw_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_tw",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_orphan",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_inuse",instance="12345abdc"} 5.0 +``` + +### 3.1.2、FFI 处理方式 +关于lua ffi 说明，可以先参考[lua扩展ffi](https://luajit.org/ext_ffi.html)，本质是lua 可以通过ffi 接口直接调用C库参数，无需经过中间栈上传参等操作。 + +ffi的注意点： + +* ffi 数组下标是从0开始，和lua下标从1开始不一样； +* 可以直接引用ffi 中的数据结构，效率要比原生lua 高很多； +* ffi 是luajit 的功能，原生lua 并不支持； + +#### 3.1.2.1、 为什么要使用ffi？ +pystring 虽然可以高效处理字符串数据，但是相比c语言中的scanf 接口来说效率还是要低很多。因此按行读取proc 数据，可以采用 ffi 接口来显著提升数据处理效率 + +#### 3.1.2.2、 ffi 数据结构和api 说明 + +proc 数据以变参为主，下面的结构体主要用于scanf 获取变参， 用于上层数据处理 + +``` +#define VAR_INDEX_MAX 64 + +// 变参整数类型，用于收集纯整数类型的数据 +typedef struct var_long { + int no; // 收集到参数数量 + long long value[VAR_INDEX_MAX]; //参数列表 +}var_long_t; + +// 变参字符串类型 +typedef struct var_string { + int no; // 收集到参数数量 + char s[VAR_INDEX_MAX][32]; //参数列表 +}var_string_t; + +// 变参 k vs 类型 +typedef struct var_kvs { + int no; // 收集到参数数量 + char s[32]; // 标题 + long long value[VAR_INDEX_MAX]; // 参数列表 +}var_kvs_t; +``` + +导出的c api + +``` +int var_input_long(const char * line, struct var_long *p); +int var_input_string(const char * line, struct var_string *p); +int var_input_kvs(const char * line, struct var_kvs *p); +``` + +综合来说： + +* var\_long\_t 适合纯整数数字输出的场景 +* var\_string\_t 适合纯字符串输出的场景 +* var\_kvs\_t 适合单字符串 + 多整形数字 组合的场景，如 /proc/stat的内容输出 + +其它重复组合场景可以先按照 var\_string\_t 来收集，然后对指定位置的数字字符串通过tonumber 进行转换。 + +#### 3.1.2.3 实际应用例子 +以[kvProc.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/kvProc.lua) 为例，它实现了一个通用kv组合的proc接口数据的数据高效的处理方法。如经常使用到的 /proc/meminfo ，是典型的kv值例子 + +``` +#cat /proc/meminfo +MemTotal: 2008012 kB +MemFree: 104004 kB +MemAvailable: 1060412 kB +Buffers: 167316 kB +Cached: 877672 kB +SwapCached: 0 kB +Active: 1217032 kB +Inactive: 522236 kB +Active(anon): 694948 kB +Inactive(anon): 236 kB +Active(file): 522084 kB +Inactive(file): 522000 kB +…… +``` +对应处理代码说明，重点需要关注**readKV**函数实现。 + +``` +local system = require("common.system") +require("common.class") +local CvProc = require("collecotor.vproc") + +local CkvProc = class("kvProc", CvProc) + +function CkvProc:_init_(proto, pffi, mnt, pFile, tName) + CvProc._init_(self, proto, pffi, pFile) -- 从基础类继承 + self._protoTable = { + line = tName, -- 表名 如/proc/meminfo 可以取 meminfo 为表名 + ls = nil, + vs = {} + } +end + +function CkvProc:checkTitle(title) -- 去除label中的保留字符，防止数据保存失败 + local res = string.gsub(title, ":", "") --去除 :和) + res = string.gsub(res, "%)", "") + res = string.gsub(res, "%(", "_") --（替换为_ + return res +end + +function CkvProc:readKV(line) -- 处理单行数据 + local data = self._ffi.new("var_kvs_t") -- 新增一个 var_kvs_t 结构体 + assert(self._cffi.var_input_kvs(self._ffi.string(line), data) == 0) --调用c api 进行读取 + assert(data.no >= 1) --确保访问成功 + + local name = self._ffi.string(data.s) -- 标题处理 + name = self:checkTitle(name) + local value = tonumber(data.value[0]) + + local cell = {name=name, value=value} -- 生存一段数据 + table.insert(self._protoTable["vs"], cell) -- 将数据存入表中 +end + +function CkvProc:proc(elapsed, lines) --处理数据 + self._protoTable.vs = {} + CvProc.proc(self) + for line in io.lines(self.pFile) do --遍历行 + self:readKV(line) -- 处理数据 + end + self:appendLine(self._protoTable) -- 添加到大表中 + return self:push(lines) --往外推送 +end + +return CkvProc +``` + +## 3.2、C 插件开发 + +在collector/plugin/sample 目录下有一个示例工程，它的本质其实就是一个so文件的编译项目。首先要看下sample 同级目录下的公共头文件 plugin_head.h，该头文件提供了数据生成的API，降低开发者实现难度。 + +``` +/// \brief 申请数据行数量，在填入数据前统一申请，根据实际情况填入 + /// \param lines 数据结构体 + /// \param num 申请行号数量 + /// \return 成功返回 0 + inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); + /// \brief 获取对应行数据，用于填入数据 + /// \param lines 数据结构体 + /// \param i 对应行下标 + /// \return 返回对应的数据行 + inline struct unity_line * unity_get_line(struct unity_lines * lines, unsigned int i) __attribute__((always_inline)); + /// \brief 设置数据行 表名 + /// \param line 行指针 + /// \param table 表名 + /// \return 成功返回 0 + inline int unity_set_table(struct unity_line * line, const char * table) __attribute__((always_inline)); + /// \brief 设置数据行 索引信息 + /// \param line 行指针 + /// \param i 索引下标 + /// \param name 索引名 + /// \param index 索引内容 + /// \return 成功返回 0 + inline int unity_set_index(struct unity_line * line, unsigned int i, const char * name, const char * index) __attribute__((always_inline)); + /// \brief 设置数据行 指标信息 + /// \param line 行指针 + /// \param i 指标下标 + /// \param name 指标名 + /// \param value 指标内容 + /// \return 成功返回 0 + inline int unity_set_value(struct unity_line * line, unsigned int i, const char * name, double value) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \param line 行指针 + /// \param name 日志名 + /// \param value 日志内容 + /// \return 成功返回 0 + inline int unity_set_log(struct unity_line * line, const char * name, const char * log) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \return 返回mount 目录 + char* get_unity_proc(void); +``` + +**数据规格限制** + +1. unity\_set\_table 中 table 参数长度应该小于32（不含） +2. unity\_set\_index 中 name、index和unity\_set\_value 中 name 参数长度应该要小于16（不含） +3. unity\_set\_index 下标从0开始，并小于 4，即最多4个索引。而且下标数值应该连续，否则数据会从留白处截断 +4. unity\_set\_index 下标从0开始，并小于 32，即最多32个数值。而且下标数值应该连续，否则数据会从留白处截断； +5. unity\_set\_log 中的log 指针需要开发者进行释放； +6. get\_unity\_proc参考2.3节中 proc_path 中的内容； + +### 3.2.1、sample 用例代码 + +适合周期性数据采集的场景，通过周期性调用call 函数来收集数据 + +参考 sample.c + +``` + + /// \brief 插件构造函数，在加载so的时候，会调用一次init + /// \param arg 当前未使用，为NULL + /// \return 成功返回 0 + int init(void * arg) { + printf("sample plugin install.\n"); + return 0; + } + + /// \brief 插件调用函数，通过调用在函数来收集要采集的指标 + /// \param t，间隔周期，如15s的采样周期，则该值为15 + /// \param lines 数值指针，用于填充采集到的数据。 + /// \return 成功返回 0 + int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 2); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "sample1"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + line = unity_get_line(lines, 1); + unity_set_table(line, "sample_tbl2"); + unity_set_value(line, 0, "value1", 3.0 + value); + unity_set_value(line, 1, "value2", 4.0 + value); + unity_set_value(line, 2, "value3", 3.1 + value); + unity_set_value(line, 3, "value4", 4.1 + value); + + value += 0.1; + return 0; + } + + /// \brief 插件析构函数，调用完该函数时，必须要确保该插件已申请的资源已经全部释放完毕。 + /// \return 成功返回 0 + void deinit(void) { + printf("sample plugin uninstall\n"); + } +``` + +### 3.2.3、threads 代码 + +sample 适合常规数据采集，周期性遍历插件拉取指标的场景。但在实际实践中，还存在数据主动推送的场景。如下图紫线路径所示： + +![dataflow](image/queue.png) + +这种场景下，可以通过创建thread 方式进行进行数据推送，相关参考代码在 collector/plugin/thread 目录 + +``` +#include "sample_threads.h" +#include +#include + +static volatile pthread_t sample_thread_id = 0; //进程id，停止的时候使用 + +static int sample_thread_func(struct beeQ* q, void * arg); //线程回调函数声明，可以通过arg 向 线程回调函数传参 +int init(void * arg) { + struct beeQ* q = (struct beeQ *)arg; + sample_thread_id = beeQ_send_thread(q, NULL, sample_thread_func); // 创建线程 + printf("start sample_thread_id: %lu\n", sample_thread_id); + return 0; +} + +static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; + while (plugin_is_working()) { + static double value = 1.0; + struct unity_line* line; + struct unity_lines * lines = unity_new_lines(); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl3"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + unity_set_log(line, "log", "hello world."); + beeQ_send(q, lines); // 往队列里面推送数据 + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } + } + return 0; +} + +int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "threads"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + value += 0.1; + return 0; +} + +void deinit(void) { + plugin_thread_stop(sample_thread_id); + printf("thread plugin uninstall\n"); +} + +``` + +**在线程回调函数中，必须要判断所有调用到的函数是否被信号打断，用于决定是否需要退出并释放相应资源。** + +如实例代码中需要获取sleep 函数的返回值，根据[sleep函数](https://man7.org/linux/man-pages/man3/sleep.3.html)的返回值说明： + +``` +Zero if the requested time has elapsed, or the number of seconds + left to sleep, if the call was interrupted by a signal handler. +``` + +需要判断是否存在sleep 函数被打断的场景。 + +## 3.3、coolbpf 插件开发 + +关于coolbpf，可以参考[这里](https://gitee.com/anolis/coolbpf) + +`/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### 3.3.1、Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### 3.3.2、bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### 3.3.3、bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` + + + diff --git a/source/tools/monitor/unity/beaver/guide/image/frame.png b/source/tools/monitor/unity/beaver/guide/image/frame.png new file mode 100644 index 0000000000000000000000000000000000000000..3892a4cb3f167ffb3c1a32be0ad7e1003e66f262 GIT binary patch literal 38680 zcmZ^~19W9gvoIXnoY;0Uv2EM7ZA?6|ZCexD*2J0Ew*8-Z-sipdu5YdXthLwfU8kzL zySlo&x~lr8f}A)!3^oi95D>higoqLl5C{_x5U?>62mm5nIMok$0Xr*+3jx(k>fHF1^_&;zUAT?l||0(MMQ~oc0m?aR{ zzc^-q_uroc-~mwo`v%Pe{$GxHApgY%VafyjZyDJ5FPYl4mnz@|?I5A)3P)M14w!PLb;So zo$YO0|FT!LvvlEO=J^Nt{{{bVZU3Q?cd|4EwD&I=AJe}i{}=ZE;tSi`Iyjj+JO9OJ z{TKiL!v0@;1q*u@dw{H+ER7}YTuhw+#Q)az-v<8wWBkjDhvBcd{*U~92okjeNN;3MS+*hwEGnXq0Yw?;0(I?g=%_0U8yAdZ#(-&VzS-${H`Tyk ze>jEnv%kN;!~LpZ^Z^u@goH#`c!0otUq28?Fn~}12?-wrMM0vwyPJ&-qTgD%QTz5G z+R52@uOsU6^3vx>@*4`6hll`D?Dtq%ROUc|Vx%~TrluyrSp3lB3i9E>SxJ!7>}VjB z>;Mrmn;~dm{abH|tWf3kxBlOf`@~R4U{D}J!U2<5P}rwhwiu|{q{kyc*AxJFD{`IV zubpY)Cm*5S<&9MNEUVCHld130T4etafdK~!K!b(LQ3FGd5&j4A16wHIxCk30=eso0 ze?VkNgk0rdq@+QBe*A@4K>-&eT0wct(f-plLZ~DG00l~19NfR1Eh7T3tRUL97XJ0b zpi`HekliuJF87AgWp;M~wn#uq!-(&FPv>e^bZ9zRv< zM6bUs%N7Rk3hxsKshangsW-wL8N&+XY&=TJl-+km~N8Y4f_qA2< z^*aef_MLD4>850F!k(Cg4t1Aoh&)+d4O8l;Tm@{Qjv2w-s*Sqz#NOBY-SYFzw3ip? z_wOm393mm1PO78NpLizfh`4!oil%DIH4Aiba_}7 zh03MBv|EUG+&_7kg>(6IP0S%1h~RPXo!l8Le<;GgWbw2)jwj`}w-c^)_;4~W$KYQS zm7kQ=bek=dRYk#7Yyc7Da&B~RmsR{382N=UOYX6KJf~wZ7y3hmfzf4RLf&i# zFI53WCv^LJc(}{d^sT&%_GF&oa{#j9*xH9j%h7VdLLms^tGzvErOwc>R!;@Fo8f$& zb9KElwqTbBCsTVQM)9R320weL+gC~nr6|y23kB8ED**`{jB)+t6I=OQMlcKPYIqHf2DN|9gH_37g4u z)?{NNzjzdo<7c*ET$L61Oh#h~`1JHM_8WhA ze7SDtm6eY)GzBZ0T3W7Hok%g}9_h6)kkNvPb?|A_?Q&U4{RTp(?~Y1+-|8wRg!Wbgle)q{DpF(Hk~=l||8G*GyfCulvL zn;I9FR_FCFNv}(+*G=0*zp!~51gW^-Kb~|$r`_G@J~KX6PFmdjqwmsu1}!Pcz!UEG z(&y*HSx7LLD0*_|`g$Kcb|-_;=kw{}dR5g>sl2bV z$(oBlL_f})RTfTUGjU5~GCKlc(7haDf_%xm zipwfjEBDBSE#synq`W>I_886#JkqY+J+5lY>P}66aLa#KS4368Wx!rTa~!)d?y8FD z$>r#=bU{~A+&i4~p&83PaXRiejyg8Ehz^Zelg}J~FI*ngo4(8bn0~|4b$8jE_I&eO zdwM+4#v{qb!ZD{Tr&5GpoUdSaIvMTz1*yC~juNnnF&A6I!}Ym*;|_e)J7SmsYFn9K zrlQFEb>Hdyjv)0!zvX?prQDHwIUGBLiX*49>P|*PEhr{eqPnaG>&{?fu@F~Dq-SSL z4B}I31MA_Sr6fho&MmB|8kwpkRA_i~ASERw$kr}ekZ3`9Y6lYWLrEeauPa;kSF^1l z-3`=8B)7Z4I$RDf4h*!a$-%*!d=6Y?!NMs*BDvzrE1g!El~oCPIx88KHv)m&3ETq& z{C;v0*W(3j2pBV#rlu?}uLXJ0Z>bq%5&m8K^4+&$MUmd1sLd)B{j9@ zP~V`dCOti+xw$!JW&$ChO&-tN$5j@n*!LrLdm(%c35h)`GrJfqf~`(LuID~bH#>Tl z%|aUW2pCPVYOM_}r~QGm<%NrtEq@A_pMilrTZ;AtPQRRQ^g?ZI`KLdp64~#nj#y{y z87-FbR^02&*>Zn=A@v--XRKVWpvTZ8ol&ht*AF}`eyA@;>CD%tbE`(@DmzDKvIErf zttHt0mF}Lkk(H35QM%HuI!@vv0UIBwyKY%%Vl!oyH}z4^iC{Wi)y`zy;&?sZU2c7q z&%Hww5w=iG%oMtJ(fUjJvl61ZVyo^mz!G$>r&xHz;IhUsa+r`tS_K{t9#1$st?pZy z0H`vkLe7TQl5v}OWDK26*s5Rjm)FPa!_+*xeP?}qWwn#tozt(}XWZMH&7_R?pV_I( z8gVn@dvQlcN5wwT1MYm>h+E*ITPp1+=_=L4gZ=SjftmzM+fcB%xLgqQzPn`5 z5S5lm97#w>RajbmcGAuHv<@Lw8lNq1aXJ#SQb{UlVWV1qhew$V?fs#wvd;@OVDU6p z{DY6rlan)|qJpwO#ZpNpDgn0Y}*rmpm#mXY9M>R!+QZ!n+7iN$iG3 z#}Iy`>4AyiS($a^m2bVeWU}!dz7zj`+aJ_siP?|5^{VxuQcQQp^FW{D1o%X@-on2A z;Xn-2lh5Lg)XNxzXgyaObkwb zJ8o=N3Jev52p_qR!VYSt_%hLx&>S+~IkBXYjQaPU3U*+a8(=)!Gd7p0=A#ZMC zLPJqyv^l(M{j zlHGzZfuJJ}**pU?)ecqHXAR+q z5<-Vj|2jJrwz5zLIE{;nhL>j7R3sPLmog-{eFMU+Fp9M3P5yGGu*|w^$*5gPOYBv6 z6mrEpt%i1#0SI!Fp=Be!E)joY6S6e4v(sjrSkU$OXA>kx4~gi0LPCn^FHRPMd(r=> z^^0iFAcOz=F(ziJvcl{y#loK@bZSC79;h-z_5Zpz3d(J2lk7n;_nRX1A&4>B9AN)43KJDDOD6Tl9ErQ zf$@}3IfD%@SQz0&R!C${A3*im5Hpb@x9Gs~YwSjN&nSX+Mn?Gc)xq9F55lAuR1|=X z#-}x;rvpIb`i9uXS%*)Ez? zJz?^zz9(t-oi~={bh)4a-{&TUE+vw_vktrz>UTjxK}!%F0OHt(qX3CT$3JSx1xyc8vlm83Oa}^swgD${c>OtmNcUsMc-KqF zN|{*dlSDUeDXroKd(cu=6MUs zM?AF)<)XU$5Ez{0LMyRt6N!K*lKRc*-K;Aik_sDfGGL=*7;ai@_lZ^$S{z5*QgU z#gG-rcaaA(mHV9*!(4KC8;yMpp37mnV7ff)fM1|h7n-(OGj{V@>YC}f)i436f)gp$ z3_21K(L>mX&0xRjwncrO#0`ih^tz^pMvVF6+MC&$iGP?KmdHL&kBtin@GSRyYI#00 z&eh?^ROv?goGw z@I_IYZ?$jUdY>skRFZ$W<5U|1<^^p-g)5rS^{1zZmR_kc6JU~1CYCgp0EBMpVyu*X3f5! ztY;qFQDD4tCqq$k$0+#wXVe=F+AXx!2h$&pKE~oQa<4RpM;ikqW z+ej>l{k%q8MW!>TuLto|f}J2b*?D%Et*^IF(~lIvfM(vc=|5c7%kR3}M_20yJ&~1d zfBYibxN9Cre|EUvklYqlR~y=1;q>ggyFa-)JJaN9V`Od|+^3I~lsMD+ePY1Ayt+Q@ zQS`W)f=6@KB(HAwU-_>YyL7QG_ZwGHiRfC zNjvW+&!J{P^Aq*3HJG5|VZCOi9#mDBwO%MIhD0zRl6~&K?h@5^e}W;f>}J$-6Oe=#Qn{>P z%5T@Qy+%r-OURXpesrsTyqdpZdMBmm>$*tYtbI9XXs~vrzr_&Pv$#+;=b053pM1l@ z%E$<;KE98~l&r&fYAF=27YpfXOluuT(JKKThJ_R&`3^?O;9%w~ig7nnZ684$@yrPQ zoJmh5Ao-cLeQ?`dXyJj0UIC@;YdNi34#g{?5+5D~j$C+2DA2|^%k}elL)Gq>z%SzK z?nIek{_+UCGZq6``Y;OhxMLf&?Qq~Iz?Ss5`mY1Qd+8{C|hxG<-zmRg`F0wDZ# zqg~U0;Br%4nNva*B>IJvo^ylI;(8rmuXntC`<5gtDh2hI6OdpY3KSA|If?<3B<8w* zUU6PsgRr+5xg;vU=k(XBldVMcSEUpY|0{E z0+2Q)jYSn3=24wSH%!FC-5s5CeAU=lNR>`kU|@-j9I1|CM+|kP1V#hgs^|0f$(b)K zQh^09vRfpu;1Hl?FO=Vs2f+FY8XWI`9+0P^!`Q6V@bM5$iJRhcAU$aW1G87cgoM>! z`P~@5udEJ~40rAKn3|JN*J#2fnqJV+{D#kuo1PhKL&tMwpue=UvC;7gZOXAjMAFvHk*A;kHL@rT8e6wbfWT6SdhW17|bLM`^28M%{Fh} zSpnG)eId$Q70vaZ4XfwTz(2G>^^ICm#2K`I^GLK<@;2cFK|FF@6_Zn}osE>E5_2=7 zpHi%eMI)`HPG0>c`G$>MVQ;W~eoi~pb2rvIKROytNs4W-P2*${5CHYHF%nq^a(+$;rW?jx1NEMjtD&;}(g~+5I42 zI61n@vr>Ie3{__6;<+lqTaSLIvWUUi_e06qIp)58V4_|}OOdK^Ztf+eJa z3C~_p>oAohDLfYR>CQo4kBFD#PNt`O9o+T#Q(A4QLOd2QwlybnmJ#Wi7a4+1jNjMS zH^w*5c7-v`ebJNua7V$Tp@*@u4@2sfDZf&YlOGO_B~C?01L~W0R~6w$oKsRtYb~^w z6taMlkR01*akE3p9*p)0@_9VS53*iHu3z`v-s0`Z12T86@_zJ@flYMZ8}7A@^6v~8 zv2=k=wCCj1(7ttd3_T#~3mcM4!H6yFCnG9J?bbKPk73R*Kck)t8Er*J4`*YaYnLUAk9#mXIQiVi)7ANf1(h^^ z6$VhyYzTN1v&bSkY+z=0abZD}9>M?tDafth{?xwTqt{buiqAtI()Aka2EO3p%m|%6 zrsH_Ipk3ZSN1R|8NR;#a`aetAG9IjPk$81CG+E50VBAx^-j8N%*g6~w#X1QkEuLe5 zZaVh3i*|dF%-aWMv<0&w~W}A#aQ#d@%g>46kDq+;xpf*M6XEh@^&X*>dI(g z3xV;5e+BdQlVn7yh384Rhqk~#@%+$j+?dI#WcczC-QX*VFVE#OH$fZPihN^``1T5| z9pmHU6Brm+RbB0g!!D@kda|ymPz9}zkJRg>BFQsq3%4n zhFf>0x_8qV!U^=Szt7>QHIr3cOL+CCBAGE0W3|~ODe1Z}_*It}?8D`4i7psCgT*{1 zfnypa@|-d<&%ne#0SZA1KHTM1_jwl*v4d^FlhaCM?@wo9hE(6;FLee{6d9#={Os|kaDX1;v)okNFjGNGi(9sN zj=cRLxsUO1t#RtIEGTi?uoS3;)qB`CxwwCzQE!`80BG+gCK||y7Y_BO9!VF>gR-As z-$W{depql64ttD{92y$RyO&;K%7V+3l68@@_ zInRc4%L^FfL3zS&om$pbUJqBXaUC3A&_7JE$VfF|Kx96G)4WyQyL1Kn4$ex;jxDYt zF;!Jn-yY7?VdgU?U=R@z;h-$64Hcb<<5DySuWIP|z&u}Gxb?uR{XdO-CNK<}*hJz2 z#A+(-1_m6EdRz`J)W`tj&`=)tj`QQUOS!IoSuotOyKx%f?gs*u!^zf zB0n!A{OQJA?D7g?BZcTIe*c6Lm+grFc$mc*ivSfHPKgg%qjtwa2|JRzB@=%v7{ zv|$$Pi2lF!&vC2hh~z3YE-^8%-<7bis(OKNB*%33hG!p9iC@T%_8lJi(dh)6Dmwka zwhqJ@^N~r!C{b$pyGgVqLjgt&Lu!r4Up*USBO)U*EZx$u=@{vWnX;M3J|pH>?6*Hw zQ01(Tm>=hlqu$;>JB!>FP*YRG<8}k1n)iE6%8|S9@BiLFc}($8P#O3cXVga`a8BB1 zOlw5fn+p6XN=%*<9&smL8UsbZw>@W?l}TUYs09ww)eteh-sXHD#XT>guIZWd6Pgc6 zVBtPdfQe{wLRq*kUlSU z&z7pGwD4yTY8Cib+3ZY8@`;qrJsbAB7Zf|hQx@g4x!{zZ@Q(9Wusm{g~B1Yf! z=N4X1TotQNc(%%Fe*a1PTmnjRwd(RQA^~4iod7C@?A+YKzWP@BD;2_-e7&Pa;Gf4^ zbpHosDJv@xll|?!pcgW>9c4VQQ76)DHJBNoYza3|7MdZg)2LPCw64hDUt#f2mjlTl`L24f;p?Dt)O zJ(=8}p$oJZZY@J7U_x$p zjUcD?N}A;tU|R}Ws^X;G?=CJj&EY=6!O0}uce9puPN3a&S8uB)c6QhC@QsM&cR8MI zvR;mg<}6)WIu|{mh}GjA7>N}H+X*NdWR=fVl`4YX8%accrw&OI_}R9U_*{~>RLzY| z@bz$S*%D6@@&USkv1Va@wD|c+1Oi_3$UH){^)iUr$-F0ic-cDJetfVvypBuJd&*`r zs;csG50>BQvxy0cVt323pS|D?qwe46^B5DecQOgyEe z)DZCb_&qa%{S=SY15LHxnk$i6T`3xqFg@_wXrJP9KEwxK`d!%arzJNfDT&!?0b5p< zLPnu*d@PZVaA^_mPu)%&fREo-4wKitR#HVJcOWeZNo2*@9_nn@8?LG2?d=^2^taVsjI~spH~iZ{)igab+B#p`G84 zDwCUZnZ{mIvWA8RVtyaW%B1~eD3GiNQZXAMzGG5wa81<2VJ9c%XN8~V=j(FWX6IY} zi*pSxRhmJJrR+9oZ_e6U%D?S2;+I+Oq(XxD zuHHssnhN)ypKtEYmLUZZonbg3ApXqcT&O82NyxmHRP*kiEnh)HW2&g2IXmBBfBy!@ z4fkEWmT!7|oY&W@zR(zR9UF%|gF{i}SwORGwEemVgea>`8r@K?4+}D7Lt+{h zsV*?cKOKPmp4m@^^26H&biHMaxIW&5n*p zP%5L6iU4uR+&iQj1k@N*gG3ZmSHCRtumbHyQ_l9{;?vV}b89OKP_VGd zYw~}LRuU6)A_oZpok@a&+hD^3j+JR?q?7$j?o_Pyn^3a-JO$WxTRFcFW4h7M4~`p) zih@fHXSOU{dTFSr#vaaMvpF0osHjx6v>FTtUR}@=DA9kkjBU}%=P1P|kBu?(ru%IY z@a-QSeXVt^-4o=GNmZ++vScnP8;jO=AM67K)3P1KlIA3r7x%t2k)~iGeE0e|jE;?s zt*)l4PPCGiPvyPdLPE-;rhzSdw6j?+ctgxfzl3w6_IzI?v%hLFG=#@tvsrI-yxQ!t zmWDh_%1Y~=n3#x<-=PHItD;CAtnu?zo$O({Lq%0~h$JJRRu&dc)*P4i8iI+vSeud0 z*hfNfA|%;+&EbvoC|6CjTsSyjtd5UM4{4>vvSB+?rRh^j{3!oCW9BG+^o>ZTy2E7W ziSoai{qd=)VwLYZ_{;vlC`AI!Y6GyFbC{!gDfU z4zoI-q9v*-9Mw%&i2_-3=bb#(*4FCk%j5;_q?`;K-0arl>#kwYr+}0DqSU-(IKh zSimY*gsfLlI-WMb_Zu4JDDl0v|KHYpjAXv=puow*v znylq981S1d=Vg+lFxpm(euoznp}2@f1ke{-lwCDhQv_YypJ?0IVhTKE@w|jZL$ zX=%-q2sMLm$hak9)Z-i#6ogdJ?d!is6~M@<0wvPZ)4iY0PZrAO0r_G3CPPOuPlB?U zG7meQyh%mHm@PDSkYK#-6eM&TTfY^19QUXVBr2fLfDh~a-WwC8V5%!B7K(n-SI0@A zs8B=pxYzNfC<&E8`8#syQt`P)c!*^#g}YPGP;ss{f%qT%(R8+UGC`nOQU|gL{GU$f|8Hmm_Jvf?SwI6`3x0u_nUq$@**y=g5 zVH+tf%L^EIDr-#hm1o_n`hvH&tI}!y2z%k8#X3GaTU%R;)O;E$;Rbo?K*cEgHu5AD->aJ1j=70|U+cW*Brg=XNn} z37NB%M)Zxz2rQj~60$gh&ggaH#+J9-tV2YL0TTF_)MuOssgD?teTkeT#X;QG)~4E) zA3Qi1w_Ga@M^IDx#3Cn^-1gH?^qgV?v3WBVQdYyQn|a8BUt0r34jLkVf&KF)!YBqz=TOqpQtaEMc zFE)vgnfi!0cTi+{zPO=qFB=OkJUyh8k%)5AbISy3YE;=Xt}C3eO~*m0aWDWtj$IGG1uKVucM& za1NmY36xHk7b`I_NDmahn5gAaZLDyBiYg~j)Xut%g@pzQal9Bb$h&C3iMx79w#&gd z2~8zuM1-#c{)>Ijh48tMR`|k#3$E&BYdvfXg}b98EPQl9L_q#e*NedAnFg9yVZJgU)vL7XymY5152aswauuBM&w5mfz-D6 z$Ezkns>f_fY$p8|9ZjTjxWL%J^of>=_F7Bsz|AQP%;R>O%gJR{md-v&0ob=3K414g zjj~=|(^68t*;#%A`7J8wx3Wk`vS9B7yf8j5D1E}KHqEK2%Qd+^)NzIV?hfzt zx-fVUfeG`>wamYVPLQr3h8TWM^>umuB_4x zXCQsfI$U!iaO<}@5zIje-tP#%SE^@4`-Mvc7t|XslX%SnOv&gd7q`p%f(3LVb_YB+ zxn-fD5LoxA%obGhj~x-g!I8Dr8Sp1C+XLDtH`>zmKNe=@9 zVR~fI?T52vj&&%Qp9u_fbW_ut_~!FJ%UVTf0|iu(%T}%~WBCy%X=r%78ezrn@{@^# z+qpTb3o?YuuGsAh>^9de%$M;wQ7IlTW({AzK98EL7FT+EpXlk?)>>vnKNrel7}auk zz1O$|&Ob#=XMa4zJMNMM6NKVZ&_?(eHo z8A$TN0J#weh`1}&qLI73A$)v4PEW@fgBaVw5T|1;79?QVAQ2eYSy*`7r|tWLd%C&^ zcHhHia%UKHIN(t2^HD)}06oCX17%0LhcGqKEqn+(Kf-L&{{QTbn z(y&_@85s_?NZ>?2;lP0ThzY6a5_9!=#C0RKvYXkC_Ph4?zndlAYaV)33#aOIH2=_) z%Jx|3^xV^KVe5PDh4yoBX!W=uN=b^r|A0?&h|D`Ny~o5P5d`ymz|hdq&8M=U^&xQY zg7(nIDaeb#fSCmHKzg|0;Z<=*+#jnL4(dY@#6<=TKci1Mr&&jc#1w?+w2t>c-Gcyg z%68R>oOBlMK~%jZK&wjlzLva$tnBK}3;!9IckZ8<8h zwT!fzlNn7@n|@j>y*X!dD<$oV^z!ad-`S8w6+{*3qh%hQ9_akl*V}3LfnJZt2VhAH2C2#Btba{q#sXHlt3Qiy6Pybw+MK#vF2@i$ zbc~SS@bMwT?D6niTPJ_y@SWpI>nDpGB^GD6|4-ZLtO^}fe4E$0{g+`=aV}r z6GOn+_HVdMU;!y0J1KcUbG%QrT`q&u8AUcdYOTZ==; zN`25?ruuD^a}8@wn{!w%Dx%)@cBm$A`?>7c)!ue zP#MN-OE}Rk_WA=;N6gL*{`4Z||o4$dWs|0|AgCXfUr=9lw^CyJj!GpZ_RH^|_U;eh%2PqdOy z0W8feL1JbD8V$TZ>)$MX5n$kwExcUUxg6fjwdQPeGYaHA1mI{SHJ!BRgyw&BBS0St zYFw@#Un2`Ozg*Lckibt8kxx>RN>C6PP59i6%jY>XY{ZoBmZIvv%47i`xIz#_MI!MT z49^L?ga(n+(4HyO5~M7|V#xw{xuT|&0_{@&xt#EqrkjKii5oghjWBK_fovdz0M`)U z=YT@)@c&ZnCJ_Z(i};6eKpryEs22*z`VOsnuoF__J=qJ%B+cU@al7sqaf0x_aa-aT zAQvS+{GO-D&`>c+Y3aG&s19;331nYPKY9i03&pAh5#EtM-{*gc$G(h@f55?uPiD4| zkOZMhA~FqCS0C;2^s?EKFIFC{)Naj{mCeZKbPARP26nZ*m=NRwkqeu(n@am;`Atj` zop1&Aw%Gr)-{g^#2&bo{EF;k+-yns8f(L%rBqL3Ax5CpWjqm>~A~>SBnYu#Yg3eb& z{uzT(?`TXXB3O*TD}-S7$ka7zhrP?^cCYsk z@R&`u$CG$RHVObZh;Xx8-6 zoh?nA`A<)K35QzDWGtEvLIwRU%-m>qt)ewD>(0(z_y3|3oc?&jw6P`2pGzz5EM;Mt z_wk(}BOAKaE2dXYfYi1q^xydem((5`5k+kNmFaxPWSWzy)h*%g^;yoVp>19!=V0RS zlC(&R4)CQ~Sh!52Ks1^tm)y^cjd8o4(kDB=Kig)xFwa5&9rQ`6c#ev=PpVbmrGxij>XqmY`vW54;J z=ME?tiqZ!jAGcJhbB6^#J0B!!(}Rr$O@W8!*+=G_>_TQ{&(!=u%}GjVd>qw^G{exd z?u^35hE_5A@w~C~4T>%E&RtG7C50RV1CVT8tK7n&Ju_Z`uI7t`#>4XtNQv$c6Q|(u zK=OHegark?IGPzK`$Z~ner_YI+6Rm!vF#yncu^{^U#?waUtxKuh^7Np;4iJ$&>4$N zn>msCAR!@fxz$r)<^Q9zMUIjt1jw_F20Zl`iNZ&E zbYcd_mi^DzxTNOJQ9WE>oC2X63N3op@(3EH+3em({?+D?0Ru8Z*3b_>m6K}inM`&9 zV*^GIX{mwGYCg|7$R6brWs8#m27nn~AF{9*S)V5apZw4+IG;DNgDC@SrI^uaLc^*7 zSl>Zku~^u~#>eN;Wo7=zXEc_0I#2BH&!`->Ij@z!0_G1x@WC4NzkJvWxm;^c~`h zxF~Jn*T0Wabj|JbH0oK=GY$|NAcBXi#m2qa%NMfb6xF$~<8SF&98`mOguyFyzwf-1 zOy2Zyo1a^}J+3ZXa;{c=E&qw|&;;X|IjL-W=PEc@Z{thCF6dfHL6Eg-l7_jbVZ!9dT0!-qx1Sd|;7nB&a3O~g9 zjG-*fDn4N%XF`TmDPVl$oAKGY}=@dy*=j!S8E!`urd9f;aa_m z+w)afc-*l9W2iu{hekMf&V){{19tnI2Y?;^dXKf-bQVA%Un7@k_Hv?38j+BQy4mGj zZ#Fgh_B2zN5(Rf;G2ZSF51E;&I6XNXT)}bTJXg`QXc9Jdqra!C)*)b+SF=UKf9AO~ zv&o0EY`z{)16sq^X2m(h+;E&vT^e5b`zd#GW{Y!ca@J;PTXW2E4D~a;Zq1wBqPxfs z8Vd1Sho6m@8$?8uOO%_}g^SKkIB^z7QEhdsh;j9BU{0$rjZ9C6Q7YKJ+FhVj0E35y zhD;`#ER`CEE;Op+tmO6jH@$)C=Q+Wi+31H#Y>+xm&+0R9mJp zeN9YE6dN7)7An=BfWzo@IsBb;mXBr>OXcmI9MT$0enLi&%V_XAn;iu~ZUCjWJY5C{ z$HHP>Vlnojiv~f?WN_3k9!%h0UYbWmukm@i8L!ldHaBPKCwV{8g7$yCZ5jLeqOSj0 z;b74HGcrH#I~1{-Tb#@9&*O33l%sb;0A(@wNsP$GAPfh8!?d%P6XLBCnZr~Dx$+%9 zmG_)`g@MMa4FBt@Uh=o_fz95!xg~+Sy!#A#&H-AojmzGBbY!Tr+{dTwq;_(B9IWw(ZHus=$y$B?KlR3u`}>XPG(7nHe#%pqhL+UCyQp#+ zwdA=O6>@qF$wE25T(mWyVndy*z`-{dC=Fu2)h8)QM zeg0iYgQiXSN4q7)=lL32H7RM~^?~LSbZOas=LCzIj&5L)%f-#&81@IR@{|J6lO&zpo^$`+abux5G-x3e}&-nC@OX*ei5gVxrg23+2^BW;c$Kwb-Tbzm%i58~~jtlzz;?G+0K1u6; zM4=P}aq3XH*8Xsw#Tj&>fUqL8Ru*4UQK57+owc>mDkF2B&+~ZM)#>}@DJt65?aO_+ z!el=jMVFAUu6Lq>kc}uCis%c70thk29GI78XB-AV5qtR-C=6(P;F@Xd)aDJnKAh*UN$_L2W@Pi83}@kFmTl?`q;ySMHa? zU_@sIa>RWBscYeWn-+Fn8D6(h$;;&Moc@>=NyHW<%TXmZO7|4Pn7>~F<+g`nI70*9 zeI%)8w=0iE4AAY137U zjgLRHe#eTUrd8I|xX-gFvu9wCIEC*}#erF%|32lJ}J1 zRiFrBE^!O!ag~H*rR`?93KHPv!S=Rj|BqMDsr{ZWdZDjqoaM0Td=v%IYHjv>_7$gO zwNp02ZQZNQ6y?&C1_=c!K33_dR2!K|?M{=n!92YNjqUPVX$(}^$=GsBicXW8Kr^=B z)N_DYk;W+zc`Lm%WC~0QxJ)>d##A^oY?sgN9s?>#nP4`>(GIg|=R%cyIA`cmUhl|V zR9Gm?#47GvHv=8x>_87GOsn6wo*&-h^U`P_C_nxXQxgMrADyCRygyHD7)?z%TC2$d zn*rAYcH)9383))#euO>=x)^=0B8YgNPnGAzybnIZb3xRBi_1%blrqGB0e>+pF3l=P zpc8`D^>&J$=6Ak|ib|4^4oXp(W*U~en}3oc`6a6yLyvYV%0P9aOp%XBqaLIA%Zzyu z%Z4)$JV<%6o<`c{(*I6ya8cyGz9t&&oCM62)G|H(KLb}91ijIt%S+RQ!f5I@HuS93Wuo; zlwMx%u$Ic%1||k}b`tYQk&+eX=WgTq?F}2UvzyCE8hRo5>>ngw!>4vusv{CI`Mnd2 zjorXvK6*XRu8kZm6o_`+-8$8Us#rz3k|y%3bi&>H9KFmX%w6AwdU5#dF%mOuV`;10 zM}07T#y;;q>%n?^^563vI9EIsY;t;X_F(d={ImQnWGH^!y>2~}E>nE+{-&5U_sPA< z5KZBwMBLy2!@Q=6t+C6x@-*&4cR~IfT!WrdD(9! z1zC_@+SouPPnI)!eD^DachQ|GD&t#Vm zd+bNcZEt6LZxfH+rDZT=ySnPBwOM5b6(n}3EGy$OpJv0u11+l8S*X!vc)r~MZGC;p z4ZI07(ry}uF8+om1EHm{LO5ju)e_j++1VM2fX8LEP(~siqwsFJ<3dW8o&Nt(_ElkV zENi;~CSiyW++BkQcXxMpcXyZI1ozZZlODIci z(&|RD_b}5YeI@WXPorUc*-*Ka9zIETRU1gRol+3?DF!PC0o%XI$p3X_)dr~d8&Yi zrw>ZBy!_oto2J;d;f3|}!IT^&c=!D-c!vvHr+gqa$k+os10Q znLn3Z)WCPD6{3;1Lk8K7cbRVp=GD7UT8xkhvR75Ulu~<(+i>jw7czSGx*WPa7c_S8 zB*Rp;+n(u1=Z(lnSu2I|^zZ3to50e~EM_LV8~<~|oCckQ-?3u@F%wKHM!v5zaSFt> zyD&AadRJUiTT@y+GHG@il3*fW`kt15d_{b@58rr0R8v#1i&>?Gt)=C$xvA+~+XUch zu(31eldj_mgt`wr29vGWCzBrH3ig0`I{ZiRf>aaDxa7XQ{Uk9c!L?Dsy5`121*#0| zlsfvG@&LY7_E!HAHlG=c$J5RE#Gwf6(I>OVvMRX(kX_Z2u4~ED+z5lc_I^c$DG?uM z$#k%|Nf4OH41$qW`+J=-EVZpxLUL#GmzTPy!FnGbJ3~TwUm{EuR3(msBtkJY^j23T zwLy0d-C6@$)JLvAF+W7cevzA>ZQ)Vn#2rPGIPNFM>kev}Fo8 z#jW#^rT^0ipmrw*?d6$-$^3z%`#)WJfS}*JctB5*9oc_T4MT*T=k~uhd+iBa6t?|{ zr~kuyI$c4b8NIGzaDxAd)r*Os>E?jq-W#V4m$OqEo%)jnMA7j!;SNu(KKjivu!<~cmB5eIjnY~u*DJ0W3VZa^d}>f(H|+$8vT18AqYRLG5hG4XRF ze#1$@8sbRCFd=RXU!M=MRQ62LHk|#fSzb0#PNS_J&R~ob53Pgn^-D7nGICC8gU!+` zzzNZ2aF!K^N*EdFH1nDL1)}%~fQG1u-=>GHX}>*lnKWs`>K`(3{7ClaP7X?~6_wQt zqW62fi?L94e0&_J_U43bPw8~q&r}lS5P=i<6XuuCw2;+%`MdOYEth9a=qAj#-?>n6 zVc~3*?ON024x#<}dKVyy>+kQsP>M(R`x%3fH|nybS25k^IT0+D)7G=sX0sbQ)Y=6S zjs&O@MA^mJke>cR&ACc{e}B`l%!;By6JujzK=mINxAk}M48FfPb`_nCr=%ZB*zDl| zItn-9Mg)|m`|~x1k2lA<9X{T`FU?;zU2q-Q)qDy8BQ%#F0zxA2vB1^Bn<_{LbLNBCZoyx($crH!FcPMn&=V)we4BrpubLw z3Q@ma`{G#yg{aS|Rw^xZz2`X^Vo4fJz~g;>(#V8`i<$XhVv`1+XLWRRlPxSR0=OIy zRDrpxv=SS%38Rvz^2_wPX=G7|h(`&1OyQucbni#%C(Ah-2^*V-De|SBo+=YlM=jw& zVE_OO#fJtA2IDDFUrm7OO8c%eOc+I$i02G;TDS2FrCACSKBQ*T3h41J-AyYTcXH;- zJ^FixztG{d4=6kSVqr^)*XLut|NG&&u$eZ(;U}iyM;KlsGLHZ6$5lZ2)x2S&?6ClFbgmCBAW`?#L#m>;1fibiU`zV1YglV} zd#^}tLnU*K?07$-@h2Ocizm=g##DuV7^HhsSKE$?hW3SWb{AdH{?DdPKQWDrce#V^ z-Zja^NjXFCGU~=10@HWqL!RM3Oos{F!vgVX+2zU9thi04HOtG(Lm@fU)o0df^WR`U z13IgtXs5MVYxRt zMMoBCf_GCRoNfEgjv?LHHSA|Ajb#qI%3bZtu0uTUp@9#Pk&d5GTyrZz3$B+H2dcHUmly6%O;!ZKB?+O+Tp6tb|w!K-Oxz z)*d9_U~fjV5Cn8+g>~5<2`BR;ars#6SJq~ zVy*EZi}hGKyP}+&TwUB*+ci2AR5GnD9;ZurY%JnhDsr$h2jRx#Cg@aV%#QF*Z(4!yi;%*wo~#!LfA9Qot<-4GD?C{le(er%!|pQa_LcL3>S{ ze>MruCW1D5Vpq$NH`2$4s>yFsVf*yrmVywR&T(&FOp;I%Os3 z)9ZX}Vmwo-OhqyHBfV-R|DZR<_k~{(9__Q?7gJ}x0XM4ke z@bVAsa)^YH%GgkgrQeh zApgV>u8=gJ84@QT6x3YSdVaWug~!mdw0zt;-(2y$!C2|<4O2_HG>RF`*7h=}5zpZ-iv&d<(r zYiVid54DuPFkYsiXw%cvC(~*h?L>%ah=kXz2Y&YU9;Z=}CmbxljK!fLrBXj)<=>}d zbWS3Y)h<$>Z3+ypbpSMpK;qWR)BR)F-SG?06ORw+_0 zfS;F_x1o6osnWHc$X4DXE>Wlh%SlQgfrk{-?j~kut4#_@d=ACMFCf{;GFb zpd?s6s6tQRbgD>J_(n0PK35naS?7oqismiq+agEq0{md5iROeFtPh%nzm+MWkDi{g zRjbxPP#tXZQZyR%ez=O1q-%^Dl98BPs;a8;H?D5CCmDd&1d=0|#$r9>3IYSmv!Kpe zdMw|kTOAxH1sV9I0dh($JmkCn10@_U$V`OEQ#jf4 z*UchPFNm+V9Gks1aECNO0zk$zR$Ct#F$@@AShz59*-X8`I`_++j&PNDS6~Iw%d;k* zCu+!CNw!U39l;6RAq`P)*fy@16W7<*t<(x*Vqw`|S<=aI@tA$TD?U78*!I%@M@^o$ zUP&hQGY9h=ENT$woWH0)ApGz>dJ1s8|2l>6LmqAQGVr^$a-w8ZOvqdmyLxqNyxgUq z>jS>xJ>KZU*G0E+L)`ApHh>{Sl##3BF+5RYi>5X&h$1{2Na+9Ya#PiJD88_8M5Rjf z=TB|$M@+I2V?T|DbUpVHL71_S_I4wCFGxLiV6U1(&^D`oTO0F6h}Z`; zL%3e&K5{~OM8p9=>TI!4sl7|9UHw8r;GZ41uG(h%`jVcG;}$qHG-P0|sH>~XH<~!k zLQ6|aL9q|`T^wp_*_)b9dwR%afshBH)={MG2vLbX45%nhx8rO3&@gds*OQE}u>Fx# z@86YerQC%--j`=avf&>De0~V2|I@^CmtFN)PuRzlS)S&GzoCE%pIq`RrQd%ZO z4G)j$f#|clVk}mL2m(0@kVU~7ZtmAkuDC>HY%aIr+RhyX${@oGBAHPNjaUO}i9vDH z$81H)p#)g`t1YXf3f4m*+az=Ah$XlWWCUL!uN#8O%skJv^{`Fyg znJRI4c{!)+iGm_kX8@GZcsLdyK{D)j`$Z3B~18hyVLbx}iq>0Cim1^Ug&jt6P z_WR)rYD<9+v%DGVrL|0q^0E+n?KWDSKv)nd?GYT#EUU$qn3Y4B2JBaSHv8_t>gwu< z)zy?Pm>w<`PBu2R-yHkiLcMcaM$I4gsHs^#eE5(XV;$3;Vj}@P$ZEr}X6DWVQ3oZkppeNnPQ34%iF6Ds6p}Df{Af%PnbSyzBS?HSiu{ zb=?1BJ~t^yqI&~YLhju64(dRSSAH!k@VB$+SgZ9~JMv9W+&#y++a1ac+LHLl2|;f!0r6~8`v}*B zG_TJS8#gy@5$pyhBQY^mnqyZyHqv+&Ycw6dwxOY65W9J>*_vH>rxdPMr!cxJQK}ko zwP!`1a@IbPH95Brj_a$>Q4HbO&o8-QKnj2wFU!oaR%t$E%ZpVKh#VtJNZ;Q|-=-b3 zivurxlMI0v-IAM@E%A-c6nX77STCtK5-I%xCS#>?^4k*KTLe2GtWmN^*n^vbm33{2 z6X!!^W#!b&^z_}Neh$5TT@4}-4^f#DV^u(dZ&aV}FuFZifuguHSfu&8h&5?L4MP~+ z{{$8Z$b;mw=IdOUOV%|AKk-m-aF`v82Z92Fkto~3)#=`RC3=8T3!C;x*T*#-o%MUP06**zR^4SJ)G!8oeS(oP!eg_PJa@QL@ zDFe})xVW(OR_2sH_7DK!=0APmO99(GolH6)ntY$B*6#s?iNs?RX*6dU-{Ii)A8*>i z!zp{caOK6iacn1(_K5xJ2v_@hLBc}Sml4oK5$(M~0s(Jc5$^g<;jxuAH+gZ{kr+(A zr+)v=CqUlG)$GgrFc|N~$=T9iO>;ko*d0b7cZet^Et>_X!?#7FC&P)NhKGNv{j_QG z-B<9oKGs-codkq9wtr7!qBklEW@hGECf$t~Y)L^+Z#bUW*~k_K)X<1QLWVL(ASeQa z5O9>I!==qe3o{m~zoGBWK|IzJtqy%Etw2!Prg5ya-AfY*bLL(b2}{)yf7 zWW@KiErn@NOFWF1@Q{)5@?<5#=lS8o4PI(m+8m;tI4c}tL@<~%eh5XTNx^n(vBNkj*q{7dLtnHxjjI2 zFtJ+YvU&Ev3ioq8rPkIyg?%f}&Pk{BHH%BVMixKK{W5B$S=c!9_7X<=X|BqtVl@J- z371Q3#h}4%s6R4NTZ<_pU2Az+ATQU9Ar2_G0Snf%_`Q3_9*G1zXA2KJJPa52H+-(p zl0_~&nID~#6SEcLxw#}82P**MNJ?6OwNGN6KtS*Z3=$Tp{qVkjaVa^$QELJ~0#@4u z<=wQj?2JgHPWBj%-5h^9oXa{rdZMARK6861rYoc6&2O)ez{m6c{)16v{Y}N zT~P3ad;qrrl}bcjo+3831fM6rR&CNwAIQ|y5*98Xk^M(yWk=`-=eUL9u#L8|Q?qcV zDp8v-`0>XRznvXzn)Pz=M51VP@ra6uh^Wnq$MXh^v?9X>GDdM_`A5sJ${8`q;ia{# zz5F<}SezUHxok0fU`tr6Zf}1{pqOe0fDxRhgP}xL-F6b7Xq_nwsrVVBpb&?N*)%wa z39RHjJU*o{(f_Qs%FWADq{NRC2e6lfaKD;NybDR@;BEs0IU^GzB@1CnMg|_dFk{bK zhY7EK!ck-0Sm1#tDeJ1$6ekW%)7MIB9`J?^TwNDWB#18ogzxDw91pPCgxmU(ogE&U zk{MV#+U%$QX;!duf&Vq3Sc-6PZ~)XQXI+^*dqws2OeF8$zehnqVPawefc%m*kAhj! z)W2XCJ2-{al8l(t`K~|mEiEmI`o-<3D>Ss0#}%8#R)yYbz4_q5v;@+HGZWL<@lTV; zuy&sKk1penjcg|HFD5nbg}1ZOCg)hJ^pMs+b^X`LEwYa0JF zU!WM@QGP2+C{vMf(GXoa=h`RHVA#X^La5~g4=A3&&@=pR4)klk^_)m1H)L>gjxS)A zvj8@n@i1h;(^c->UDLI8Z*)dRBg(;H!zv_`B_m@aJI}*=$I=Ox{(iAQ5m8Qcb$RyT z+mp7t^{$9E&qP$VobK-2yfoBQ1G~xLvEe+kMJ#MUSu9cPzS;kKaBTFL$5aO3)?d+R z^TEGa`TayeD9)aAc8;5|Dd@FB`y@JSNLavgEL#Em#AAc=G4CFTC%B;R-pj9qz&+4-`tL?e+Gy z`1?Z?uu4mM?0Hru z71~ikQxK*U-2CaIEsrOQ=h=FG0?Np~z96l}k+r$|^Zry@GF?`?>*wB%NKD3RO;-M^ zs52B>kXK%=$JJif0FQ)TQfl-OmD4YTE=NCpqL+myWpX@J zIdwkVD|VgNi#RL4qN#%`&e{tMBx`QgifF0M%0B*%#gdRjNg8D=;o)K8(ok7Q7LN*~ z;JAqb>yNp}k(Ki(^c)iKG!ZCUCZ+Pdw&qWd!4x||(IFsJVRBMBVm7kzfr^gmgJw8Y ze0xPv`S;J>l7Gh#xT{rsW~RkK{8zgQPyi^<00C06L6-4vs5DXJn+b6uZZ09Ve-!_by;VS`Pa*Q_}v|IK?-Gb~KqXa`v;4gxT02<=UuG}MWV?r%M)hL0$6&6yY zrlx(6RaXa6)E9E;NK(wz)d5snrYm~H`|&mzih1yV6v%r71lEjt&s&?whif2fkUi;R zPa{84X;2vC(5Gd5p4>O1& zyYEX|Ehw<*6$Jiy9}FEJz*p{sl0Lvz1Bt%Y3W0!tBE+)~pR~P?QsU#|<(o`bRDNM9WD#`o@wGa|fq}1$RfyIw zA0U+EV8B`}F4hOrB{1(k0wK^aH1L$7H z7WYdN<5DCI!1RHDK$Ri%bW(Z&j^P0LJ0Qv_D_@_TvALaPmz5oE4aO@fDh`v92Z8)8 z2nlpJ#6deJrOa6Q4E%gbhlI215w@DYHW_9wT2j!FZ@B`Yhds*1k0v$J#e z;_JB!g)e;GcRs95+q5-F*A^1!grorGhJzA-ErFI$Mxf5t~Ih=~{ahvv+t1a*FxeM^K(~{=RdD@n$d{HQWJstigSf7uk z0^o7BhY}TwWR%Mq6b-Yep_C!w15xCg1wt@Bn%92NF76c z)Is(RNN)(hvanXuzh=T!Ndus@8(sa+pM4SNHYUs=f}^(Ne-H1bWBYW{(%j5ZINJEg z&CS&xjlU)Xx)=-YklM?JfrSU8UDO;Li-3ej7q@p3-~dHM2Sgtm-@W{%d#TfE_rh!| zhJty=^?Wr2I8x+PFKiGKWkHKXewg@rqmLKEbmtMOmC9u*gu4LB&hvEE(*U32L_{FT zhw6m1czHjR_|=2{Q%@PDNSda)#!!Rexr4bm#nF-43y^gQEaEqMLO$bh&9ANDH#CxK zVy4{bJuTIljsq*GoU;B%Y}oDTB3XPM-GcNWSv0IfGF50W+JJOQN(wj)PbSdw<##M` zo9AuPp?Hbi3_&&#$9{BA2puqk@A}uJesG$~h-%}tk_*5AotM2M?iGvDcSR3s@?B2| zFzJqqkB8Z2vR!Qj?m(_6?>VX#k^HRROmUNtkW_<6kl>UOYHH}y1fUTc>7Y>E6GfOx z<-Q1g(0~5{4!+%1aXSOtvTRw^_Rf%_9IZMZaLXL6w4f^nQb-Zz_=ADlpuPWsy*?3g2yDP*4|B$Q)l|6oMU;vXPnDmm%V$J45Hz@%UlneG1B_`5FPaX&8jp~23oe)- zv=HVKwsLJWdAvClB`07a$p;9Z@TNewYe3cg3|x|koUFbtURPoXj1M?Fz1B+&6>2rm zYS{&pl$4M(p=rEK#4SgKk&4`4Q-4BS{3bR)CHZ~6NG79u%dPS7jQm@e7Cl8Q7IY5T zuXw+MJ(5NRpT*kkW8sMU?brLug98gG5jbqzcG_?EU(UU3$xq-%k`fbxs)GJFCzinI z=;?Z2&j5EDXT8L|=>tImUS9kMI9~A2%`HGZFo)>ilD}PoM76ZJuL|E@9JT^8qqIKC4Au`a2h=|7{;tQdErATgP1C5E6 zg)mpQ6K}hmK8|#M$EYY1&;7Mz3<&s?=E?> z=n%}h4G2*Fz}D4;2MQpPZ~=nuyiV#A{I{@AyW0cmUnO53PdiQlR23LdKR>@$E+Kb@ zY-IPdb^Z}MC=M%CI5&OUK>xRSScyfYl)CYxgpx|_k<)xc5H$bL->+^ohZq@wyPQYE zc6I1L?r#E$UXYx|ds)%Qh6X;>{T#swnAq*b-GCAGCRg@ydovn)*0f=U57IUp*sF9Lg% z?{fdZjl~F~zzDoMlUl1cR|LfHnvIDpxaV$iHyFkd7_Gm5i;aN%_3TF+p`HV46d_kq z)47wgS(GB6P`@7ta_KVN`E2|9`>rSXM#d`?)YMaRb4;vrU%wVJGFjC(>|kPwtAh5= z?muzisoc|~1nlY7A@GMNtIVRH&MePF{rd3!Zg8-BdAYE*)&m*Yn~Ew8l}bhb%~4U7 z&-=^dr&7|>xDAsYJ`HL+;tegxYsxa1l9GY~jt&+)732E0*o%k{6x!*F6jLP=Km%WS zZ8=BV*WYhu$-1z>%)vGd`}Ulg+6~&@$?q2f_1=g5eQ_YNwvYp&Hp%A~{7y$k25#2p zIlZEsYx7SoGZR*`%cov2NuQD?eLJk9qodGg%0l5Gmn7JwpNSO>O87+C@Sv^Y_;s1J zmfK(7zi;{ZJ?u}Vksw8#_qN(Yc+{SszM=}?_Fwk)YAv-t2e)|_P2`bRn{YQZ9Q4K^ zJJ9H6eE1v*R^HuFTnh7ZK#IweS!{NuD%AD7)piO?s5}0wO0_rL?w#gwUHZ6}=LYkw z;~hNCo!x+<&EK=?T|hIMT}&3b9MDv3SD%=IND`O%%*HvN{vVWE#!v{+M1W` z7Mn~-`3(b`|4bq)n^{&i9eE{7fX%x1^`{VmkPQ6iOY#0*u?#72x{ICNx&lmbrH~&O z0V)as=ngd0_b;D5CAuUU7`nH(UWP{$*n=Qa;EQh6y^#?&*Ij944y916@!`#!v=CW# zL2P{n@e%FaqY=S*+yi{f!OfRn?uk+C%Ar7eL~o!kX2jW`MBo>eBvq-EmANq5YHQuX z!dmN0-9xebWV$yq)e%2LeeNN`N!O+)5SRJzj&cPzYx+r3o^U3alysEJ38__7E_G+8 zHIv)@ZtO1pWhXZMhj1Sf6fhc%(tIZz;%J{Uk7#Kp6ScEHs!6{gWW3BQ^@A341v$>~ z+tTM5V?t4UeIqclk*r-4_Os7S-2_ZS=4L1|TuW@4O0lLk3Jc}GepellXIST=)NWf? z*D3aQ5W&R^P(cUzvjpc?1Bu{0B(+IQH$~CeQ!*Du~YV!7(x(3M3^sKyyF~Zd%a!@p#DuxU&CWA6G6) z?{Dr$enK8RB`uSQFD4br#s$4=qyKUi)ahcBPwha$#~~hN$FAjsTvd`2)^lA{itHak z4uQ=3ZEr;95oeM>70ikQu3-+}mz<6cC1vG-T{`XBEjG4D>|ZN0&e!oE5o5%^ryT0H z4F2*{<_EXu>rmn@Ue0j}s0!}Yt@YudzvY!z027CY#8vd9z)!L-woeNC#LE*l?QO{W z6loJ%V*yeYNxDRxL*L_su96QY=y7Od%(E=Dqm^_O(HUr-vpYxT@l*1Gd6WiJ^V+on zFE3swcogAW#!d$wch?ui4lcI+i{B&K(Pa-ZTd^FNcXx5fxIHaz3OXf!y8on)Hs6z&7OsfSSq-k znTdrt$^WbC%yP+aQ7vC3i5CBkqI^+I+>Z>7oi=)ro=Bk$qN;tJwAc9&6F`E~AbZrm zWPUSu#E>^!Lp5OJ8e9GbTB0Hz1>%#)6{Mruc0slM7WPM?+)JjH6&fD35ms6URR>3b z#(;X!mB9YYs;cFrg_TY%qGuSRigB;hcYS{&pA2=kVu;oE)CIM=Kbw839y%q>+I=U+ zf#HqeX@)i|2`uw;wZGC0)E$~hJk*>N^RBGdZ#aL&>xfU*PS$$QG_)7E~mg=Wb#zRRjwy zhtmcmrQNXqT{O4PwcB{I$Ld(?C$unEq&!>EI=Lar;h3iUTo6Ux8g`^$VX&;X<)r3M z=bwzKBSucBF4`^5*84jJf}S7yRsrtFUa~(?b;WbkK$4i>2b6GidVjP~@8@@&nrbW> zy&8;wnF>LGHC``Hno|@ziZ>Pv#^Uci*%AyOquHI3 z{wj}Lrr_`#>C_(}g`^-giBf53xk&%x{r{8AzMR%bH zC`&0T$+U4NEPD^!%ZN>ca9%GOn`-X+8Y7MfTX^Pw`fNN%+gb38Q5b;BS z5|qfp`ZR28kaA+^kjw=jGpPN2+mV=cZT0mg6S+K!g?E`e87x-w4I*B(0Rx^VO>Ou- zZj-%2rdjC#E*O0G3v7adl&6YL?{AhoZ?mjql;fmDQ}G zlSk54)ea6D<-SVNAYj+`Yb+Oe(H=ipiX%H`}gI9Ez zh$X7|)ks@AP_XyMOP5{x1{J6Mj~-ZiZ2R>cfoRIP($2(ROUJ*c za;g^2z`(#hG~MO=x_SdPz^zHnn;VPgEb?XWUQ6{hh-LFsy$OK(kRnJONi!jCu*IYO z!1skRzI|yT_bar-L)8AEWs~(18+j3s$5}!6W?*MjbpuDEz4i6(Txb63iYX$L&Hm8^ zC@QCtXygTh8*$jarjGA!POw-n+T{O6dOE5*3;N(mXng)h@2WnDmYd!%IIE!{u0gwD zCz@I?FI!X5s~qD5EvE$H5#?+lL^&<%?C>+VA$U*^`AW$xk_%&~dMLAirDcJ;XYZp+ za&dgI-dcx)LGrxd(RqaEkSE0+j6N#Hd%br~50J2$ixnS93xQbe`R;3&>jkF>Q);9g zl7wgNcZzr%I<&e?dUJw_ooK=VSV(paE=ktTg|fY>+=n=i9z8*8No!nI+?3N&q|)z?n3f0TEb0=kli6ifJ4?Tj^rZEj@11M+ z&*0!l1r|_58dA*sQGEzf-ZQG0=F`Z2v*;?Rne&Whm7(+uZK!a$XD)Hpnc7;7G-X+( zw$~wAmgJ!2sNPJK{VEz9Ldk!FLQ;n9{}i{A$vtJ~WJ&TaZWaMU7AZc;XC^jL^u6Q< zS&tsyx{mM63`jKU4!NpTr_$2A+O2nV^Ct`p z60r*t6QSI_eSLw6g!A)z5kj|kg&7=}#PrNR3os>a5Kv=s(nyI(Ajn?>FyBgUU<6_0 zECuRuIptJSFd>Df3l*SCI%=u`IU2E0d2`{^roD-AnU3 z-}{G4*D>@oUbzHulGzy)-PZ$Q`7Yd;By3|OXc3~i%}e#w-T=KE z4(=Z8rPlV)eX(7X9X>Kz@OdaV*OO5H?b}U*JpM${N9NUtdn9m$5v8ODbou!p}3d*9sWgYx<2z;$ZwGlUgSFa5$t!q7QSpH2{TXIwtg zxescFUg%NB+-XUHqqk#gvUM>9q@e6`MlTiyrtQu~hEvu;Ve%_yh z>MDvKZ?AjYDgVKkvlhrWVLE%PIS^z2)0yTAg`4wn+04%wS{7PW#EvAlK$&YURgdK509r+FVDBss;Y|RgoIJ6t7}5S+gy+3pKokdux84VxdrccvtGBQ z0+2YIzS#ec85>!}z#`(lUFHV0$;kQw5zP87d?yo7qpu9Nv!xooFsgcfZa+33L3pO;2Jk zs1S?5qBf42TA;1|={5^pyXzwB7BG2iB_TCj4Qx_0=LkZK5%5D^0$ zGv##Ec6cF2x=a)7GacKh*p?Od&YBK4q(2Qi)z;lq((RYs@5_NvUZ3N;&%RH7Ga3e; zq9=|L>o$lckt~k28_+MNql<{r-pLa8vKsYtf3mJIY?ZJ+F1R9>;VW@Q(&A61Gp*3| zHNUe{1&xuCph%@u!*n@waQsMP)rE<$St;KzegDZ>>(@zKRFoZrU{^AkL4VoO-~3`L zV>l_O%i@>!5FjyeaL%AYyYJkz$t908Cu%gao zL!&#JQu0z5hmzi4Dj#c3{u2HpmFeo9ypOK_=sG-cpi7VbIHX z$w*&6yDIR|&0%>*ZK7IjYj+e~!!FF;x7`P}_&OQiUa8OCeDQ_&YX_mW;EyOpzi-q? z!oPv_pvl-(4+_8I4$l0~!~Kao?rc?+ayCY{GZjC?h#)x`DRMYZ)3Ig1=o6$|Hb&^; z_40I*^6dW_eF%NiwyqcAehE(~@Yypb1~6>rg6o&rLw>hQClc3MJEAY~w8up6S-U8z z#GEv9zENR0&)FKF+sKg-O=XwpoRz|YnenJ$`Dv38iqcBLk3A@|u)>N>N}1`Wa7Y5O zW29gHnC!X==d&5P3bx?O&Hyr5uty@znpaxLRaEEnljGxS_&R~@8~H~FJ#VB80s)?I zYbznIEWgCvH#zvv#1v-O-%3X)`|aPFf#x7wI!EqJ#S{lF1-R1DzDf4um-q#UT;O#( ztO616e_A92(c&*y$_Oq&Th@)ZP{F2LcpPClWzMZ!UrVzf~AV3Q%Ifx4&`n z__v1AgNub^F9wp zY5-^(_!u8hTysVP_%e3Uwu6H=HytD6w_%#>X5_!`CM5XbJc^T%k(rL2QBo$fq!GVFI>aVYdRpU*~7hK$j85!=-V={x0@M{&Cttl9z zfgl||K32ewLGFoJ%=B;P-$1b+kR;$?VWDAQIBoVv)|pnl-e^`96hKznXD4e1C*w?k zO}mC`;%^$!(ItwDLz6Qim5FJjsW@B478jRSR$M#Z*m864n`)xg1&Rbx?g&NuqIo@* z-`9CZ>!KwBXXn@a7eawn=IZiX*0!*;WXzZJkdj(F-)!pZLnR{NRmXExu0=DDmh_79g1}!`p-5eu3JME# znmm|kS!RL!R-QJ`p2*T=wk6FL=QW?l>0)^x#`>5$oA{zD_I$~y}<4i3&qyzET87JS+f1*9kZDoMqBkW#_m$dmc7PvYFXIITit zm&_+418?O7Ob#?OG#?)yS=ki7Gkj&^Z@+bVryYu&y*_=?#mBd$s{(|kudU8y-|`IA z)KWmi95aR+t%FpA^fl$>)O2)o^z=2Nw?rv^CUSiq6_TimMrB2^>8Tvfl<3&O zGOIxVKbFaT2)N*}kPP(nI{Q?(lroxDWV^vd0E7<5%uN3qO}3}U_a#g>g}6{A^+FEi z%Ox?g-vb>rwGn)XMu4B(qbO}8d;%=gll-!bOfaQ{l$hQAGUg7!N0i>-My|1fKx2pQ z)M+%<@4EX-zijn>*_g^?y_l?U`0{q}cs~$OH&r}Q#*A5K1peXzLV*F|_eMhR4w06y zMLeJtF5C780JG;9j)ZmMpKDz3P$aI&K!dlNJy<>XMOD$CbM*y745V~8qm@&w((6`! zl2s8S5pLxG=`pdfv6i!yefZkZncP27dFrGN!v~3jmN?*qJZ|>v&me?%Uokl_NDqsX zx+pIi*%uYbo@(4anK}x%I6E4$itiynJ6~Vd)YR-@7@Hb9pP#3)na4HX4;t4bCnPN{ zEesagDb3DJVe?q2Rq3<=v|0N@SyjplvH&UjVtPaAsC&lN_O|o-?ON(iSDAl1jr1sF zwl7JPLkGIqX+K~9zaH_o>-mjJ8!wW(ooB6aciH=;C_sB&6DIEE^#tTV(${|>An06K z$)F9X&&n6TPo}45!Q(WMlCe3_^?i1}S#APoP(lLe=yq~XQD;5#^f~dbXzqD$wdv~DJ;e_htZjy zf>2vGjd>wBY&51_nneZPa>8fW|}~4Cgm3M1uQ8 z>-T$_UjAareiYB@~@8J%PS7C=pzG3pivr$4bJB;_CNz@r&ICrlzch z{onb&z3UpU*YA0E6}>bxnBz2d8{fIOx|$g#a3VstkLz^k{NX)^{U0HO^HYKsmWGCM zNmS)ds>R}1;+dxSK!k%H-zP%IbnkCaitJaFUPvcjtS*n@RwQJhudi?6-bCtSD+Web zkQq4e`x+3b3Z&LEnvMzepsJ@1@Fr94k;v_#VYof@!m+X7Wq}jVz@V>_+W<8edgKvk zxF0h>?$=1|q%eGrTip^7ONjGjHCmV2Rnjl*@*1+lunxfd0b>m{HSRV$QYS12 zP|P7z=!RDG=!i_`fDj}Gx#s2i(8hmY2lrP$<2F15tobWmo4D9jSQ{5$r+K8q^Lh80k^TC<3L7`oe-;1wBn43|5;vstvQ3g;!MH+a zb`0srzn%Sztjmu|2wGND?vHPP%`5_TV8Y|JBkxS^@AHW>uyQ>n6n78xJ3wBq^i=kN zSGw1B?)BE2$Ken^U|GW`bZ&Iql>y@0(I3V02Zl7dtokFp2JuT9X+TP~Dj`Zis_AdF;^DS$2NQ3jPl4`P$uyt8Qg;gve z={CuvCdrUZ|G;U7gD^+oM@g%r%n0;3dWL6!Gsrdo>2&}`J%wK~NREj5^+yMX!-;h5 z!>eu*i?Tx_txjj_l2A}}Zv!6Zs_U=DDz(R3pB&OlO0I^%5S@_>bo>tVfrmS(Bx{nb z(X<|qtCZ6d9!ee`U48wyjt(cb+x@AmP$3Z@JKs7sl?7M~%aK6Iq%dt;w2?p`J-fV; z^R%QPL!&4NfqWbRUt`BBO&N2X; zx(293z6AR@?B}<(&PSkkz<+oQHE+$#RK7jA{EYi!4kAY$A*iSrF`gWyiVhBRc21Q0 z8Z!{}?EZ9jJUa^|C9{Lzl#()vhD*lVvG4C+m#Pi(j^JVJY$SC&Mf!pCZ}SNLQAF87 z9KaqA2c7=xGBVG&I1jaK9=ASqtz6g^B^5OiV`DHPQBNl}FTU)7rDYMZ#RUc9DA)bi z?B9`Uf1NHhU4Cuw0u#k{B~bXn<2?mg5e@!r7Pvs%pl?~NDcbR7W(qo<7>MZTO!~ho z)|!U61KTC&{<6l_(bfI5@=c6=q0M@bp1wsUT`d3tr>zYaPfotu-}UX5O9T!MRs*N3 z!uqTGMLSwp{eNs%I7v3jGfA(t?JCZXG*3YC0Q2r$12gkBgfCl8Qxkq+wqamzf>)=b z-FSCwkmkEvsFRr3T^sOB5b;365g2rKmd^4x5LF2Q_WsB(6d@w$?k~g!?NNeJQP~p{ z-4XQu2&UtCvcdg(77#)ZM|)PFdH{>{fuejY5Q{hAxT_*8YHP?}t~P-~z+(m}jU{G1 zS$RxvXxR82iyh;S*?IMR+oqrAxtqaogCN3(fIExL>Ju6DQXrWuex&18s#n74dE?02 zv8$yWl9Xi9Nr#H}vT%K-+3u z!&P{J6OK76Gm&5d?_f(0DX(F`|I$ z2@{e{yFP-UJg=`SV*JtmEjs16{G;Z?KQEb#{Nyl$?AD4B_6?S-*As=LEtMh8G z+RXp2v-AFjv-{e(mT2+JFj1mKAEFy1dJqX=)I1MmAn9a}jFn-_`NK&IdGkNXQi70H`g;svvFt?q z8TS8hsuPFYlJJuRvbJjp9bvO>Nc;rQ&F*Uf`X1qPB}2kzDK7}LPEH-H;8^pFo%RPa zG4G^?pzN_+u+;Qb zrcn!rCvhfe=OO4eB@;ONjdKz&4G`;tPYM1;3A2?WBnWOQdqQ-B3L3$us->@_<;w5w z;N(!+iz-oi5iIZ{ssA41MM(|CBLcIDLJY80TV#s-WH% zJsKNBb8@4dA2a&E!s<^m^Cq4x+2Bjd)9MT_?0@QY7JPuwi_`O8V`s9~_10zGWSiWY zm{n-nq5&b5n)g4gh$`kJQt;CfZ-E6yyEZYUHbGe?cMN_Weop9pSdugD6gj!>mJUJp z;t2RGz0xdJ=k`bHLfgXB3+2*YUC_Ld%WNt0iPVw8R{wO;R?^w*z6nb$uquDGz|($^ zEo|{7EWz7#QA$v8yBYPH^t`AOnpnHyA3-0v(|zQM1iD``)c;7fjR zjvK5MxYqP5{>TrZ)m0%mdQlqASV&s>%nPMHZd_YZU-ILW1SNGxYOa3n9Dkrk4^@CN z?ONExu`|TW?`D0s-L{uoN*k3`{}SBKj_&G-)6`DQkFvQgQfTg9hLPJbdJ6fgiNKr0 zsA4pbOFf)#fa8&okv#$-@7BpeXz=$5SG>C0KSi@oTM3j2&EIK+aEpdPm0tb)cK+cI z$7e(~reW5oe^73Jyt8#t;BUM*QzzTlBwdy!Q$`S^!`R1O!n!Zx>x8b31paH8%}?kerO?`(aJ@h7@2kxA>}R`6V5l5vEWGX0sKHxNbYAc{|Zv5=+b-X_;NMv zom{}GP?jS8a8|IP(&m06mxnQ!03%%Ou(l>68XCduGBxVb5o-v#k(voQnb@W-L^AAh z7!Qfs*!?D~U=WK5{|QJg>O^u4yqa(nMG-TNHeKb zRArx4QFjA}OyDNOf5nS4yYWyl`aw_Ue)nLf$aj^Rl`IfM5*(&y+sk?lnG3$-oekT1TTqSTmUwkIOaOC#NQd%)}3$O`x%(J^d0XhK_1K3wq;ttpr zylabHNixbl#nssFfM>S<8KZg4=`r04(-Jhl+FEBs`_e`VBb(i4hJr`{gk^WSv8mFqwX!{s>)muWk+%-<@K_D$rf~YPy~oSVYs=WL z&!zumr%w2@$zsDziUxvdAh%Hw#JM^rJ(y?N;;!wjjVdnvH`B}e^|!OhL&LC6)N<{f zMH7=xb>(pDulV+0-6do6>#2i%w#n4qWZjUQY+o4t`_Z-6njhg!tigUexpm?t;=$z?nzM8L8aad>;`&<8qwefjSp}``OYW6W z3ziYt9T=Ll1)L5Qmk6XnIU9vZM()EBeBuK{MBHAm9z+LrQ8bMID$T2tDciF^E^H%i zD(eR)osVN1ulon*bJOK28Ut(~!+iFd{}f>SnB^~t0u##F2hMmZ+00`^!FF!n!b?{v zeIK_!cFW5AhJ?o{;rAZaaJhSI#bW#DmE-Oxh;4vs0>02%)s`9{wR@OaM=8|m|u zR>9^Q8N2Lb8XTih(a?A{rUyWO=iTB);R6rb!}-9aqP0&bcva9z^lULC_tPSzSDf-Y_cO@^Q)RgjbhXDbL)|ovq z&fqrRbLvfOCnG>KyUIfaO*vDy)9R`X${|@G;k1B=bJL;&n~x$C?Q}8oU-nPqqr)(P zp|HMMk1<4g-rcO!8#?-+qRXM-bVXHyaG)4=o4$_zkkYcx!z-($6-+@k*8DyVK_|B9 z2K3@C_R;&5>*J4To*F!{b;LAGdX23^X&#JWe1aNn~&X!5Gl3A!Ra;wZg%U4tj zBoVo3@s>8Z!jV6IYTI=2KEHUey9t~uV*7#g6IHToz@FTC4+BNJ8xrQ>Cl<>a!4p?l z*RmZ3+a%8q7Ta3Tv(w3-2@de}cWZKy29PJf7x7){!w$Vz-d6KnTiHIZeV;fPr)IgJ zy>!{NAF#8}8ge18PJRw{MV=T0=2{^L4Rkqy`E5w{Nazvs zQuZ0&;qC54pU*9E@RWfyRz`_;dg?Qe93u*^4^k!kQuchXBNSCPN#2^ki#TR>#X8W@|v>Dr`-JN@RhOmh-K1XC#8%Oc(T1ix!6yXwW!wBz8)*4 zQ!UXZoL^R6_Hp6&Sm(4u(4^gM1i{AOqD`(zE({e;8Tf=*>)KV}M4{F?xQ$Gw!uu^O z_4sv?Wjyb&zZn>GCF8S*kDDg|HAa$|P~gd$F%&<+AgJFcp-9eI3fDuh%?O44hl(ZI>9NKKnejG9&p z;SZa;RhT^PH(76NWNIXD4u|3IW%KUs1=nt5t)Xll+_DM5ET_Cr6yvn&SVjZ5Wu4b6 z1*;pf0`AYsnefcga9V!A*g5c`Jq>m~T%J)1IGc_!>Y#ux%u<6&(ySL zn4)7Mh!`3OMZLP09qS4u%6-$?e==k)6R_1|P;C_UnRbPx#&Y@QH`Ico@vvoou4(;B zI(^s@^II>be;MAw_iz|{)AXX>0?{e>-P2!>(Q+5bvw0j^|8N2}L71qet4yC$oT8Wo zM<}z!6w7P?kJWA32nTmJCD#$jaP==}R_!&Lw;9)RC+LIHSKZ6u9&%r&%aJGum=@I>~%3X|59jzu)rQBXZw9R1PH%o!_$qn3yk8!tn zcPCtWCUe%w`K46-;^5-WsZLNHzU#Yvx~BI%tF<=&b-@SWFW{n6Aw130vMgD0X}cH; zzDH4&v#pOYjAVE+cYkNl2abDyfW!{=P!+P!fGn3LeM($^luP*sZ0GFM5?V=xs#-$U z5&J%a!aL)zD;w6`{NeXu_kb#xMeoNO&=2M+?|s85O0cO#Pv7@(4O!XlWbj?glR@-Z zhcm*=6#ttU+TIGn$~?^|1GTKq`$>z}ws1P=K|lz2{ks(+#B_17C5Cn#^( z4SnEAp!0nB@1)mS*vKoIG6--)Xg)38-Q{avQH#<}AZnlpnY*0UWeSvu_kGD?z;Z(S z_NZv}y2jaZf)TP8C;Wu+23fFmYPs=lu9+Q(vGqn;gpTOfKHLK088fY7;NDTHXvJ4A zKZb4w4i3|I2j2)WTWoB}K1<^$z*I{RYF}vyI#OF(wxr=V=~x>=3d%%M6j@|^H!8B| zp@H-o?q%%CMfJmwGe%pt8x7X1DH%eO@&Cyd&&X^@Pf|*)YEqYNlqhM6D@1Hnn4FyY z4iDUFprw_SA_5x2-~PpX+T2v- z%=Ho?L8%c)> c*@yX+>tHIh+7M~NH4)*_P}6=?p<)sGKdn;7)Bpeg literal 0 HcmV?d00001 diff --git a/source/tools/monitor/unity/beaver/guide/image/queue.png b/source/tools/monitor/unity/beaver/guide/image/queue.png new file mode 100644 index 0000000000000000000000000000000000000000..01fc076bf35716308fccc70cc0c37e9718810a9c GIT binary patch literal 14649 zcmd_RWl&wuvIh!;0Kwf|LvVL@cXtTxuyJ5}ziKQZ2j!sTi-V+= zGZ+{g+TR--EIkwS5(3UrRTH2oC(CVOZ%c3V-QL)g-rd&WuND}uJ2!~5H3b+EyW85> zIdi-7k^UpW4Wj?D8Ayr$5dm29k!s2*5{uY7nG&JzwC&b zI-59IIsh!~?TG){H8Qq$0q~KM{>|t=*T2dMur&Mcoa~(c{aK(7Wcd4rfr*}x;Xk%P zs=R->+{&iT_BJkm&8yj20{EGE{}KM*?ElW~Us?)ImZqTc{#D~=`nSse-y zZT{hc8iya2m*GDR%?~>Vd;Ap)jGabGR7lkw{45LFFzVrBIOjmzbtDGfI83uDcdSQz zqExd;yFj!`SyiQ|F?w#_T%};399IlO>{QX~qRSWxg)GkTLW=d|OI{i0iQ z<f4-yeo#F8Ul18QI`9xxKStAND)TV9|jp*AB9+NN6k^u zI2;V677Tob4NkC-7({PkfM|UTXo^sf5F9eGUIUmYY~cTY6&OVpC-ntr${40Uo||s1 zUu}1GQevI8oa}aaEf#XU%`c|-{75f99U$)VKni)0cNSw2)iGyb=;{l2b#)hZzqb?_ z0PXSoyq+FS=`+PAqsb>J-Vj8JT`b>&7R9aU~# z?;)j>Dc49x7qdB}WheW(1GfrRV}{tH^qd@}jC%SQ{93IV`sUnD_m41jzb!3KX#LHa z1rRM`F%g~~!~?%SZsHYl#*{bQ8GXZky79ZoauH&`(CB+VP{yE35EeVpp$koa78s+M zHVI2lW+e{5UJJOHakiQI!bfS7+2Z&S`X<+5RYCpvNt8WG*VDsNNVvAufMq4GZD_sH zjAe&=%k#>SInI=!W}CBVO+=UQN@y#Gi#=DR>DUgso<)oaac6M~+dnnFV55%arxO2h z)3DNI<~B8wUG|{tB9tfJLTm@_=JEAP(WYSO>r`nqM!TJU>(k5rMonrfL`%#WCzQdW zyaCWX8N7coiHGVweTlu$R^_3%UmMXY^I8)teFENWaS>h_iYZVo+#tOBt)L_}Mvp%^ zE-9~p*X0iV04g7okCl6bqtqNQwTfi#$Mg=@Nd4n8dJvH6NMVzPkA)yw3%fn~LVch_ zayXPAR-e1wZi5?1BeqE3@+2!4m~{sqya;F>7z3y~d|~P{=s_0nF;F1l>+(KzQWsC2 zygn|g$ixd@`F)tzEz~{DS3vZ~m-9XH%Hj_cI-Ec7mg?v-DJ{E9z03%E!?sfaB`1^s zPoLYl z*2Q5#$O5zTHSuyboPx~hPG_X*cU{{sY#UbJx-?T#ygcRJvF!I`kZ*yY_{XNAoKLJE*bIAI z-QJ5v&Hyef-20dok9WPFAfF|_w|#tb0qNa8@9kGpi!(<(;iC+aMRE4hMT-|)fTfg{ z{etqn;-ENl(V!T@OtdQM-f2lg==08C$y)_=Gnt?NbTg)$0(Viz9esFT(^gTr=$ptM zOKMB*3v3-l8o6{_W8`9b$O>F1jWr1I2tw9i2c{=dtjTN@uX^;AY)L-`zbL}WH*;Gv zS4uY;BNGVo+@4}I;!Y6IAk5lCk#N_j3}pAcvsRp?xj=R6Hyh9ZgI| zWagU(1`Q|H#or5*N2ML^P5NS|^m$djVu&0d%s=6-I9hC!`DDOb&9pfl$h6SQX|aj* zX^ZK&F#VVkbeCgLF*fCkw4>cQA`ku=Hcje zIa(;EuF(8a(E;QLQe!Q0DjWOF0I%VKy}Sh+HSnE2jSR1ak?(nKJZ8D8C_A(`;i@~h zCIFiI@}U9Yb+el6zf@BrnrF(CiIxX_#w)wME1LYnga87m6V*Q;RZfx->S1)^uJ0ci zHITQv7i*{uXO!>JzO%fWpR4&j&0cpfKmc^}=DV?`WpQwwuV&ixZF3lEOO!OIfVDZj z2AYN;5fN%U%tvW8AnzV=(omf&1nE!&N{j7nv9Q0WD*veV43+((pFpQ>;7BPrMgC^AlfYNo@IZof{S8d_a;zF|HgZYL+ zp43$>o5|yjn6eeOm4=g6eO_H!??d?h1SGP!#IohyQh7L^dzDbiLEB;U)OosjPF^Yefw1mMIc!i;V#5ANR=J#u<|QWgd%7 zlOC_-Gy|#UmyLe-%3H<*6(EN=NoK?PK3j`qb^PiiJ<&EFf6*F$0gpA@7K}@!6rGTs zumNiF8;0^2dtD?yCaBM${0sjzj$O5Xh?|9IcV&wd+uU2>Guapm=ln2n(=c+?2v)Th z6@z$s4qxLUC4P${+#*&g<_D&PrKmX{Dh7Zc9+~kE#Fc_{0KU_MWLVRpcZj$2N_oLy zDXuG@1O%EF*c1)5nmyZ&AFmU_f2)XrP$l#asvxVK=7C$*$qXHw&Bh<;deHI}@gQu7Y7hghS+$DTmD{GMr{xoE73~p0y6AyoQDw2NJI%|W z0T4ifSY%raLKk9DWw~~s%3QUIe$IV1%w(=MmKXC2@rzz&GQ!t zfAG%oMyd^Ye&xlb#*0>p7~R+6adXlrRzyzloLIR^#qD-+q5Jve zI}Uro=xN?DlY*&BYq6R3=DBCNrxo^u%YM6X0kXULY4Ypa@mo`3Q#PYQt~H5G`1*`d zTeu$1V$yNavh2O{J{Nj4+)}jTi`>1AQFHCywuiMOILdgCvwfHK4&K5%8`Z*>DTI1R z%afX80X44qp=C4!K*oHAf|b1R;^jymaX0)5u|&l20Mr=Nku3LipBG2smnNaYg&5(< zF}E?dc1HU4@XJ~2A$Ds@%i@N*a*|x$o6SHq6Gk$JlVe=e?T`yc68k1lJpp1(QPqXcd4Zs@|rj5$@9IBZmi zE);Kad(DeETUp)ehryrr;8;AKxjctxmUj4Hce%BdVUuba8ai2>k-l znJN>(O&9rT%mc|3>G4JFp61=mx_(-2RL%JsBh`oZOzGmN;5f)m=(ae?2)IOr(l+S1 z?Y95qvma#d)M2i}bM~xg?Z>GkLhCrgIUcNAq(84dv#=8PgU!3MG$Cy_URG*6?~A$Q z+&Ila^l|iIG&hC7pohpTID+y`U~R<-PdZP(YJYP@i$a{-lje1u-fXxwCxrUEo2OeJW)?})sAP6-9313Rre zVLltSQxZg44NDHg*kM7vD%Yx)H0HDFTb5eJBt`>F)*mC**JC}dtZ?JA&@8O=L|l84 zz0ZJ;#!mrR4fNxON=ILT$rxrUXK7ZIU^R4Q>v}8e*%6EacC1`9G&Cl;CP6m+sLQB! zt^rTKmPW#4jKr@3mG+jta+6Ij2`y?S52_*ooM~0*SmT-7lV}2H(rF5<+v1swxiq;r zL!{E^v+M>CHtYzLiL6R39*eGlN;67Q*M*W&InRm{FyWa_1<{kbBodqMH{XuryJz{Psk_pCm-=|6p3HNOXf%^dxHI@Of5}M{i@Ko_k*-S zx%2{?p=v5IhfSS*$h*<{+S-B*&Wh0{5+TO`Y1NV3CA;RDZ}8&i2I-8G%qa{j$kz-XOun2c;?MOCR@S{1z53Bwt~ zZxZ`P5tf|POxp^s#*%6h6{eX~`FQ!#CL`=sD-U#qUTl>9SEpZH{b$NxCvzv5WmxM8 z8#GhNEiw?YQRps>)nnE3=<|VTn>5!n8!|lyXAULQT=CpYW3(#x+KbUyE(mEaTKCR! z&1NE|B2fc695sDNS-5$W$y?fYE9Yb98Q(K*Tl5&GYbG*{HcT41l_W&Mj$EGZ+_gGw z+5H=>po%7=m{-f0Qqf!ATo6VxDKkQk-B?Un%A0T)Sm8rwaswPiTRX(Npk#ivj5Mnz zsZPw+>EUOp=DccNF|nUkkEwT&^jrnqLX*g2Ok!k>k18E_?EWxm2Dnl;qf4h-*|uG8 z(Ux_aK_a?Y7hCOv?SX}_@Wi`a18lsYQ)5FB_hJ)?=Hups1XhTTh#fPlbIGU+p3`GK z!>7(Q3pVG%>J&xg%cfJxePE`!Kvr8-&lrCqh~_{LNfIe3D}E#UfCg&+>9m%>vjIf; zc-C>Y_tbxw$4@Cr;1Lt}n})c`#wX?POYzKMp8S!c;*?l(g+|=Ou$#v| z!)sj&QR?Dm9m@W)CH9fd4`!qKTYKjD(;&1Po7>&lXjB%j9aD=mBLz$dIQTePUk9>; z>{;U3ZfxoL?AnXalZd0(6}iid^s;m%&ZFdYT&ozuUOt~0fq|Ewdx!RNi-$oOX*HP|JPEV2Z!rjMs#oio~ zywwn-5D0VY#ZYTGRmXy;5@Xw;UJreGC;blf^@_F4B4?0=@eg(TfB!&Mw(4LtW)K)h zPN&V_)n#8hS0e%BSE*JFbVfIiHOJYIb59ooi+Qx;@=(5Q1yo;E_{n`h2T1dzV@!`7 z-lX4L&A4|6G(B53M5Y1chph}@ml`mH4SM45@0#)U#0=bRKt-h(!s^hN4+aA?A zm*iQi8qE4BIx}e`s`8QxH0m{UW!104Mc@CHm3aTX%cgf^B5qwsimKyIltR=}Q<_%+ z>7|yeoY=RDrSl~Z4S`CStFkoDe@+@oh41uz%aTWR__J4d!NUG8&pGvl?y9z`1~z^-1B8a1i%ldQmtXeW^GAA_0i^_BHEZ1b$WG8zk1>{QPR!}{n zHv3!oe1mh7L?Cwv2xg40)#>i&s(-`0OZKSuAk`$V1Zro8qRaij+u0xW+&@zGMnuOQ z)V$N2i@!9EOcq_`4AKNfL^SnO_|^Pcc$9utgKPdQ*W}Rv!%S%DwZIFJR#_57bF`A- zAGK=8A>COGT?VD5r21=G9fEIe!JW{#D{?>9q?)xiW^7$`y>@Sn>8Z|;ol}Ezq1LnENp?s?oYbUNVLE_IR?)P006Hq?EruOK7Sv=| z9b6_KNZvYMb)wXw=Pjr>BR_Wh+e1+!1ph`&BV;b<1E(_FfL#f7CEu$$mtk|K3bmbWr3YF zTf=D~>a>*mxXuXXBswfmz9C(5T~&ekeV_J!X+13~6y>ZCJ8DL;39*szQsE!0cK3G& zk;v2fTG3k48ihRL)q%6>q5ryI*1W2IF@)7wzrb#LTeyR39seDYNiNgp%V3dUr*6N% zgi!?kBD&h$utrZ_mm$Mp$`R!?nLIHhi)dZ*AnHRD*4v`gNL}&OOUN0V?7}>25-}06 z(Y4dn$hu4N(-;ft(aENsPilB_4TG&cYHc+*=AR-9vdZMDp{Y&0Jr2_Bj>un%HZVpn z2mM1##yL8wv%J``EY~cY?|tQUCQb~9f`VQ*=JQh%B)(KZEOTH8KqA&F&%t z`ve+iqhr4hqO=B|>X0gH{F;x1*A2_3BUQ?#{CrZ(g(bp#1Y>lc58V9|4s}+-V*Jn+ zbZ!MEv{h-U?|Xc)yR zR=He^f3pz}P+LWQk2D)G11P%+e~dU!IP*9+S==t^!Vf=Vnm+oLD!(&FFE=l3HOJ&5 z?mJ{A<}@lte|ceKP^PmgvZbEif%$>dWr=p1|-4LhPF*9V5B6eeEv- z9^D_Ji=0jta+wJ^DlAquC<&$Vji2OCL_LDq0!&H=76YOY7|klo!f5(1qP7EHg#|1H zaICk!B!wF%c&b2~h9Qf=O!6I4pLrPg5+x9wNFUE6Y6JPJuu;X}8ogO})W++<52#q- z1jL`D0>$1oUnAS<+j!=9en03MDErj|>al=9mVuvvWKamwwZ^ORs2z01B8c`wUKKXh zrc!%>C}V>UN_W!~#}uhSp?vJzGTaI&3MF~|x{O+rV=amI!CfnFv{=UA!l-IKa>H99 zqhYB;{^#bZYnLz6%W!kV^^;8{=_P0AXN(f?2aIQl7mbyrM(slu^M}`5djd+n+k@^# z6x7{I#fhR5D@Xz>c|J%M6*k_FwPq96(Xp`yBKdL({S&`4ysXd9YRiL;^xY4yt*$DV zi)c@qC5J@ZdU|yFt*i(tDKxZvq5CkZG+KKcYMhFtr+LiWB!&hDmn&~SKd98>;6#dk zcjalUMgaaH<$A@>Q>~ShbcHLdUN>%*9;m6G9gz;`nJZ2=gUj%Oq#AA@<=0*tWL&n&w!zxvvV6Yia;8I~>an}54bxfewGsa_;e7qWdK3m250tcjl z-&!wuXVXbp-g$ZO5*H_wmeu7pgI_84o6Ibmw(uq5s5VJsM###{=jDTDvvj(6b|Hs* zBX(#rdAk0x-r$w6QXfaNTuM7LhiKSKCga$q@{>Qlpc%3p0U0Fx-rMM(Qep`D zgWRm92q*WqUUMSer;}Sl3oo}tB{>;4zF>kf6(q1cC&k--LIT{-j zhbj_wd|IkA-5JVSYGY{y9Oa@?sLQ2YpZp}*vzjaY7^NzEIJebrbx^+egNBMW{|!TT zI{=Pa50_!dWVNM#aiK^?4R$ar$H(<%t&ISfoL4|j$xj%&qS0iv-sMX7jZ*x4x6$%f zAdA<-B)2!&6kM`FL$pU`qK;D?pIbewS&#Un*;Yt z=4}3uiMAkU=(#d9#Y)YJZXbJt_S@k?DfgrN<6*u`cc~L^zD}2u>#LpC-Tqn0_F@_= zAl|ut*UH?w9u_^YzQ>?rNKbffTfMq+v6_HBC5BLMWYKLVhM<~(U`Oj0p7H?=BdyK- zR)1^AuS)iSfGK5_4_PV4o7;ieS#Dx59}0&yyL+3FwO~4(ayF}AU@;R@SA~Yb3y^_t zIBGgX1PYqm-<4rlL;p-*qc9!SMwdszgna@j=o8jhKfy6o-Mj$Zc>MAl6DGayo zVwLxL%aUBrOXoCTWH$nf%4zK89ITEYZ$lO<9;a2fqR|Gtl=OG>+8-C7&Zz3WORXP&w z#r=n=P>q6djsU=Jx(_Qsp|WLl3nwtMva;djeoJeLZ0hr`p0hVQCPrHgof7ti7rnj?1B zJ*d!%_wGG4I6sQ))|*7LlE7K1)<5LPT9 z0TvoduAN?#O}o3hH!2>S%&mQ{)zF>gCaQ`ECwgkBYN2REWEc__0@49t6%`VZap(KY zbb&6*_QGIfrpzN>lNX2i{$J=0C55cN=U4rp*X`s*vi;NP*B?rx)6(s^Eo$b;z0i@G zO$l%v$;^aev&(FCcbqw%PlXA*s>JN_Oe2J7L#GKSC-Pk?v|cn>Z8p+$p13`lA`+-p zE}0Aw3A-e84td7}%h+ff^9w`?Z_KY78k+z|>8+5>-q zT8{Hl`HaUD*34=A@!5K^NVLXWBe1$9h3^-X@0OO%HYq%A1~6dAUdUrgh!%bkjB+3eIBA-O)aY&w2T>KQSFQbDm+( z=>;nF>FSWQ6Wzt*<72y{W7{86U?6hu?TUokxuJIg-m|y+K}<l9@|86aCE1W74EQ4VZ@qLo z!7!Na;tr}5L@}`bf=4HjChNuXjUIyv^(sHQLxWY6+GPc)Ds>_r*GkDa(!9L9W?;6S3G+p#TL)zlX$x;a zWo2c#T1B(PwDIXuT{2RZ@Gf*fx6ey)VPU0q(+{H2Iz1*Lvm8;end^<&EnO{5w%d%H$Z)>YE=;>C>6OP8?@V(ysvJo$F zG)fdOK0dxsr4x(CF*z-Y&|2D^++!{t+|7zBXJc~;deLIDjEas<-iObvncNOI>Z?EP z3xOk`-B>wQ_L%^O<9ndp;b63!DU{FRp0Cupx!wb|@sR2SAg2oWy-ycN_&x1m1r#p! zjTSzqhXni}=Iq=GgU6{i>JPP=FOMMRK}-#qEm7dGSz@H7F4JvuLQM7fo_1^+87PIlk z-2?^qpZL;&Qjm$#6i|M@6Bs>jEfRRc=i-)65JblU>P;ncUC&lZYuId8ZnyfwQkN{< zGpaqLHdpO?1~UiZ$Rw_rU7BUkkJ8UH;pmH{lfIp;wjlAbArbP1m)tpbGV}~aVtu?l zY)Hi5$4TOM`9-Ycb^2+aWeMIcvVy8XJR0|~K94A>OJ=9)N@VgkMZ4F-IjB;4=HvR; z8~}Q5sezMz;%~37jP;>uCx;fm$;nAS!QELaNX0yH^TSS$8_7A&p+akQuLV)`+OV+U zVz~@OEs^NW%B{iF=ut>HKx?&)4xXiRC3gM;>HB$=1jQr?wtA8S$t)Kt%MdwWn z+3sQ`uQJaZt?uq41~@?V<7$}bvBULDJJ8q*VsvyAn?keJ5IjP=`m&55od&ahJVUU{ z^RBi4r5wJjtPDE}xcaHB78x8~>J!gBrNy7lwva${9mY9C44sG-P63S?{lUOsLOxG; z0`9pXB_1@chMo);YOrRc2wjaL2Y$Xv1Hymnoi#DwI zvlNHHbB1ll`>V8nur`{_yS}3tfGQ}GfY2veM%Btbb|+{dEG;b$KM;|SB(Pd##jAo_`uf+* z=H|YsS8{sYpSE!4<9UF#7y=`a*IST5zEAel8d9mBnOH+Z11{uSxuHJ{m1H8l9_|UO zMUk!0kK`c)SmD^$bRjV9(D@Zj#}IaE?!k%6BW(GG`uhBWg8sg~TKylQRsN>(?dpR5 zyX)`}rIlN!e;)pLA+3gQba^ep=NP;F_ynK#!ihjdLK1*zN#|~vP>qpE5@HAiJ|{{k zpSc8b_v4F$X@zEw=rt;HbHultoDjn`zr_hKsm+fK8eqg>2W%iBG3olqLlN-URV&md zY^F%B{R1HIulCN)y1bv9tbDc04M8z}>Ui`R4@i)k4y;oGB$&j)K8mJrwI_qr%=EuP zxJ6SF5F?|bH`**Sqq4zaQqC{Z!XyMTXY~H*@&e&LdP~5j&-r|f%f>Z$Pz2G*3Li=# zo!$0FkcQbrx>&y1eSyi`r}@RG?Ys_d4_HL%(H|l$3bn z!Th=_luGb@y^<(wR4I`!Ej3p02Mr%^BwKwlWJl?jW3sZ;L;2@PG#bbD1Tja5no``x z129acXdb^F!4Ly``xH|3baiX*_V3SD#gZl7zlW{ly;8$T$d1mHsZ3*vwBY(4>%J>F zigby4qq$9Yi_>HP$tYb=((=<4d?>gwz^EJa3V;JX@;!q?REOOZfEsd`v2WfXE)(l#>YR z*!;fmk?{C=M_VL!mxg@2tb~N$kdT@Z&~{E=k1JSu5*bTV@8K9Sg{sUh#V~%bY%&bc z^SsP+P#rXp@ae^8)5|HC&(5;p;8+zYpY82!?d|E@i9q#Px92^|%?G?!!XF!b` z2>C>xh$I5{QoVJmb=#Osh75Uc<%YkjIDiWJ*<6xDGZn1d9+@)=&$vp>={j1crFy;N zL6P1Z;#1L|Kd?kZO)VTdI|Xkmrsl{+@gA?5FLE;d22$GTycA*~&MHBHuONJDRcboQ z4g|R$_}pPMvrlso2MvDtS*?Xn{aKS3!h$1v-Zy5w(}Rj! zRKHTwwa!>Jjx?&nDtanQI3z^1RIy1o1QwZ-wxV~)*}~qK z7c${4Q!%Ufr!w}2;!DK#L9ylllWw3-%E0LbwOm(bq zQ6pmVP24!tLL4}{BH?qs?xk#_u>BOj@fbngf4WsppN?(JaCTpDS{3$JSYdLsaR;Ge8eps$5#j{4eh9oYdkYnb0W-Og#T&<5`#~+;T zG{cscocVQ@fM8^oP`(&WR9Ji47vc^EeT!ic`}G+_T*D@7QOMaA zhMj}qfnhk}D^eNKqx2D2)yw4oWKl7u)S4eYeM+-jo{3MO71Uk}!`%CdjJD8p0<+bX zlOUW<&BM_N!iFn)7&jvap^$~v<3G{TF+K!^Ktoj^&dlPRM@m^S zZZe?yACq%%VEb+OXz+-FO%x$1fPO!&3W>ks48aI8!>fMMyGG{mY4dngtl)%-htvxi z!qJi_>Ph^eBk>vUljH7OgM*PRU6hIfxwyShKwaW2U&r5l<8ZD&8gn6uX;Sx-4`ew; zh`OJ*KeIgSsYe`t49+M)$^@$YuHOpqn_(kFhoJan1u0ol8JfHZp0V-PdvdC{4cd!N z3;}+@g&-YdDkB(-G)z;=y{FN8}_I_9kGfZ|9H}kFQ;*!xh5;2W5+hsOMy*i8V>O;rnP=Y3U3e zJK(1z`JwvB%$~y1VHm%W@Xpebb}bHjC8Cvgf7r#32#>|(G+*lHpO1FM&$t^h;WH^n z5Eoroum(QQuWz6pet*$3Um!96@&KJgf7anLm>A`JKzQ6i!tw#~JXbwYUqj5$z8d`TTdt4o>eSudcAYP0+nBO{tue;(Gv9MfsmE~&V5cve_nx?36cTS{hC?SfEk`RsTYL`o#e<3 z6BLDPwX_)d{dn~e9H!v+%vxci)x+;&RjI#Dm|P-X)cInE4Bzv1-!5W%G_z>77WmV4 z1%Lx}eltvTz=*HZ-MP_&d0WE>qv#`ty(d~Wo&eT96hqj>VLlnRKLIoryqPJQQ{aF4 z6qH?ET^|D1G|N`5fh69e)8#p6KEb_x{LOx*a4wF_ZCa5HrcESlzChe39P6l%Xy3d{9PZZqtj}KHq*dsha`) zTpGHlx#?5JqV5Sof=;vSY}2M0gb0TOK*0NlhGH_Omc}I&^;V%$bQaC%hED$Wmy?o` zNRp#Wz~mLWx{}Q_Wl_Imjo^bU2OS3pCgAjQYC|7Oq5*^VG+vRYZZXtqt%d8h?~u8F zrTPg3{`iO}M*nATXcWjNu|R~`fmUFMB|(E}AT7HzE6Bx&|BbZ-h_rzG4Kg|C?28a4 z9z{CnybBp99uw7@rvvgdF!3PoLmU~6DhC$ip~5S6px?;#b#hY zx?~QAm>`pZpj9Bs5#s_;G|7ju+mh zlzfdxurF>vee$|H9G^O1IEG-o+egOYt4f)&{BnI%LIN>MEcZ&0%=po)yxV#0-7#CK z`q_LyVc>WaBbr=0<+u}?eK`i~V4L&7WG0vA&3@KTWRL0mWg{cd2+vvIzvwYJVtBr8 z5}%N!(~|k`@x2lo2L**BrH4N{aS;buV;i*5RxO)!3<2_de&Ht`%q4OM|MGBzgd_cS^((%bWxkdzdckdT%YZI8(RSfTh; z?b{tN7Rp>@3ykk|Fw9g7zlJKw>ZAKr+JQYum)_acwQpGK#qqzUpAP^xw};1voo!)> icFihLI`qqrPdhJEH-J!EUC@z1Fex#4(Q0AC!2bv9VrD@A literal 0 HcmV?d00001 diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 7595760e..2b002141 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -35,9 +35,11 @@ cp beeQ/run.sh ${APP}/beeQ/ mkdir ${APP}/collector mkdir ${APP}/collector/native +mkdir ${APP}/collector/outline cp collector/native/*.so* ${APP}/collector/native/ cp collector/native/*.lua ${APP}/collector/native/ cp collector/*.lua ${APP}/collector/ +cp collector/outline/*.lua ${APP}/collector/outline cp collector/plugin.yaml ${APP}/collector/ mkdir ${APP}/common diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 0221f07c..9c23dcdd 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -9,4 +9,8 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../install/ export LUA_PATH="../../lua/?.lua;../../lua/?/init.lua;" export LUA_CPATH="../../lib/?.so;../../lib/loadall.so;" -./unity-mon +yaml_path=$1 +[ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" + +echo $yaml_yaml_path +./unity-mon $yaml_path diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index cc3f9066..8e3f3ec8 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -7,7 +7,7 @@ config: mode: specify name: test_specify # mode: hostip - proc_path: /mnt/home/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ + proc_path: /mnt/host/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ # proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ outline: diff --git a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c index ceee0758..169e5062 100644 --- a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c +++ b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c @@ -17,6 +17,7 @@ int init(void * arg) { } static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; while (plugin_is_working()) { static double value = 1.0; struct unity_line* line; @@ -29,7 +30,10 @@ static int sample_thread_func(struct beeQ* q, void * arg) { unity_set_value(line, 1, "value2", 2.0 + value); unity_set_log(line, "log", "hello world."); beeQ_send(q, lines); - sleep(1); + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } } return 0; } -- Gitee From 22208dea742fd8d737493485d03c7ce4b525475c Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 22 Feb 2023 18:05:44 +0800 Subject: [PATCH 12/77] modify link. --- source/tools/monitor/unity/beaver/guide/guide.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index d6b61fa3..654a28c5 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,8 @@ # 目录 - -1. [插件化与热更新](/guide/hotplugin.md) -2. [面向对象设计](/guide/oop.md) -3. [字符串处理](/guide/pystring.md) -4. [页面开发](/guide/webdevel.md) -5. [proc和probe记录表](/guide/proc_probe.md) -6. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file +1. [开发手册](/guide/guide.md) +2. [插件化与热更新](/guide/hotplugin.md) +3. [面向对象设计](/guide/oop.md) +4. [字符串处理](/guide/pystring.md) +5. [页面开发](/guide/webdevel.md) +6. [proc和probe记录表](/guide/proc_probe.md) +7. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file -- Gitee From 42ad0363a9bfd4c393442778cccb509a831bcd4e Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Thu, 23 Feb 2023 07:38:27 +0000 Subject: [PATCH 13/77] proc_schedstat: Add max and min statistics Signed-off-by: Hailong Liu --- .../plugin/proc_schedstat/proc_schedstat.c | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index f4691264..5d1075f8 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -21,6 +21,7 @@ struct sched_stats { long nr_cpus; char *real_proc_path; struct unity_line **lines1; +struct sched_stats curr_min, curr_max; struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; int init(void * arg) @@ -73,6 +74,9 @@ int init(void * arg) printf("WARN: proc_schedstat install FAIL calloc 4\n"); return ret; } + + curr_min.delay = -1ULL; + curr_max.delay = 0; printf("proc_schedstat plugin install.\n"); return 0; } @@ -143,6 +147,16 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline1[cpu], 0, "cpu", cpu_name); unity_set_value(uline1[cpu], 0, "pcount", delta[cpu].pcount); unity_set_value(uline1[cpu], 1, "delay", delta[cpu].delay); + if (delta[cpu].delay > curr_max.delay) { + curr_max.delay = delta[cpu].delay; + curr_max.pcount = delta[cpu].pcount; + strcpy(curr_max.cpu_name, cpu_name); + } + if (delta[cpu].delay < curr_min.delay) { + curr_min.delay = delta[cpu].delay; + curr_min.pcount = delta[cpu].pcount; + strcpy(curr_min.cpu_name, cpu_name); + } } } @@ -158,6 +172,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline2, 0, "summary", "avg"); unity_set_value(uline2, 0, "pcount", sum_cnt/nr_cpus); unity_set_value(uline2, 1, "delay", sum_delay/nr_cpus); + tmp = curr; curr = oldp; oldp = tmp; @@ -168,16 +183,28 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) int call(int t, struct unity_lines* lines) { int i = 0; static double value = 0.0; - struct unity_line* line2; + struct unity_line *line2, *line3, *line4; - unity_alloc_lines(lines, nr_cpus+1); + unity_alloc_lines(lines, nr_cpus+3); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); - unity_set_table(lines1[i], "sched_moni"); + unity_set_table(lines1[i], "proc_schedstat"); } line2 = unity_get_line(lines, nr_cpus); - unity_set_table(line2, "sched_moni"); + unity_set_table(line2, "proc_schedstat"); full_line(lines1, line2); + + line3 = unity_get_line(lines, nr_cpus+1); + unity_set_table(line3, "proc_schedstat"); + unity_set_index(line3, 0, "max", curr_max.cpu_name); + unity_set_value(line3, 0, "pcount", curr_max.delay); + unity_set_value(line3, 1, "delay", curr_max.pcount); + + line4 = unity_get_line(lines, nr_cpus+2); + unity_set_table(line4, "proc_schedstat"); + unity_set_index(line4, 0, "min", curr_min.cpu_name); + unity_set_value(line4, 0, "pcount", curr_min.delay); + unity_set_value(line4, 1, "delay", curr_min.pcount); return 0; } -- Gitee From d5c5fa17c9a2c2d0182d0ec26ec3cdcb1f82e7db Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 23 Feb 2023 23:06:53 +0800 Subject: [PATCH 14/77] change insert to index. use yaml to manage lua plugin. --- source/tools/monitor/unity/beaver/export.lua | 18 +++++-- source/tools/monitor/unity/beaver/frame.lua | 4 +- .../monitor/unity/beaver/guide/develop.md | 4 ++ .../tools/monitor/unity/beaver/identity.lua | 4 +- .../tools/monitor/unity/beeQ/proto_queue.lua | 14 ++++-- source/tools/monitor/unity/collector/loop.lua | 38 ++++++--------- .../unity/collector/outline/pipeMon.lua | 16 +++++-- .../tools/monitor/unity/collector/plugin.lua | 12 +++-- .../tools/monitor/unity/collector/plugin.yaml | 3 ++ .../monitor/unity/collector/proc_mounts.lua | 4 +- .../monitor/unity/collector/proc_netdev.lua | 6 +-- .../unity/collector/proc_snmp_stat.lua | 8 +++- .../monitor/unity/collector/proc_sockstat.lua | 4 +- .../monitor/unity/collector/proc_statm.lua | 4 +- .../tools/monitor/unity/collector/vproc.lua | 4 +- .../tools/monitor/unity/common/lineParse.lua | 11 +++-- .../tools/monitor/unity/common/pystring.lua | 24 +++++++--- .../tools/monitor/unity/httplib/httpComm.lua | 7 ++- .../unity/test/curl/beaver/beavers.lua | 4 +- .../tools/monitor/unity/test/host/hostIp.lua | 4 +- source/tools/monitor/unity/test/string/py.lua | 7 +++ source/tools/monitor/unity/tsdb/foxTSDB.lua | 48 +++++++++++++------ 22 files changed, 165 insertions(+), 83 deletions(-) diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index e1436c60..b6f3b52e 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -24,11 +24,13 @@ local function qFormData(from, tData) local res = {} local len = #tData local last = 0 + local c = 0 for i = len, 1, -1 do local line = tData[i] if from == line.title then if last == 0 or last == line.time then - table.insert(res, line) + c = c + 1 + res[c] = line last = line.time else break @@ -40,8 +42,10 @@ end local function packLine(title, ls, v) local tLs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(tLs, string.format("%s=\"%s\"", k , v)) + c = c + 1 + tLs[c] = string.format("%s=\"%s\"", k , v) end local label = "" if #tLs then @@ -56,15 +60,18 @@ function Cexport:export() self._fox:resize() self._fox:qlast(self._freq, qs) local res = {} + local c = 0 for _, line in ipairs(self._tDescr) do local from = line.from local tFroms = qFormData(from, qs) if #tFroms then local title = line.title local help = string.format("# HELP %s %s", title, line.help) - table.insert(res, help) + c = c + 1 + res[c] = help local sType = string.format("# TYPE %s %s", title, line.type) - table.insert(res, sType) + c = c + 1 + res[c] = sType for _, tFrom in ipairs(tFroms) do local labels = system:deepcopy(tFrom.labels) @@ -74,7 +81,8 @@ function Cexport:export() labels.instance = self._instance for k, v in pairs(tFrom.values) do labels[line.head] = k - table.insert(res, packLine(title, labels, v)) + c = c + 1 + res[c] = packLine(title, labels, v) end end end diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 09e95484..29b98103 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -21,11 +21,13 @@ end local function waitDataRest(fread, rest, tReq) local len = 0 local tStream = {tReq.data} + local c = #tStream while len < rest do local s = fread() if s then len = len + #s - table.insert(tStream, s) + c = c + 1 + tStream[c] = s else return -1 end diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 4e78d9e3..11945179 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -59,6 +59,10 @@ config: outline: # 外部数据入口，适合接入外部数据场景 - /tmp/sysom # 外部unix socket 路径，可以指定多个 +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] # 控制lua 插件加载 + + plugins: # 插件列表 对应 /collector/plugin 路径下编译出来的c库文件。 - so: kmsg # 库名 description: "collect dmesg info." # 描述符 diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 27582ac6..1c4e93e3 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -25,8 +25,8 @@ end local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/beeQ/proto_queue.lua b/source/tools/monitor/unity/beeQ/proto_queue.lua index f1988d89..ecece0f1 100644 --- a/source/tools/monitor/unity/beeQ/proto_queue.lua +++ b/source/tools/monitor/unity/beeQ/proto_queue.lua @@ -21,12 +21,14 @@ function CprotoQueue:que() end function CprotoQueue:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -34,12 +36,14 @@ function CprotoQueue:load_label(unity_line, line) end function CprotoQueue:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -56,7 +60,8 @@ function CprotoQueue:load_log(unity_line, line) end function CprotoQueue:_proc(unity_lines, lines) - for i=0, unity_lines.num - 1 do + local c = #lines["lines"] + for i = 0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), ls = {}, @@ -66,7 +71,8 @@ function CprotoQueue:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 1b278181..7e970dc8 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -7,19 +7,7 @@ require("common.class") local CprotoData = require("common.protoData") local procffi = require("collector.native.procffi") - -local CprocStat = require("collector.proc_stat") -local CprocMeminfo = require("collector.proc_meminfo") -local CprocVmstat = require("collector.proc_vmstat") -local CprocNetdev = require("collector.proc_netdev") -local CprocDiskstats = require("collector.proc_diskstats") -local CprocSockStat = require("collector.proc_sockstat") -local CprocSnmpStat = require("collector.proc_snmp_stat") -local CprocMounts = require("collector.proc_mounts") -local CprocStatm = require("collector.proc_statm") -local CprocBuddyinfo = require("collector.proc_buddyinfo") local Cplugin = require("collector.plugin") - local system = require("common.system") local Cloop = class("loop") @@ -27,21 +15,23 @@ local Cloop = class("loop") function Cloop:_init_(que, proto_q, fYaml) local res = system:parseYaml(fYaml) self._proto = CprotoData.new(que) - self._procs = { - CprocStat.new(self._proto, procffi, res.config.proc_path), - CprocMeminfo.new(self._proto, procffi, res.config.proc_path), - CprocVmstat.new(self._proto, procffi, res.config.proc_path), - CprocNetdev.new(self._proto, procffi, res.config.proc_path), - CprocDiskstats.new(self._proto, procffi, res.config.proc_path), - CprocSockStat.new(self._proto, procffi, res.config.proc_path), - CprocSnmpStat.new(self._proto, procffi, res.config.proc_path), - CprocMounts.new(self._proto, procffi, res.config.proc_path), - CprocStatm.new(self._proto, procffi, res.config.proc_path), - CprocBuddyinfo.new(self._proto, procffi, res.config.proc_path), - } + self:loadLuaPlugin(res, res.config.proc_path) self._plugin = Cplugin.new(self._proto, procffi, que, proto_q, fYaml) end +function Cloop:loadLuaPlugin(res, proc_path) + local luas = res.luaPlugins + + self._procs = {} + if res.luaPlugins then + for i, plugin in ipairs(luas) do + local CProcs = require("collector." .. plugin) + self._procs[i] = CProcs.new(self._proto, procffi, proc_path) + end + end + print("add " .. #self._procs .. " lua plugin.") +end + function Cloop:work(t) local lines = self._proto:protoTable() for k, obj in pairs(self._procs) do diff --git a/source/tools/monitor/unity/collector/outline/pipeMon.lua b/source/tools/monitor/unity/collector/outline/pipeMon.lua index 1298192d..81f20696 100644 --- a/source/tools/monitor/unity/collector/outline/pipeMon.lua +++ b/source/tools/monitor/unity/collector/outline/pipeMon.lua @@ -35,11 +35,11 @@ end function CpipeMon:setupPipe(fYaml) local res = system:parseYaml(fYaml) - for _, path in ipairs(res.outline) do + for i, path in ipairs(res.outline) do if unistd.access(path) then unistd.unlink(path) end - table.insert(self._paths, path) + self._paths[i] = path socket.unix = require("socket.unix") local s = socket.unix.udp() @@ -58,14 +58,20 @@ local function trans(title, ls, vs, log) local values = {} local logs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(labels, {name=k, index=v}) + c = c + 1 + labels[c] = {name=k, index=v} end + c = 0 for k, v in pairs(vs) do - table.insert(values, {name=k, value=v}) + c = c + 1 + values[c] = {name=k, value=v} end + c = 0 for k, v in pairs(log) do - table.insert(logs, {name=k, log=v}) + c = c + 1 + logs[c] = {name=k, log=v} end return {line = title, ls = labels, vs = values, log = logs} end diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 41b21746..4ccf946b 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -55,12 +55,14 @@ function Cplugin:setup(plugins, proto_q) end function Cplugin:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -68,12 +70,14 @@ function Cplugin:load_label(unity_line, line) end function Cplugin:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -90,6 +94,7 @@ function Cplugin:load_log(unity_line, line) end function Cplugin:_proc(unity_lines, lines) + local c = #lines["lines"] for i=0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), @@ -100,7 +105,8 @@ function Cplugin:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index b682bbe3..ca6170ee 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -13,6 +13,9 @@ config: outline: - /tmp/sysom +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] + plugins: - so: kmsg description: "collect dmesg info." diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 4de16c32..6ad8644a 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -23,8 +23,10 @@ local function get_lines(fName) local fName = fName or "/proc/mounts" local f = assert(io.open(fName, "r")) + local c = 0 for line in f:lines() do - table.insert(lines, line) + c = c + 1 + lines[c] = line end return lines end diff --git a/source/tools/monitor/unity/collector/proc_netdev.lua b/source/tools/monitor/unity/collector/proc_netdev.lua index 7a2f0954..2682bfdb 100644 --- a/source/tools/monitor/unity/collector/proc_netdev.lua +++ b/source/tools/monitor/unity/collector/proc_netdev.lua @@ -27,7 +27,7 @@ function CprocNetdev:_getNewValue(ifName, data) local now = {} local index = self:_netdevIndex() for i = 1, #index do - table.insert(now, tonumber(data.value[i - 1])) + now[i] = tonumber(data.value[i - 1]) end self._lastData[ifName] = now self._lastIfNames[ifName] = 1 @@ -43,13 +43,13 @@ function CprocNetdev:_calcIf(ifName, data, res, elapsed) } for i, index in ipairs(index) do local nowValue = tonumber(data.value[i -1]) - table.insert(now, nowValue) + now[i] = nowValue local value = (nowValue - res[i]) / elapsed local cell = { name = index, value = value } - table.insert(protoTable.vs, cell) + protoTable.vs[i] = cell end self:appendLine(protoTable) diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index 199bdbea..917cd7d4 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -58,12 +58,14 @@ end function CprocSnmpStat:pack(labels, logs) local vs = {} + local c = 0 for k, v in pairs(labels) do local value = { name = k, value = tonumber(v) } - table.insert(vs, value) + c = c + 1 + vs[c] = value end self:appendLine(self:_packProto("pkt_status", nil, vs)) if #logs > 0 then @@ -82,6 +84,7 @@ function CprocSnmpStat:check(now) local labels = self:createLabels() local logs = {} if self._rec then + local c = 0 for k, v in pairs(now) do if self._rec[k] and self._rec[k] < v then -- local delta = v - self._rec[k] @@ -91,7 +94,8 @@ function CprocSnmpStat:check(now) labels[lk] = lv + delta end end - table.insert(logs, string.format("%s: %d", k, tonumber(delta))) + c = c + 1 + logs[c] = string.format("%s: %d", k, tonumber(delta)) end end end diff --git a/source/tools/monitor/unity/collector/proc_sockstat.lua b/source/tools/monitor/unity/collector/proc_sockstat.lua index d906ae97..bf7eb80d 100644 --- a/source/tools/monitor/unity/collector/proc_sockstat.lua +++ b/source/tools/monitor/unity/collector/proc_sockstat.lua @@ -17,6 +17,7 @@ end function CprocSockStat:proc(elapsed, lines) CvProc.proc(self) local vs = {} + local c = 0 for line in io.lines(self.pFile) do local cells = pystring:split(line, ":", 1) if #cells > 1 then @@ -31,7 +32,8 @@ function CprocSockStat:proc(elapsed, lines) name = title, value = tonumber(bodies[2 * i]) } - table.insert(vs, v) + c = c + 1 + vs[c] = v end end end diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index d3ebeb60..9a9fff79 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -16,6 +16,7 @@ end function CprocStatm:proc(elapsed, lines) CvProc.proc(self) local heads = {"size", "resident", "shared", "text", "lib", "data", "dt"} + local c = 0 for line in io.lines("/proc/self/statm") do local vs = {} local data = self._ffi.new("var_long_t") @@ -26,7 +27,8 @@ function CprocStatm:proc(elapsed, lines) name = k, value = tonumber(data.value[i - 1]), } - table.insert(vs, cell) + c = c + 1 + vs[c] = cell end self:appendLine(self:_packProto("self_statm", nil, vs)) end diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 33c016a5..30d8fa3d 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -31,8 +31,10 @@ function CvProc:copyLine(line) end function CvProc:push(lines) + local c = #lines for _, v in ipairs(self._lines["lines"]) do - table.insert(lines["lines"], v) + c = c + 1 + lines["lines"][c] = v end self._lines = nil return lines diff --git a/source/tools/monitor/unity/common/lineParse.lua b/source/tools/monitor/unity/common/lineParse.lua index ee8de957..33d6486a 100644 --- a/source/tools/monitor/unity/common/lineParse.lua +++ b/source/tools/monitor/unity/common/lineParse.lua @@ -81,18 +81,23 @@ function module.pack(title, ls, vs) local line = title if system:keyCount(ls) > 0 then local lss = {} + local c = 0 for k, v in pairs(ls) do - table.insert(lss, k .. "=" .. v) + c = c + 1 + lss[c] = table.concat({k, v}, "=") end line = line .. ',' .. pystring:join(",", lss) end local vss = {} + local c = 0 for k, v in pairs(vs) do local tStr = type(v) if tStr == "number" then - table.insert(vss, k .. '=' .. tostring(v)) + c = c + 1 + vss[c] = table.concat({k, tostring(v)}, "=") elseif tStr == "string" then - table.insert(vss, k .. '=' .. json.encode(v)) + c = c + 1 + vss[c] = table.concat({k, json.encode(v)}, "=") else error("bad value type for " .. tStr) end diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index 25278335..ea9a08da 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -55,8 +55,10 @@ end local function setupDelimiter(delimiter) local rt = {} + local i = 0 for c in string.gmatch(delimiter, ".") do - table.insert(rt, checkDelimiter(c)) + i = i + 1 + rt[i] = checkDelimiter(c) end return table.concat(rt) end @@ -78,18 +80,22 @@ function pystring:split(s, delimiter, n) local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(s, delimiter, beg) if (iBeg) then - table.insert(result, string.sub(s, beg, iBeg - 1)) + c = c + 1 + result[c] = string.sub(s, beg, iBeg - 1) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end else - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end end @@ -112,19 +118,23 @@ function pystring:rsplit(s, delimiter, n) rDel = setupDelimiter(rDel) local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(rs, rDel, beg) if (iBeg) then - table.insert(result, string.sub(s, len - (iBeg - 1),len - beg)) + c = c + 1 + result[c] = string.sub(s, len - (iBeg - 1),len - beg)) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end else - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end end diff --git a/source/tools/monitor/unity/httplib/httpComm.lua b/source/tools/monitor/unity/httplib/httpComm.lua index 12b691f3..3fdff02b 100644 --- a/source/tools/monitor/unity/httplib/httpComm.lua +++ b/source/tools/monitor/unity/httplib/httpComm.lua @@ -87,12 +87,15 @@ function ChttpComm:packHeaders(headTable, len) -- just for http out. end local origin = originHeader() + local c = 0 for k, v in pairs(origin) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end for k, v in pairs(headTable) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end return pystring:join("\r\n", lines) .. "\r\n" end diff --git a/source/tools/monitor/unity/test/curl/beaver/beavers.lua b/source/tools/monitor/unity/test/curl/beaver/beavers.lua index 619c4334..af296673 100644 --- a/source/tools/monitor/unity/test/curl/beaver/beavers.lua +++ b/source/tools/monitor/unity/test/curl/beaver/beavers.lua @@ -86,13 +86,15 @@ function Cbeavers:poll() if err then print("socket select return " .. err) end + local c = 0 for _, read in pairs(reads) do if type(read) == "number" then break elseif read == self._server then local s = read:accept() print("accept " .. s:getfd()) - table.insert(self._ss, s) + c = c + 1 + self._ss[c] = s local co = coroutine.create(function(o, s) self._proc(o, s) end) self._cos[s:getfd()] = co coroutine.resume(co, self, s) diff --git a/source/tools/monitor/unity/test/host/hostIp.lua b/source/tools/monitor/unity/test/host/hostIp.lua index 1b0266a6..84b4d4a0 100644 --- a/source/tools/monitor/unity/test/host/hostIp.lua +++ b/source/tools/monitor/unity/test/host/hostIp.lua @@ -9,8 +9,8 @@ local socket = require("socket") local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index 04cb3142..ee816857 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -14,6 +14,13 @@ assert(#ret == 3) assert(ret[1] == "hello") assert(ret[2] == "lua") assert(ret[3] == "language") +ret = pystring:split("hello lua language lua language") +assert(#ret == 5 ) +assert(ret[1] == "hello") +assert(ret[2] == "lua") +assert(ret[3] == "language") +assert(ret[4] == "lua") +assert(ret[5] == "language") -- 自定符号分割 ret = pystring:split("hello*lua *language", "*") diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index d5efb6fd..8fab6339 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -240,6 +240,7 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday self:curMove(start) -- moveto position + local lenMs = #ms for line in self:loadData(stop) do local time = line.time for _, v in ipairs(line.lines) do @@ -261,37 +262,50 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday end tCell.values = values - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell end end return ms end +function CfoxTSDB:_qlast(date, beg, stop, ms) + if not self._man then -- check _man is already installed. + if self:_setupRead(beg) ~= 0 then -- try to create new + return ms + end + end + + if self.cffi.check_pman_date(self._man, date) == 1 then + return self:query(beg, stop, ms) + else + self:_del_() + if self:_setupRead(beg) ~= 0 then -- try to create new + return ms + end + return self:query(beg, stop, ms) + end +end + function CfoxTSDB:qlast(last, ms) assert(last < 24 * 60 * 60) local now = self:get_us() - local date = self:getDateFrom_us(now) local beg = now - last * 1e6; - if not self._man then -- check _man is already installed. - if self:_setupRead(now) ~= 0 then -- try to create new - return ms - end - end + local dStart = self:getDateFrom_us(beg) + local dStop = self:getDateFrom_us(now) - if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day - return self:query(beg, now, ms) + if self.cffi.check_foxdate(dStart, dStop) ~= 0 then + self:_qlast(dStart, beg, now, ms) else - local dStop = self:getDateFrom_us(now) - local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 local now2 = now - ms = self:query(beg1, now1, ms) - return self:query(beg2, now2, ms) + self:_qlast(dStart, beg1, now1, ms) + self:_qlast(dStop, beg2, now2, ms) end end @@ -307,6 +321,7 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) budget = budget or self._qBudget self:curMove(start) local inc = false + local lenMs = #ms for line in self:loadData(stop) do inc = false local time = line.time @@ -339,7 +354,8 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) end tCell.logs = logs - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell inc = true end end @@ -364,11 +380,13 @@ function CfoxTSDB:qDayTables(start, stop, tbls) end self:curMove(start) + local lenTbls = #tbls for line in self:loadData(stop) do for _, v in ipairs(line.lines) do local title = v.line if not system:valueIsIn(tbls, title) then - table.insert(tbls, title) + lenTbls = lenTbls + 1 + tbls[lenTbls] = title end end end -- Gitee From 5692aec0fb00e0c02ffff441be38e0559a60d6a2 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 10:25:29 +0800 Subject: [PATCH 15/77] coolbpf: update Signed-off-by: Shuyi Cheng --- source/lib/internal/ebpf/coolbpf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lib/internal/ebpf/coolbpf b/source/lib/internal/ebpf/coolbpf index 0562b139..735ae613 160000 --- a/source/lib/internal/ebpf/coolbpf +++ b/source/lib/internal/ebpf/coolbpf @@ -1 +1 @@ -Subproject commit 0562b1397b8a8997b16d752d874dc5ad74149149 +Subproject commit 735ae6138430822502e1bd5ae2a366a8668ecd7e -- Gitee From 067b990ab7d893b2911d9172f5503a8d9641ff32 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 10:25:48 +0800 Subject: [PATCH 16/77] unity: malloc -> calloc Signed-off-by: Shuyi Cheng --- source/tools/monitor/unity/collector/plugin/bpf_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 6acf256f..745da01f 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -40,7 +40,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ -- Gitee From 0ac2d30fcb5ad533dcc12c4a7342095a4d5bc0c2 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 14:47:45 +0800 Subject: [PATCH 17/77] unity: fix calloc Signed-off-by: Shuyi Cheng --- source/tools/monitor/unity/collector/plugin/bpf_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 745da01f..e9d76240 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -40,7 +40,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = calloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(1, sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ -- Gitee From fedf3e8b0abef08afcc728bb2cad77ee2c742209 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sun, 26 Feb 2023 01:41:23 +0800 Subject: [PATCH 18/77] add net retrans monitor. --- .../monitor/unity/beaver/localBeaver.lua | 4 + .../monitor/unity/collector/native/Makefile | 2 +- .../monitor/unity/collector/native/fastKsym.c | 166 ++++++++++ .../monitor/unity/collector/native/fastKsym.h | 20 ++ .../monitor/unity/collector/native/sig_stop.c | 19 +- .../tools/monitor/unity/collector/plugin.yaml | 25 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../monitor/unity/collector/plugin/bpf_head.h | 70 ++++- .../unity/collector/plugin/cpudist/Makefile | 8 + .../collector/plugin/cpudist/cpudist.bpf.c | 40 +++ .../unity/collector/plugin/cpudist/cpudist.c | 86 ++++++ .../unity/collector/plugin/cpudist/cpudist.h | 8 + .../collector/plugin/net_health/Makefile | 8 + .../plugin/net_health/net_health.bpf.c | 29 ++ .../collector/plugin/net_health/net_health.c | 108 +++++++ .../collector/plugin/net_health/net_health.h | 8 + .../collector/plugin/net_retrans/Makefile | 8 + .../plugin/net_retrans/net_retrans.bpf.c | 283 ++++++++++++++++++ .../plugin/net_retrans/net_retrans.c | 215 +++++++++++++ .../plugin/net_retrans/net_retrans.h | 48 +++ .../unity/collector/plugin/plugin_head.h | 1 + .../tools/monitor/unity/common/pystring.lua | 5 +- .../tools/monitor/unity/httplib/httpBase.lua | 15 +- source/tools/monitor/unity/test/string/py.lua | 2 + source/tools/monitor/unity/tsdb/foxTSDB.lua | 6 +- 25 files changed, 1169 insertions(+), 17 deletions(-) create mode 100644 source/tools/monitor/unity/collector/native/fastKsym.c create mode 100644 source/tools/monitor/unity/collector/native/fastKsym.h create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.h create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 1f253c64..cb062006 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -176,12 +176,16 @@ function CLocalBeaver:write(fd, stream) stream = string.sub(stream, sent + 1) sent, err, errno = socket.send(fd, stream) if sent == nil then + if errno == 11 then -- EAGAIN ? + goto continue + end system:posixError("socket send error.", err, errno) return nil end else -- need to read ? may something error or closed. return nil end + ::continue:: end res = self._cffi.mod_fd(self._efd, fd, 0) -- epoll read ev only assert(res == 0) diff --git a/source/tools/monitor/unity/collector/native/Makefile b/source/tools/monitor/unity/collector/native/Makefile index 03d3d2f8..d185cf7f 100644 --- a/source/tools/monitor/unity/collector/native/Makefile +++ b/source/tools/monitor/unity/collector/native/Makefile @@ -1,7 +1,7 @@ CC := gcc CFLAG := -g -fpic LDFLAG := -g -fpic -shared -OBJS := procffi.o sig_stop.o unity_interface.o +OBJS := procffi.o sig_stop.o unity_interface.o fastKsym.o SO := libprocffi.so all: $(SO) diff --git a/source/tools/monitor/unity/collector/native/fastKsym.c b/source/tools/monitor/unity/collector/native/fastKsym.c new file mode 100644 index 00000000..ea0e2db8 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.c @@ -0,0 +1,166 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#include "fastKsym.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int tfd = 0; +static int sym_cnt = 0; +static struct ksym_cell * gCell = NULL; + +static int load_ksyms(int fd, int stack_only) { + int ret = 0; + int count = 0; + struct ksym_cell cell; + void * addr; + char buf[128]; + + FILE *pf = fopen("/proc/kallsyms", "r"); + + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open /proc/kallsyms failed, errno, %d, %s", errno, strerror(errno)); + goto endOpen; + } + + while (!feof(pf)) { + if (!fgets(buf, sizeof(buf), pf)) + break; + + ret = sscanf(buf, "%p %c %64s %31s", &addr, &cell.type, cell.func, cell.module); + if (ret == 3) { + cell.module[0] = '\0'; + } else if (ret < 3) { + fprintf(stderr, "bad kallsyms line: %s", buf); + goto endRead; + } + + if (!addr) + continue; + + if (stack_only && (cell.type != 't') && (cell.type != 'T')) { + continue; + } + cell.addr = (addr_t) addr; + + ret = write(fd, &cell, sizeof (cell)); + if (ret < 0) { + fprintf(stderr, "write file failed, errno, %d, %s", errno, strerror(errno)); + goto endWrite; + } + count ++; + } + + fclose(pf); + return count; + + endWrite: + endRead: + fclose(pf); + endOpen: + return ret; +} + +static int sym_cmp(const void *p1, const void *p2) +{ + return ((struct ksym_cell *)p1)->addr > ((struct ksym_cell *)p2)->addr; +} + +static int sort_ksym(int fd, int count) { + int ret = 0 ; + struct stat sb; + void *pmmap; + + ret = fstat(fd, &sb); + if (ret < 0) { + fprintf(stderr, "fstat file failed, errno, %d, %s", errno, strerror(errno)); + goto endStat; + } + + pmmap = mmap(NULL, sb.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (pmmap == NULL) { + fprintf(stderr, "mmap file failed, errno, %d, %s", errno, strerror(errno)); + ret = -EACCES; + goto endMmap; + } + + qsort(pmmap, count, sizeof (struct ksym_cell), sym_cmp); + + gCell = (struct ksym_cell*)pmmap; + + return ret; + endMmap: + endStat: + return ret; +} + +int ksym_setup(int stack_only) { + int ret; + + FILE *pf = tmpfile(); + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open file failed, errno, %d, %s", errno, strerror(errno)); + goto endTmpfile; + } + + tfd = fileno(pf); + + ret = load_ksyms(tfd, stack_only); + if (ret < 0) { + goto endLoad; + } + sym_cnt = ret; + + ret = sort_ksym(tfd, ret); + if (ret < 0) { + goto endSort; + } + + return ret; + endSort: + endLoad: + close(tfd); + endTmpfile: + return ret; +} + +struct ksym_cell* ksym_search(addr_t key) { + int start = 0, end = sym_cnt; + int mid; + + if (sym_cnt <= 0) { + printf("sym_cnt: %d", sym_cnt); + return NULL; + } + + while (start < end) { + mid = start + (end - start) / 2; + + if (key < gCell[mid].addr) { + end = mid; + } else if (key > gCell[mid].addr) { + start = mid + 1; + } else { + return &gCell[mid]; + } + } + + if (start > 0) { + if ((gCell[start - 1].addr < key) && (key < gCell[start].addr)) { + return &gCell[start - 1]; + } + } + if (start == sym_cnt) { + return &gCell[end - 1]; + } + return NULL; +} diff --git a/source/tools/monitor/unity/collector/native/fastKsym.h b/source/tools/monitor/unity/collector/native/fastKsym.h new file mode 100644 index 00000000..57bf8be2 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.h @@ -0,0 +1,20 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#ifndef FASTKSYM_FASTKSYM_H +#define FASTKSYM_FASTKSYM_H + +typedef unsigned long addr_t; + +struct ksym_cell { + addr_t addr; + char func[64]; + char module[31]; + char type; +}; + +int ksym_setup(int stack_only); +struct ksym_cell* ksym_search(addr_t key); + +#endif //FASTKSYM_FASTKSYM_H diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index ec3d865b..c006072d 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -4,8 +4,10 @@ #include "sig_stop.h" #include - #include +#include +#include +#include "fastKsym.h" static volatile int working = 1; @@ -38,7 +40,22 @@ static void sig_register(void) { sigaction(SIGQUIT, &action, NULL); } +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + void plugin_init(void) { + bump_memlock_rlimit1(); + ksym_setup(1); sig_register(); working = 1; } diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index ca6170ee..96bc4339 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -38,6 +38,10 @@ plugins: - so: unity_nosched description: "nosched:sys hold cpu and didn't scheduling" + - so: net_health + description: "tcp net health." + - so: net_retrans + description: "tcp retrans monitor." metrics: - title: sysak_proc_cpu_total @@ -150,4 +154,23 @@ metrics: head: value help: "nosched:sys hold cpu and didn't scheduling" type: "gauge" - + - title: sysak_cpu_dist + from: cpu_dist + head: value + help: "task cpu sched dist." + type: "gauge" + - title: sysak_net_health_hist + from: net_health_hist + head: value + help: "net_health_hist" + type: "gauge" + - title: sysak_net_health_count + from: net_health_count + head: value + help: "net_health_count" + type: "gauge" + - title: sysak_net_retrans_count + from: net_retrans_count + head: value + help: "net_retrans_count" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index b29bc4e8..af6ab29b 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 6acf256f..99adf37f 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -4,15 +4,22 @@ #ifndef UNITY_BPF_HEAD_H #define UNITY_BPF_HEAD_H +#include -#define DEFINE_SEKL_OBJECT(skel_name) \ - struct skel_name##_bpf *skel_name = NULL; \ - static pthread_t perf_thread = 0; +#ifdef COOLBPF_PERF_THREAD -#define DESTORY_SKEL_BOJECT(skel_name) \ - if (perf_thread > 0) \ - kill_perf_thread(perf_thread); \ - skel_name##_bpf__destroy(skel_name) +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } #define LOAD_SKEL_OBJECT(skel_name, perf) \ ( \ @@ -40,7 +47,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(1, sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ @@ -57,4 +64,51 @@ __ret; \ }) +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); +#else +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); +#endif + +#define coobpf_map_find(OBJ, NAME) bpf_object__find_map_fd_by_name(OBJ, NAME) +#define coobpf_key_next(FD, KEY, NEXT) bpf_map_get_next_key(FD, KEY, NEXT) +#define coobpf_key_value(FD, KEY, VALUE) bpf_map_lookup_elem(FD, KEY, VALUE) + +#include "plugin_head.h" #endif //UNITY_BPF_HEAD_H diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/Makefile b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile new file mode 100644 index 00000000..9fc693eb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := cpudist.bpf.c +csrcs := cpudist.c +so := libcpudist.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c new file mode 100644 index 00000000..1dc01053 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c @@ -0,0 +1,40 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#include +#include + +BPF_ARRAY(cpudist, u64, 20); +BPF_HASH(start, u32, u64, 128 * 1024); + +struct sched_switch_args { + u16 type; + u8 flag; + u8 preeempt; + u32 c_pid; + char prev_comm[16]; + u32 prev_pid; + u32 prev_prio; + u64 prev_state; + char next_comm[16]; + u32 next_pid; + u32 next_prio; +}; +SEC("tracepoint/sched/sched_switch") +int sched_switch_hook(struct sched_switch_args *args){ + u64 ts = ns(); + u64 *pv; + u32 prev = args->prev_pid; + u32 next = args->next_pid; + + if (next > 0) { + bpf_map_update_elem(&start, &next, &ts, BPF_ANY); + } + pv = bpf_map_lookup_elem(&start, &prev); + if (pv && ts > *pv) { + hist10_push((struct bpf_map_def *)&cpudist, (ts - *pv) / 1000); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c new file mode 100644 index 00000000..5ea28bac --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c @@ -0,0 +1,86 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + + +#include "cpudist.h" +#include "../bpf_head.h" +#include "cpudist.skel.h" + +#define CPU_DIST_INDEX 8 +#define DIST_ARRAY_SIZE 20 +DEFINE_SEKL_OBJECT(cpudist); +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("cpudist plugin install.\n"); + ret = LOAD_SKEL_OBJECT(cpudist, perf); + dist_fd = coobpf_map_find(cpudist->obj, "cpudist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < CPU_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[CPU_DIST_INDEX]; + const char *names[] = {"us1", "us10", "us100", "ms1", "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "cpu_dist"); + + cal_dist(values); + for (i = 0; i < CPU_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("cpudist plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(cpudist); +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h new file mode 100644 index 00000000..1bcac9a4 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#ifndef UNITY_CPUDIST_H +#define UNITY_CPUDIST_H + +#endif //UNITY_CPUDIST_H diff --git a/source/tools/monitor/unity/collector/plugin/net_health/Makefile b/source/tools/monitor/unity/collector/plugin/net_health/Makefile new file mode 100644 index 00000000..55801edc --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_health.bpf.c +csrcs := net_health.c +so := libnet_health.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c new file mode 100644 index 00000000..5efee8bb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -0,0 +1,29 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// +#include +#include + +BPF_ARRAY(outCnt, u64, 2); +BPF_ARRAY(netHist, u64, 20); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +SEC("kprobe/tcp_validate_incoming") +int j_tcp_validate_incoming(struct pt_regs *ctx) { + struct tcp_sock *tp = (struct tcp_sock *)PT_REGS_PARM1(ctx); + u64 ts = BPF_CORE_READ(tp, srtt_us) >> 3; + u64 ms = ts / 1000; + if (ms > 0) { + addCnt(0, ms); + addCnt(1, 1); + hist10_push((struct bpf_map_def *)&netHist, ms); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c new file mode 100644 index 00000000..ea63d234 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -0,0 +1,108 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_health.h" +#include "../bpf_head.h" +#include "net_health.skel.h" + +#define NET_DIST_INDEX 4 +#define DIST_ARRAY_SIZE 20 + +DEFINE_SEKL_OBJECT(net_health); +static int cnt_fd = 0; +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("net_health plugin install.\n"); + ret = LOAD_SKEL_OBJECT(net_health, perf); + cnt_fd = coobpf_map_find(net_health->obj, "outCnt"); + dist_fd = coobpf_map_find(net_health->obj, "netHist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < NET_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +static int get_count(unsigned long* values) { + int key; + static unsigned long rec[2]; + unsigned long now[2]; + + key = 0; + coobpf_key_value(cnt_fd, &key, &now[0]); + key = 1; + coobpf_key_value(cnt_fd, &key, &now[1]); + + values[0] = now[0] - rec[0]; rec[0] = now[0]; + values[1] = now[1] - rec[1]; rec[1] = now[1]; + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[NET_DIST_INDEX]; + const char *names[] = { "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 2); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_health_hist"); + + cal_dist(values); + for (i = 0; i < NET_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + get_count(values); + line = unity_get_line(lines, 1); + unity_set_table(line, "net_health_count"); + unity_set_value(line, 0, "sum", values[0]); + unity_set_value(line, 1, "count", values[1]); + return 0; +} + +void deinit(void) +{ + printf("net_health plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_health); +} + + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.h b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h new file mode 100644 index 00000000..dd3cebc2 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_HEALTH_H +#define UNITY_NET_HEALTH_H + +#endif //UNITY_NET_HEALTH_H diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile new file mode 100644 index 00000000..09638c6e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_retrans.bpf.c +csrcs := net_retrans.c +so := libnet_retrans.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c new file mode 100644 index 00000000..70769cba --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c @@ -0,0 +1,283 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include +#include +#include "net_retrans.h" + +struct liphdr { + __u8 ver_hdl; + __u8 tos; + __be16 tot_len; + __be16 id; + __be16 frag_off; + __u8 ttl; + __u8 protocol; + __sum16 check; + __be32 saddr; + __be32 daddr; +}; + +#define MAX_ENTRY 128 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +BPF_PERF_OUTPUT(perf, 1024); +BPF_STACK_TRACE(stack, MAX_ENTRY); +BPF_ARRAY(outCnt, u64, NET_RETRANS_TYPE_MAX); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +static inline int get_tcp_info(struct data_t* pdata, struct tcp_sock *ts) +{ + pdata->rcv_nxt = BPF_CORE_READ(ts, rcv_nxt); + pdata->rcv_wup = BPF_CORE_READ(ts, rcv_wup); + pdata->snd_nxt = BPF_CORE_READ(ts, snd_nxt); + pdata->snd_una = BPF_CORE_READ(ts, snd_una); + pdata->copied_seq = BPF_CORE_READ(ts, copied_seq); + pdata->snd_wnd = BPF_CORE_READ(ts, snd_wnd); + pdata->rcv_wnd = BPF_CORE_READ(ts, rcv_wnd); + + pdata->lost_out = BPF_CORE_READ(ts, lost_out); + pdata->packets_out = BPF_CORE_READ(ts, packets_out); + pdata->retrans_out = BPF_CORE_READ(ts, retrans_out); + pdata->sacked_out = BPF_CORE_READ(ts, sacked_out); + pdata->reordering = BPF_CORE_READ(ts, reordering); + return 0; +} + +static inline int get_skb_info(struct data_t* pdata, struct sk_buff *skb, u32 type) +{ + u16 offset; + u8 ihl; + void* head; + struct liphdr *piph; + struct tcphdr *ptcph; + + addCnt(type, 1); + pdata->type = type; + pdata->sk_state = 0; + + head = (void*)BPF_CORE_READ(skb, head); + offset = BPF_CORE_READ(skb, network_header); + piph = (struct liphdr *)(head + offset); + ihl = _(piph->ver_hdl) & 0x0f; + ptcph = (struct tcphdr *)((void *)piph + ihl * 4); + + pdata->ip_dst = _(piph->daddr); + pdata->dport = BPF_CORE_READ(ptcph, dest); + pdata->ip_src = _(piph->saddr); + pdata->sport = BPF_CORE_READ(ptcph, source); + return 0; +} + +static inline void get_list_task(struct list_head* phead, struct data_t* e) { + struct list_head *next = BPF_CORE_READ(phead, next); + if (next) { + wait_queue_entry_t *entry = container_of(next, wait_queue_entry_t, entry); + struct poll_wqueues *pwq = (struct poll_wqueues *)BPF_CORE_READ(entry, private); + if (pwq) + { + struct task_struct* tsk = (struct task_struct*)BPF_CORE_READ(pwq, polling_task); + if (tsk) { + e->pid = BPF_CORE_READ(tsk, pid); + bpf_probe_read(&e->comm[0], TASK_COMM_LEN, &tsk->comm[0]); + } + } + } +} + +static inline void get_sock_task(struct sock *sk, struct data_t* e) { + struct socket_wq *wq = BPF_CORE_READ(sk, sk_wq); + if (wq) { + struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); + get_list_task(phead, e); + } +} + +static inline void get_task(struct data_t* pdata, struct sock *sk) { + pdata->pid = 0; + pdata->comm[0] = '\0'; + + get_sock_task(sk, pdata); +} + +static inline void get_socket_task(struct data_t* e, struct sock *sk) { +// struct socket* psocket; +// struct socket_wq *wq; + + e->pid = 0; + e->comm[0] = '\0'; + +// psocket = BPF_CORE_READ(sk, sk_socket); +// wq = BPF_CORE_READ(psocket, wq); +// if (wq) { +// struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); +// get_list_task(phead, e); +// } +} + +static inline int get_info(struct data_t* pdata, struct sock *sk, u32 type) +{ + struct inet_sock *inet = (struct inet_sock *)sk; + + addCnt(type, 1); + pdata->type = type; + pdata->ip_dst = BPF_CORE_READ(sk, __sk_common.skc_daddr); + pdata->dport = BPF_CORE_READ(sk, __sk_common.skc_dport); + pdata->ip_src = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); + pdata->sport = BPF_CORE_READ(inet, inet_sport); + pdata->sk_state = BPF_CORE_READ(sk, __sk_common.skc_state); + return 0; +} + +static inline int check_inner(unsigned int ip) +{ + int i; + const unsigned int array[3][2] = { + {0x0000000A, 0x000000ff}, + {0x000010AC, 0x0000f0ff}, + {0x0000A8C0, 0x0000ffff}, + }; + + if (ip == 0) { + return 1; + } +#pragma unroll 3 + for (i =0; i < 3; i ++) { + if ((ip & array[i][1]) == array[i][0]) { + return 1; + } + } + return 0; +} + +static inline int check_ip(struct data_t* pdata) { + return check_inner(pdata->ip_src) && check_inner(pdata->ip_dst); +} + +SEC("kprobe/tcp_enter_loss") +int j_tcp_enter_loss(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat != 1) { + return 0; + } + get_task(&data, sk); + get_info(&data, sk, NET_RETRANS_TYPE_RTO); + data.stack_id = 0; + get_tcp_info(&data, (struct tcp_sock *)sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_probe0") +int j_tcp_send_probe0(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == 0) { + return 0; + } + + get_info(&data, sk, NET_RETRANS_TYPE_ZERO); + data.stack_id = 0; + get_task(&data, sk); + get_tcp_info(&data, (struct tcp_sock *)sk); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} + +SEC("kprobe/tcp_v4_send_reset") +int j_tcp_v4_send_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + if (sk == NULL) { + struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); + get_skb_info(&data, skb, NET_RETRANS_TYPE_RST); + get_task(&data, NULL); + data.stack_id = 0; + } + else { + get_info(&data, sk, NET_RETRANS_TYPE_RST_SK); + get_task(&data, sk); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + } + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_active_reset") +int j_tcp_send_active_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_RST_ACTIVE); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + + get_task(&data, sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +#define TCP_SYN_SENT 2 +#define TCPF_SYN_SENT (1 << TCP_SYN_SENT) +SEC("kprobe/tcp_retransmit_skb") +int j_tcp_retransmit_skb(struct pt_regs *ctx){ + struct sock *sk; + unsigned char stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == TCP_SYN_SENT) + { + struct data_t data = {}; + + get_info(&data, sk, NET_RETRANS_TYPE_SYN); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_rtx_synack") +int j_tcp_rtx_synack(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_SYN_ACK); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c new file mode 100644 index 00000000..2f5a1969 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -0,0 +1,215 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_retrans.h" +#define COOLBPF_PERF_THREAD +#include "../bpf_head.h" +#include "net_retrans.skel.h" + +#include +#include + +#include +#include +#include + +static volatile int budget = 0; // for log budget +static int cnt_fd = 0; +static int stack_fd = 0; + +const char *net_title[] = {"rto_retrans", "zero_probe", \ + "noport_reset", "bad_sync", \ + "net_proc", "syn_send", "syn_ack"}; + +int proc(int stack_fd, struct data_t *e, struct unity_line *line); +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + int ret; + if (budget > 0) { + struct data_t *e = (struct data_t *)data; + struct beeQ *q = (struct beeQ *)ctx; + struct unity_line *line; + struct unity_lines *lines = unity_new_lines(); + + printf("receive: %d msg.\n", data_sz); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + ret = proc(stack_fd, e, line); + if (ret >= 0) { + beeQ_send(q, lines); + } + budget --; + } +} + +DEFINE_SEKL_OBJECT(net_retrans); +int init(void *arg) +{ + int ret; + printf("net_retrans plugin install.\n"); + + ret = LOAD_SKEL_OBJECT(net_retrans, perf); + cnt_fd = coobpf_map_find(net_retrans->obj, "outCnt"); + stack_fd = coobpf_map_find(net_retrans->obj, "stack"); + return ret; +} + +static int get_count(unsigned long *locals) { + int i = 0; + + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + coobpf_key_value(cnt_fd, &i, &locals[i]); + } + return i; +} + +static int cal_retrans(unsigned long *values) { + int i; + static unsigned long rec[NET_RETRANS_TYPE_MAX] = {0}; + unsigned long locals[NET_RETRANS_TYPE_MAX]; + + get_count(locals); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +int call(int t, struct unity_lines *lines) { + int i; + unsigned long values[NET_RETRANS_TYPE_MAX]; + struct unity_line* line; + + budget = t; //release log budget + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_retrans_count"); + + cal_retrans(values); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + unity_set_value(line, i, net_title[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("net_retrans plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_retrans); +} + +#define LOG_MAX 256 +static char log[LOG_MAX]; + +static char * transIP(unsigned long lip) { + struct in_addr addr; + memcpy(&addr, &lip, sizeof(lip)); + return inet_ntoa(addr); +} + +static const char * resetSock(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_v4_rcv") == 0) { + if (e->sk_state == 12) { + return "bad_ack"; // TCP_NEW_SYN_REC + } else { + return "tw_rst"; + } + } else if (strcmp(cell->func, "tcp_check_req") == 0) { + return "bad_syn"; + } else if (strcmp(cell->func, "tcp_v4_do_rcv") == 0) { + return "tcp_stat"; + } else { + return "unknown_sock"; + } + } + } + return "failure_sock"; +} + +static const char * resetActive(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_out_of_resources") == 0) { + return "tcp_oom"; + } else if (strcmp(cell->func, "tcp_keepalive_timer") == 0) { + return "keep_alive"; + } else if (strcmp(cell->func, "inet_release") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_close") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_disconnect") == 0) { + return "tcp_abort"; + } else if (strcmp(cell->func, "tcp_abort") == 0) { + return "tcp_abort"; + } else { + return "unknown_active"; + } + } + } + return "failure_active"; +} + +int proc(int stack_fd, struct data_t *e, struct unity_line *line) { + snprintf(log, LOG_MAX, "task:%d|%s, tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ + transIP(e->ip_src), htons(e->sport), \ + transIP(e->ip_src), htons(e->sport), \ + e->sk_state); + switch (e->type) { + case NET_RETRANS_TYPE_RTO: + case NET_RETRANS_TYPE_ZERO: + case NET_RETRANS_TYPE_SYN: + case NET_RETRANS_TYPE_SYN_ACK: + { + char buf[LOG_MAX]; + snprintf(buf, LOG_MAX, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " + "snd_wnd:%d, rcv_wnd:%d, lost_out:%d, packets_out:%d, retrans_out:%d, " + "sacked_out:%d, reordering:%d", + e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, + e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, + e->sacked_out, e->reordering + ); + strncat(log, buf, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST: + strncat(log, "noport", LOG_MAX); + break; + case NET_RETRANS_TYPE_RST_SK: + { + const char *type = resetSock(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST_ACTIVE: + { + const char *type = resetActive(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + default: + break; + } + unity_set_table(line, "net_retrans_log"); + unity_set_index(line, 0, "type", net_title[e->type]); + unity_set_log(line, "log", log); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h new file mode 100644 index 00000000..5d953924 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h @@ -0,0 +1,48 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_RETRANS_H +#define UNITY_NET_RETRANS_H + +#define TASK_COMM_LEN 16 + +enum { + NET_RETRANS_TYPE_RTO, + NET_RETRANS_TYPE_ZERO, + NET_RETRANS_TYPE_RST, + NET_RETRANS_TYPE_RST_SK, + NET_RETRANS_TYPE_RST_ACTIVE, + NET_RETRANS_TYPE_SYN, + NET_RETRANS_TYPE_SYN_ACK, + NET_RETRANS_TYPE_MAX, +}; + + +struct data_t { + char comm[TASK_COMM_LEN]; + unsigned int pid; + unsigned int type; + unsigned int ip_src; + unsigned int ip_dst; + unsigned short sport; + unsigned short dport; + unsigned short sk_state; + unsigned short stack_id; + + unsigned int rcv_nxt; + unsigned int rcv_wup; + unsigned int snd_nxt; + unsigned int snd_una; + unsigned int copied_seq; + unsigned int snd_wnd; + unsigned int rcv_wnd; + + unsigned int lost_out; + unsigned int packets_out; + unsigned int retrans_out; + unsigned int sacked_out; + unsigned int reordering; +}; + +#endif //UNITY_NET_RETRANS_H diff --git a/source/tools/monitor/unity/collector/plugin/plugin_head.h b/source/tools/monitor/unity/collector/plugin/plugin_head.h index 202a110f..ee00783d 100644 --- a/source/tools/monitor/unity/collector/plugin/plugin_head.h +++ b/source/tools/monitor/unity/collector/plugin/plugin_head.h @@ -39,6 +39,7 @@ struct unity_lines { #include "../../beeQ/beeQ.h" #include "../native/sig_stop.h" #include "../native/unity_interface.h" +#include "../native/fastKsym.h" inline struct unity_lines *unity_new_lines(void) __attribute__((always_inline)); inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index ea9a08da..c38a0d1f 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -75,6 +75,9 @@ end function pystring:split(s, delimiter, n) local result = {} + if not delimiter or delimiter == "" then -- for blank, gsub multi blank to single + s = string.gsub(s, "%s+", " ") + end local delimiter = setupDelimiter(delimiter or " ") local n = n or 2 ^ 63 - 1 @@ -124,7 +127,7 @@ function pystring:rsplit(s, delimiter, n) local iBeg, iEnd = string.find(rs, rDel, beg) if (iBeg) then c = c + 1 - result[c] = string.sub(s, len - (iBeg - 1),len - beg)) + result[c] = string.sub(s, len - (iBeg - 1),len - beg) beg = iEnd + 1 nums = nums + 1 if nums >= n then diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 432b622e..3ec1ff4f 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -31,10 +31,19 @@ end local function checkKeep(tReq) local conn = tReq.header["connection"] - if conn and string.lower(conn) == "close" then - return false + if tReq.vers == "1.0" then + if conn and string.lower(conn) == "keep-alive" then + return true + else -- for http 1.0, close as default + return false + end + else + if conn and string.lower(conn) == "close" then + return false + else -- for http 1.1 and newer, for keep-alive as default + return true + end end - return true end function ChttpBase:call(tReq) diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index ee816857..8befc8ea 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -21,6 +21,8 @@ assert(ret[2] == "lua") assert(ret[3] == "language") assert(ret[4] == "lua") assert(ret[5] == "language") +ret = pystring:split("Node 0, zone DMA 1 0 0 1 2 1 1 0 1 1 3") +assert(#ret == 15) -- 自定符号分割 ret = pystring:split("hello*lua *language", "*") diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index 8fab6339..807e1ea1 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -291,7 +291,7 @@ function CfoxTSDB:qlast(last, ms) assert(last < 24 * 60 * 60) local now = self:get_us() - local beg = now - last * 1e6; + local beg = now - last * 1e6 local dStart = self:getDateFrom_us(beg) local dStop = self:getDateFrom_us(now) @@ -299,6 +299,7 @@ function CfoxTSDB:qlast(last, ms) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:_qlast(dStart, beg, now, ms) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -405,6 +406,7 @@ function CfoxTSDB:qDate(dStart, dStop, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -433,6 +435,7 @@ function CfoxTSDB:qNow(sec, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -461,6 +464,7 @@ function CfoxTSDB:qTabelNow(sec) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDayTables(beg, now, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 -- Gitee From 833f3eeb34b5632b368b1aafd26b2feafd9ebe58 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Sun, 26 Feb 2023 13:43:06 +0800 Subject: [PATCH 19/77] unity: netlink: add netlink to monitor packet drop count Signed-off-by: Shuyi Cheng --- .../monitor/unity/collector/plugin/Makefile | 2 +- .../unity/collector/plugin/netlink/Makefile | 19 ++++ .../unity/collector/plugin/netlink/netlink.c | 90 +++++++++++++++++++ .../unity/collector/plugin/netlink/netlink.h | 11 +++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/netlink.c create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/netlink.h diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index af6ab29b..c2a64f2b 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans netlink all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/netlink/Makefile b/source/tools/monitor/unity/collector/plugin/netlink/Makefile new file mode 100644 index 00000000..cfe0e649 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := netlink.o +SO := libnetlink.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c new file mode 100644 index 00000000..350bec23 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c @@ -0,0 +1,90 @@ +#include "netlink.h" +#include +#include +#include + +static char buffer[4096]; + +int get_conntrack_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("conntrack -S", "r"); // 将命令ls-l 同过管道读到fp + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, " drop=")) != NULL) + { + buf += strlen(" drop="); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + +int get_tc_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("tc -s qdisc", "r"); // 将命令ls-l 同过管道读到fp + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, "dropped ")) != NULL) + { + buf += strlen("dropped "); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + + +int init(void * arg) { + printf("netlink plugin install\n"); + return 0; +} + +int call(int t, struct unity_lines* lines) { + struct unity_line* line; + + unity_alloc_lines(lines, 3); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "netlink"); + unity_set_value(line, 0, "conntrack_drop", get_conntrack_drop()); + unity_set_value(line, 1, "tc_drop", get_tc_drop()); + + return 0; +} + +void deinit(void) { + printf("netlink plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.h b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h new file mode 100644 index 00000000..4a55af11 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h @@ -0,0 +1,11 @@ +#ifndef __NET_LINK_H +#define __NET_LINK_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + + +#endif \ No newline at end of file -- Gitee From 54af888a5849428feaa8c06fba0015e55e2b317d Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Sun, 26 Feb 2023 14:00:37 +0800 Subject: [PATCH 20/77] unity: netlink: remove comment Signed-off-by: Shuyi Cheng --- .../tools/monitor/unity/collector/plugin/netlink/netlink.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c index 350bec23..4bae1997 100644 --- a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c @@ -9,7 +9,7 @@ int get_conntrack_drop() { int total_drop = 0, i; FILE *fp = NULL; - fp = popen("conntrack -S", "r"); // 将命令ls-l 同过管道读到fp + fp = popen("conntrack -S", "r"); if (!fp) return -1; @@ -40,7 +40,7 @@ int get_tc_drop() { int total_drop = 0, i; FILE *fp = NULL; - fp = popen("tc -s qdisc", "r"); // 将命令ls-l 同过管道读到fp + fp = popen("tc -s qdisc", "r"); if (!fp) return -1; @@ -76,7 +76,7 @@ int init(void * arg) { int call(int t, struct unity_lines* lines) { struct unity_line* line; - unity_alloc_lines(lines, 3); // 预分配好 + unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); unity_set_table(line, "netlink"); unity_set_value(line, 0, "conntrack_drop", get_conntrack_drop()); -- Gitee From ed8be4e7ca459e805938d979c48065ce84fc4266 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Sun, 26 Feb 2023 07:18:16 +0000 Subject: [PATCH 21/77] unity: Add irqoff check Signed-off-by: Hailong Liu --- .../tools/monitor/unity/collector/plugin.yaml | 5 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../collector/plugin/unity_irqoff/Makefile | 8 + .../plugin/unity_irqoff/unity_irqoff.bpf.c | 151 ++++++++++++ .../plugin/unity_irqoff/unity_irqoff.c | 233 ++++++++++++++++++ .../plugin/unity_irqoff/unity_irqoff.h | 48 ++++ 6 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 96bc4339..3c5ae4a1 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -42,6 +42,9 @@ plugins: description: "tcp net health." - so: net_retrans description: "tcp retrans monitor." + - + so: unity_irqoff + description: "irqoff:detect irq turned off and can't response" metrics: - title: sysak_proc_cpu_total @@ -152,7 +155,7 @@ metrics: - title: sched_moni_jitter from: sched_moni_jitter head: value - help: "nosched:sys hold cpu and didn't scheduling" + help: "nosched/irqoff:sys and irqoff hold cpu and didn't scheduling" type: "gauge" - title: sysak_cpu_dist from: cpu_dist diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index c2a64f2b..cec50dc7 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans netlink +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile new file mode 100644 index 00000000..9e86c359 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_irqoff.bpf.c +csrcs := unity_irqoff.c +so := libunity_irqoff.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c new file mode 100644 index 00000000..136445aa --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c @@ -0,0 +1,151 @@ +#include +#include +#include "sched_jit.h" +#include "unity_irqoff.h" + +#define PERF_MAX_STACK_DEPTH 127 +#define MAX_ENTRIES 10240 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +BPF_PERF_OUTPUT(perf, 1024); + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 10000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct arg_info); +} arg_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct tm_info); +} tm_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u64); + __type(value, struct info); +} info_map SEC(".maps"); + +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +static inline u64 get_thresh(void) +{ + u64 thresh, i = 0; + struct arg_info *argp; + + argp = bpf_map_lookup_elem(&arg_map, &i); + if (argp) + thresh = argp->thresh; + else + thresh = -1; + + return thresh; +} + +SEC("perf_event") +int hw_irqoff_event(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.stamp = now; + event.delay = delta; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } + + return 0; +} + +SEC("perf_event") +int sw_irqoff_event1(struct bpf_perf_event_data *ctx) +{ + int ret, i = 0; + struct tm_info *tmifp, tm; + + tmifp = bpf_map_lookup_elem(&tm_map, &i); + if (tmifp) { + tmifp->last_stamp = bpf_ktime_get_ns(); + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = bpf_ktime_get_ns(); + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + return 0; +} + +SEC("perf_event") +int sw_irqoff_event2(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp, tm; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + tmifp->last_stamp = now; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.delay = delta; + event.stamp = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = now; + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c new file mode 100644 index 00000000..81f89ea9 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c @@ -0,0 +1,233 @@ +#include +#include +#include +#include +//#include +//#include +//#include +#include +#include +#include +#include +#include +#include +#include "unity_irqoff.h" +#include "sched_jit.h" +#include "unity_irqoff.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +struct env { + __u64 sample_period; + __u64 threshold; +} env = { + .threshold = 50*1000*1000, /* 10ms */ +}; + +static int nr_cpus; +struct sched_jit_summary summary, *percpu_summary; +struct bpf_link **sw_mlinks, **hw_mlinks= NULL; + +DEFINE_SEKL_OBJECT(unity_irqoff); + +static int +open_and_attach_perf_event(struct perf_event_attr *attr, + struct bpf_program *prog, + struct bpf_link *links[]) +{ + int i, fd; + + for (i = 0; i < nr_cpus; i++) { + fd = syscall(__NR_perf_event_open, attr, -1, i, -1, 0); + if (fd < 0) { + /* Ignore CPU that is offline */ + if (errno == ENODEV) + continue; + fprintf(stderr, "failed to init perf sampling: %s\n", + strerror(errno)); + return -1; + } + links[i] = bpf_program__attach_perf_event(prog, fd); + if (!links[i]) { + fprintf(stderr, "failed to attach perf event on cpu: %d\n", i); + close(fd); + return -1; + } + } + return 0; +} + +/* surprise! return 0 if failed! */ +static int attach_prog_to_perf(struct unity_irqoff_bpf *obj) +{ + int ret = 0; + + struct perf_event_attr attr_hw = { + .type = PERF_TYPE_HARDWARE, + .freq = 0, + .sample_period = env.sample_period*2, /* refer to watchdog_update_hrtimer_threshold() */ + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + + struct perf_event_attr attr_sw = { + .type = PERF_TYPE_SOFTWARE, + .freq = 0, + .sample_period = env.sample_period, + .config = PERF_COUNT_SW_CPU_CLOCK, + }; + + if (!open_and_attach_perf_event(&attr_hw, obj->progs.hw_irqoff_event, hw_mlinks)) { + ret = 1<progs.sw_irqoff_event1, sw_mlinks)) + ret = ret | 1<progs.sw_irqoff_event2, sw_mlinks)) + ret = 1<num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event e; + e = *((struct event *)data); + e.delay = e.delay/(1000*1000); + if (e.cpu > nr_cpus - 1) + return; + update_summary(&summary, &e); +} + +static int irqoff_handler(void *arg, struct unity_irqoff_bpf *unity_irqoff) +{ + int arg_key = 0, err = 0; + struct arg_info arg_info = {}; + int arg_fd; + + /*memset(summary, 0, sizeof(struct sched_jit_summary)); */ + struct perf_thread_arguments *perf_args = + calloc(sizeof(struct perf_thread_arguments), 1); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); + return -ENOMEM; + } + perf_args->mapfd = bpf_map__fd(unity_irqoff->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + + arg_fd = bpf_map__fd(unity_irqoff->maps.arg_map); + arg_info.thresh = env.threshold; + env.sample_period = env.threshold*2/5; + err = bpf_map_update_elem(arg_fd, &arg_key, &arg_info, 0); + if (err) { + fprintf(stderr, "Failed to update arg_map\n"); + return err; + } + + if (!(err = attach_prog_to_perf(unity_irqoff))) + return err; + return 0; +} + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err; + + nr_cpus = libbpf_num_possible_cpus(); + if (nr_cpus < 0) { + fprintf(stderr, "failed to get # of possible cpus: '%s'!\n", + strerror(-nr_cpus)); + return nr_cpus; + } + + bump_memlock_rlimit1(); + + sw_mlinks = calloc(nr_cpus, sizeof(*sw_mlinks)); + if (!sw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc sw_mlinks or rlinks\n"); + return err; + } + + hw_mlinks = calloc(nr_cpus, sizeof(*hw_mlinks)); + if (!hw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc hw_mlinks or rlinks\n"); + free(sw_mlinks); + return err; + } + + unity_irqoff = unity_irqoff_bpf__open_and_load(); + if (!unity_irqoff) { + err = errno; + fprintf(stderr, "failed to open and/or load BPF object\n"); + return err; + } + + irqoff_handler(arg, unity_irqoff); + + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "irqoff"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + free(sw_mlinks); + free(hw_mlinks); + printf("unity_irqoff plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h new file mode 100644 index 00000000..1b024f9e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h @@ -0,0 +1,48 @@ +#ifndef __IRQOFF_H +#define __IRQOFF_H + +#define TASK_COMM_LEN 16 +#define CPU_ARRY_LEN 4 +#define CONID_LEN 13 + +struct info { + __u64 prev_counter; +}; + +struct tm_info { + __u64 last_stamp; +}; + +struct arg_info { + __u64 thresh; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif +#endif /* __IRQOFF_H */ + -- Gitee From e89aaa95033f52e8f7a0fd99cb409f8f961d63f2 Mon Sep 17 00:00:00 2001 From: "muya.zj" Date: Wed, 22 Feb 2023 14:57:12 +0800 Subject: [PATCH 22/77] add numainfo and cpufreq depend numactl-devel rpm --- .../tools/monitor/unity/collector/plugin.yaml | 19 ++++- .../monitor/unity/collector/plugin/Makefile | 4 +- .../unity/collector/plugin/cpufreq/Makefile | 19 +++++ .../unity/collector/plugin/cpufreq/cpufreq.c | 73 +++++++++++++++++++ .../unity/collector/plugin/cpufreq/cpufreq.h | 14 ++++ .../unity/collector/plugin/numainfo/Makefile | 19 +++++ .../collector/plugin/numainfo/numainfo.c | 66 +++++++++++++++++ .../collector/plugin/numainfo/numainfo.h | 14 ++++ 8 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/cpufreq/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c create mode 100644 source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.h create mode 100644 source/tools/monitor/unity/collector/plugin/numainfo/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c create mode 100644 source/tools/monitor/unity/collector/plugin/numainfo/numainfo.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 3c5ae4a1..33752eab 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -35,6 +35,7 @@ plugins: - so: proc_loadavg description: "collect load avg" + - so: unity_nosched description: "nosched:sys hold cpu and didn't scheduling" @@ -45,6 +46,12 @@ plugins: - so: unity_irqoff description: "irqoff:detect irq turned off and can't response" + - + so: numainfo + description: "collect numainfo" + - + so: cpufreq + description: "collect cpufreq" metrics: - title: sysak_proc_cpu_total @@ -135,7 +142,7 @@ metrics: - title: sysak_proc_buddyinfo from: buddyinfo head: value - help: "buddyinfo of system from /proc/loadavg" + help: "buddyinfo of system from /proc/buddyinfo" type: "gauge" - title: sysak_IOMonIndForDisksIO from: IOMonIndForDisksIO @@ -177,3 +184,13 @@ metrics: head: value help: "net_retrans_count" type: "gauge" + - title: sysak_numainfo + from: numainfo + head: value + help: "numainfo of system from /sys/devices/system/" + type: "gauge" + - title: sysak_proc_cpufreq + from: cpufreq + head: value + help: "cpufreq of system from /proc/cpuinfo" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index cec50dc7..badac63e 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,9 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink + +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq + all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/cpufreq/Makefile b/source/tools/monitor/unity/collector/plugin/cpufreq/Makefile new file mode 100644 index 00000000..9f44037a --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpufreq/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := cpufreq.o +SO := libcpufreq.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) diff --git a/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c new file mode 100644 index 00000000..b3a14c14 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c @@ -0,0 +1,73 @@ +// +// Created by muya. +// + +#include "cpufreq.h" +#include + +int init(void * arg) { + printf("cpufreq plugin install, proc: %s\n", get_unity_proc()); + return 0; +} + +int call(int t, struct unity_lines* lines) { + struct unity_line* line; + int ret; + FILE *fp = NULL; + char str[128]; + int len = 0; + char result[16] = {0}; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "cpufreq"); + + errno = 0; + + if ((fp = fopen("/proc/cpuinfo", "r")) == NULL) { + ret = errno; + printf("WARN: numainfo install FAIL fopen\n"); + return ret; + } + while (fgets(str, sizeof(str), fp)) { + char *pLast = strstr(str, "@"); + if (NULL != pLast) { + pLast = pLast + 2; + while (*pLast != 'G') + { + len++; + pLast++; + } + memcpy(result, pLast-len, len); + // printf("res is %s\n", result); + unity_set_value(line, 0, "hardware_freq", atof(result)*1000); + memset(result, 0, 16); + len = 0; + } else { + char *pLast = strstr(str, "MHz"); + char *pLast2 = strstr(str, ":"); + if (NULL != pLast && NULL != pLast2) { + pLast2 = pLast2 + 2; + while (*pLast2 != '\n') + { + len++; + pLast2++; + } + memcpy(result, pLast2-len, len); + // printf("res2 is %s, %d\n", result, len); + // printf("res22 is %s\n", str); + unity_set_value(line, 1, "curr_freq", atof(result)); + memset(result, 0, 16); + len = 0; + break; + } + } + } + if (fp) + fclose(fp); + return 0; +} + +void deinit(void) { + printf("cpufreq plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.h b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.h new file mode 100644 index 00000000..669bdeff --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.h @@ -0,0 +1,14 @@ +// +// Created by muya. +// + +#ifndef UNITY_CPUFREQ_H +#define UNITY_CPUFREQ_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + +#endif //UNITY_CPUFREQ_H diff --git a/source/tools/monitor/unity/collector/plugin/numainfo/Makefile b/source/tools/monitor/unity/collector/plugin/numainfo/Makefile new file mode 100644 index 00000000..841dca5f --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/numainfo/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic -lnuma +LDFLAG := -g -fpic -shared -lnuma +OBJS := numainfo.o +SO := libnumainfo.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) diff --git a/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c new file mode 100644 index 00000000..8ec7f836 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c @@ -0,0 +1,66 @@ +// +// Created by muya. +// + +#include "numainfo.h" +#include +#include + +int init(void * arg) { + printf("numainfo plugin install, proc: %s\n", get_unity_proc()); + return 0; +} + +int call(int t, struct unity_lines* lines) { + + // get numa node number + // yum install numactl-devel + int num_nodes = numa_max_node() + 1; + // num_nodes = 2; + // read from /sys/devices/system/node/node0/numastat + // printf("numa %d\n", num_nodes); + struct unity_line* line; + int i, j, ret; + FILE *fp; + char fname[128]; + + unity_alloc_lines(lines, num_nodes); // 预分配好 + + // unity_set_index(line, 0, "mode", "numa_num"); + // unity_set_value(line, 0, "numa_num_sum", num_nodes); + + for (i = 0; i < num_nodes; i++) { + char numa_name[10]; + sprintf(numa_name, "%s%d", "node", i); + // printf("numa is %s\n", numa_name); + line = unity_get_line(lines, i); + unity_set_table(line, "numainfo"); + unity_set_index(line, 0, "node", numa_name); + fp = NULL; + errno = 0; + if (sprintf(fname, "%s%s%d%s", get_unity_proc(), "/sys/devices/system/node/node", i, "/numastat") < 0) + printf("sprintf error\n"); + // printf("fname is %s\n", fname); + if ((fp = fopen(fname, "r")) == NULL) { + ret = errno; + printf("WARN: numainfo install FAIL fopen\n"); + return ret; + } + for (j = 0; j < 6; j++) { + char k[32]; + unsigned long v; + errno = fscanf(fp, "%s %ld\n", k, &v); + if (errno < 0) + return errno; + // printf("k is %s\n", k); + unity_set_value(line, j, k, v); + } + if (fp) + fclose(fp); + } + return 0; +} + +void deinit(void) { + printf("sample plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.h b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.h new file mode 100644 index 00000000..14140fdb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.h @@ -0,0 +1,14 @@ +// +// Created by muya. +// + +#ifndef UNITY_NUMAINFO_H +#define UNITY_NUMAINFO_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + +#endif //UNITY_NUMAINFO_H -- Gitee From b607c403808548a8bca7350635e25b268596b43f Mon Sep 17 00:00:00 2001 From: "muya.zj" Date: Thu, 23 Feb 2023 14:59:12 +0800 Subject: [PATCH 23/77] add gpuinfo Signed-off-by: muya.zj --- .../tools/monitor/unity/collector/plugin.yaml | 10 +++ .../monitor/unity/collector/plugin/Makefile | 2 +- .../unity/collector/plugin/gpuinfo/Makefile | 19 ++++++ .../unity/collector/plugin/gpuinfo/gpuinfo.c | 67 +++++++++++++++++++ .../unity/collector/plugin/gpuinfo/gpuinfo.h | 14 ++++ 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/plugin/gpuinfo/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.c create mode 100644 source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 33752eab..0b3e222a 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -52,6 +52,10 @@ plugins: - so: cpufreq description: "collect cpufreq" + - + so: gpuinfo + description: "collect gpuinfo" + metrics: - title: sysak_proc_cpu_total @@ -194,3 +198,9 @@ metrics: head: value help: "cpufreq of system from /proc/cpuinfo" type: "gauge" + - title: sysak_gpuinfo + from: gpuinfo + head: value + help: "gpuinfo of system from nvidia-smi" + type: "gauge" + diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index badac63e..39feaf94 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -5,7 +5,7 @@ OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq gpuinfo all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/gpuinfo/Makefile b/source/tools/monitor/unity/collector/plugin/gpuinfo/Makefile new file mode 100644 index 00000000..bc149c06 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/gpuinfo/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := gpuinfo.o +SO := libgpuinfo.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) diff --git a/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.c b/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.c new file mode 100644 index 00000000..943f68fe --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.c @@ -0,0 +1,67 @@ +// +// Created by muya. +// + +#include "gpuinfo.h" +#include + + +int init(void * arg) { + printf("gpuinfo plugin install, proc: %s\n", get_unity_proc()); + return 0; +} + +int call(int t, struct unity_lines* lines) { + + + + FILE *fp = NULL; + char buffer[256]; /* Temporary buffer for parsing */ + float mm_total, mm_used, mm_free, temp, powerdraw, gpu_util, mem_util; + struct unity_line* line; + + + // make sure nvidia-smi installed + // if use container, use -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi + if ( access("/usr/bin/nvidia-smi",0) ) { + // printf("nvidia-smi not exists\n"); + return 0; + } + + fp = popen("nvidia-smi --query-gpu=\"memory.total,memory.used,memory.free,temperature.gpu,power.draw,utilization.gpu,utilization.memory\" --format=nounits,csv,noheader", "r"); + memset(buffer, 0, sizeof(buffer)); + + // // for test + // char command[128]; + // if (sprintf(command, "cat %s%s", get_unity_proc(), "/proc/gpuinfo") < 0) + // printf("sprintf error\n"); + // fp = popen(command, "r"); + + + if (fp != NULL) + { + while (fgets(buffer, sizeof(buffer), fp)) + { + sscanf(buffer, "%f, %f, %f, %f, %f, %f, %f", &mm_total, &mm_used, &mm_free, &temp, &powerdraw, &gpu_util, &mem_util); + } + pclose(fp); + } + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "gpuinfo"); + unity_set_index(line, 0, "gpu_num", "gpu0"); + unity_set_value(line, 0, "mm_total", mm_total); + unity_set_value(line, 1, "mm_used", mm_used); + unity_set_value(line, 2, "mm_free", mm_free); + unity_set_value(line, 3, "temp", temp); + unity_set_value(line, 4, "powerdraw", powerdraw); + unity_set_value(line, 5, "gpu_util", gpu_util); + unity_set_value(line, 6, "mem_util", mem_util); + + return 0; +} + +void deinit(void) { + printf("gpuinfo plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.h b/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.h new file mode 100644 index 00000000..b2d57462 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/gpuinfo/gpuinfo.h @@ -0,0 +1,14 @@ +// +// Created by muya. +// + +#ifndef UNITY_GPUINFO_H +#define UNITY_GPUINFO_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + +#endif //UNITY_GPUINFO_H -- Gitee From f0f3a70207b154332c6b197b203eb85d0c43b9fe Mon Sep 17 00:00:00 2001 From: "muya.zj" Date: Mon, 27 Feb 2023 14:09:08 +0800 Subject: [PATCH 24/77] cpuinfo: fix atof and do not use default --- source/tools/monitor/unity/collector/plugin.yaml | 16 ++++++++-------- .../unity/collector/plugin/cpufreq/cpufreq.c | 2 +- .../unity/collector/plugin/numainfo/numainfo.c | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 0b3e222a..40143837 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -49,9 +49,9 @@ plugins: - so: numainfo description: "collect numainfo" - - - so: cpufreq - description: "collect cpufreq" + # - + # so: cpufreq + # description: "collect cpufreq" - so: gpuinfo description: "collect gpuinfo" @@ -193,11 +193,11 @@ metrics: head: value help: "numainfo of system from /sys/devices/system/" type: "gauge" - - title: sysak_proc_cpufreq - from: cpufreq - head: value - help: "cpufreq of system from /proc/cpuinfo" - type: "gauge" + # - title: sysak_proc_cpufreq + # from: cpufreq + # head: value + # help: "cpufreq of system from /proc/cpuinfo" + # type: "gauge" - title: sysak_gpuinfo from: gpuinfo head: value diff --git a/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c index b3a14c14..fe224ec4 100644 --- a/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c +++ b/source/tools/monitor/unity/collector/plugin/cpufreq/cpufreq.c @@ -4,6 +4,7 @@ #include "cpufreq.h" #include +#include int init(void * arg) { printf("cpufreq plugin install, proc: %s\n", get_unity_proc()); @@ -55,7 +56,6 @@ int call(int t, struct unity_lines* lines) { } memcpy(result, pLast2-len, len); // printf("res2 is %s, %d\n", result, len); - // printf("res22 is %s\n", str); unity_set_value(line, 1, "curr_freq", atof(result)); memset(result, 0, 16); len = 0; diff --git a/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c index 8ec7f836..67017fda 100644 --- a/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c +++ b/source/tools/monitor/unity/collector/plugin/numainfo/numainfo.c @@ -31,7 +31,7 @@ int call(int t, struct unity_lines* lines) { for (i = 0; i < num_nodes; i++) { char numa_name[10]; - sprintf(numa_name, "%s%d", "node", i); + snprintf(numa_name, 10, "%s%d", "node", i); // printf("numa is %s\n", numa_name); line = unity_get_line(lines, i); unity_set_table(line, "numainfo"); -- Gitee From fdd34c9ed776eff03d9f09c9e2d8693470c0258d Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Mon, 27 Feb 2023 14:41:20 +0800 Subject: [PATCH 25/77] coolbpf: update Signed-off-by: Shuyi Cheng --- source/lib/internal/ebpf/coolbpf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lib/internal/ebpf/coolbpf b/source/lib/internal/ebpf/coolbpf index 735ae613..c9a6c68b 160000 --- a/source/lib/internal/ebpf/coolbpf +++ b/source/lib/internal/ebpf/coolbpf @@ -1 +1 @@ -Subproject commit 735ae6138430822502e1bd5ae2a366a8668ecd7e +Subproject commit c9a6c68b9dd375618cdfd5b5885752e88c2faf60 -- Gitee From 3bf8d9ed211748e5b897630d7e599c3d0a7cc662 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 27 Feb 2023 15:38:45 +0800 Subject: [PATCH 26/77] fix query bugs. --- source/tools/monitor/unity/collector/loop.lua | 7 +- .../tools/monitor/unity/collector/plugin.lua | 1 - .../collector/plugin/net_health/net_health.c | 10 +-- .../plugin/net_retrans/net_retrans.c | 2 - .../tools/monitor/unity/collector/vproc.lua | 7 +- .../tools/monitor/unity/common/pystring.lua | 67 +++++++++++++++++++ source/tools/monitor/unity/test/bpf/hello.py | 53 +++++++++++++++ .../monitor/unity/test/bpf/net_health.py | 47 +++++++++++++ source/tools/monitor/unity/test/string/py.lua | 16 +++++ 9 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 source/tools/monitor/unity/test/bpf/hello.py create mode 100644 source/tools/monitor/unity/test/bpf/net_health.py diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 7e970dc8..28b23464 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -34,11 +34,10 @@ end function Cloop:work(t) local lines = self._proto:protoTable() - for k, obj in pairs(self._procs) do - lines = obj:proc(t, lines) + for _, obj in pairs(self._procs) do + obj:proc(t, lines) end - lines = self._plugin:proc(t, lines) - --print(#lines.lines) + self._plugin:proc(t, lines) local bytes = self._proto:encode(lines) self._proto:que(bytes) end diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 4ccf946b..2c0c5899 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -120,7 +120,6 @@ function Cplugin:proc(t, lines) end self._ffi.C.free(unity_lines.line) -- should free memory. end - return lines end return Cplugin diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c index ea63d234..231f80bd 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -26,16 +26,10 @@ int init(void *arg) static int get_dist(unsigned long *locals) { int i = 0; unsigned long value = 0; - int key, key_next; - key = 0; - while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { - coobpf_key_value(dist_fd, &key_next, &value); + for (i = 0; i < DIST_ARRAY_SIZE; i ++) { + coobpf_key_value(dist_fd, &i, &value); locals[i ++] = value; - if (i > DIST_ARRAY_SIZE) { - break; - } - key = key_next; } return i; } diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c index 2f5a1969..0cc8ca67 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -32,8 +32,6 @@ void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) struct unity_line *line; struct unity_lines *lines = unity_new_lines(); - printf("receive: %d msg.\n", data_sz); - unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); ret = proc(stack_fd, e, line); diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 30d8fa3d..05024bc8 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -31,13 +31,12 @@ function CvProc:copyLine(line) end function CvProc:push(lines) - local c = #lines - for _, v in ipairs(self._lines["lines"]) do + local c = #lines["lines"] -- not for #lines + for _, line in ipairs(self._lines["lines"]) do c = c + 1 - lines["lines"][c] = v + lines["lines"][c] = line end self._lines = nil - return lines end function CvProc:_packProto(head, labels, vs, log) diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index c38a0d1f..4c0e6f7c 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -73,6 +73,73 @@ local function setupPatten(s) return patten end +function pystring:shift(s, n) -- position for right, negative for left + local len = string.len(s) + if len == 0 then + return s + end + n = n % len + if n == 0 then + return s + elseif n > 0 then -- "abcd >> 1" + local offset = len - n + local s1 = string.sub(s, offset + 1) + local s2 = string.sub(s, 1, offset) + return s1 .. s2 + else -- "abcd << 1" + local offset = len + n + local s1 = string.sub(s, offset + 1) + local s2 = string.sub(s, 1, offset) + return s2 .. s1 + end +end + +function pystring:lower(s) + return string.lower(s) +end + +function pystring:upper(s) + return string.upper(s) +end + +function pystring:swapcase(s) + local swaps = {} + for i=1, #s do + local ch = string.byte(s, i) + if ch >= 65 and ch <= 90 then + swaps[i] = string.char(ch + 32) + elseif ch >= 97 and ch <= 122 then + swaps[i] = string.char(ch - 32) + else + swaps[i] = string.char(ch) + end + end + return table.concat(swaps) +end + +function pystring:capitalize(s) + if #s < 1 then + return s + end + local s1 = string.sub(s, 1, 1) + local s2 = string.sub(s, 2) + return string.upper(s1) .. s2 +end + +function pystring:capwords(s) + local lines = pystring:split(s, "\n") + local rLines = {} + for i, line in ipairs(lines) do + local rWords = {} + local words = pystring:split(line, " ") + for j, word in ipairs(words) do + rWords[j] = pystring:capitalize(word) + end + rLines[i] = table.concat(rWords, " ") + end + return table.concat(rLines, "\n") +end + function pystring:split(s, delimiter, n) local result = {} if not delimiter or delimiter == "" then -- for blank, gsub multi blank to single diff --git a/source/tools/monitor/unity/test/bpf/hello.py b/source/tools/monitor/unity/test/bpf/hello.py new file mode 100644 index 00000000..3e4cead6 --- /dev/null +++ b/source/tools/monitor/unity/test/bpf/hello.py @@ -0,0 +1,53 @@ +import ctypes as ct +from pylcc.lbcBase import ClbcBase + +bpfPog = r""" +#include "lbc.h" +#define TASK_COMM_LEN 16 +struct data_t { + u32 c_pid; + u32 p_pid; + char c_comm[TASK_COMM_LEN]; + char p_comm[TASK_COMM_LEN]; +}; + +LBC_PERF_OUTPUT(e_out, struct data_t, 128); +SEC("kprobe/_do_fork") +int j_wake_up_new_task(struct pt_regs *ctx) +{ + struct task_struct* parent = (struct task_struct *)PT_REGS_PARM1(ctx); + struct data_t data = {}; + + data.c_pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&data.c_comm, TASK_COMM_LEN); + data.p_pid = BPF_CORE_READ(parent, pid); + bpf_core_read(&data.p_comm[0], TASK_COMM_LEN, &parent->comm[0]); + + bpf_perf_event_output(ctx, &e_out, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} + +char _license[] SEC("license") = "GPL"; +""" + +class CeventOut(ClbcBase): + def __init__(self): + super(CeventOut, self).__init__("eventOut", bpf_str=bpfPog) + + def _cb(self, cpu, data, size): + e = self.getMap('e_out', data, size) + print("current pid:%d, comm:%s. wake_up_new_task pid: %d, comm: %s" % ( + e.c_pid, e.c_comm, e.p_pid, e.p_comm + )) + + def loop(self): + self.maps['e_out'].open_perf_buffer(self._cb) + try: + self.maps['e_out'].perf_buffer_poll() + except KeyboardInterrupt: + print("key interrupt.") + exit() + +if __name__ == "__main__": + e = CeventOut() + e.loop() \ No newline at end of file diff --git a/source/tools/monitor/unity/test/bpf/net_health.py b/source/tools/monitor/unity/test/bpf/net_health.py new file mode 100644 index 00000000..ad10f46f --- /dev/null +++ b/source/tools/monitor/unity/test/bpf/net_health.py @@ -0,0 +1,47 @@ +from pylcc import ClbcBase +import time + +bpfPog = r""" +#include "lbc.h" + +LBC_ARRAY(outCnt, int, u64, 2); +LBC_HIST10(netHist); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +SEC("kprobe/tcp_validate_incoming") +int j_tcp_validate_incoming(struct pt_regs *ctx) { + struct tcp_sock *tp = (struct tcp_sock *)PT_REGS_PARM1(ctx); + u64 ts = BPF_CORE_READ(tp, srtt_us) >> 3; + u64 ms = ts / 1000; + if (ms > 0) { + addCnt(0, ms); + addCnt(1, 1); + hist10_push(&netHist, ms); + } + return 0; +} + +char _license[] SEC("license") = "GPL"; +""" + + +class CnetHealth(ClbcBase): + def __init__(self): + super(CnetHealth, self).__init__("net_health_bpf", bpf_str=bpfPog) + + def loop(self): + while True: + time.sleep(20) + print(self.maps['outCnt'].get()) + print(self.maps['netHist'].get()) + + +if __name__ == "__main__": + e = CnetHealth() + e.loop() diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index 8befc8ea..a4d17e12 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -72,3 +72,19 @@ assert(pystring:endswith("hello world", "world")) -- find assert(pystring:find("hello world.", "hello") == 1) + +-- shift +assert(pystring:shift("abcd", 1) == "dabc") +assert(pystring:shift("abcd", -1) == "bcda") + +-- swapcase +assert(pystring:swapcase("Hello, World!") == "hELLO, wORLD!") + +-- capitalize +assert(pystring:capitalize("hello") == "Hello") +assert(pystring:capitalize("") == "") +assert(pystring:capitalize("H") == "H") + +-- capwords +assert(pystring:capwords("hello world.") == "Hello World.") +assert(pystring:capwords("hello world.\nhere you are.") == "Hello World.\nHere You Are.") \ No newline at end of file -- Gitee From b548f20faf2a4ebb8435f979613971c7d50c79e9 Mon Sep 17 00:00:00 2001 From: zhilan Date: Tue, 21 Feb 2023 17:09:36 +0800 Subject: [PATCH 27/77] unity: Add pod alloc page monitor --- .../unity/collector/native/plugincffi.lua | 1 + .../tools/monitor/unity/collector/plugin.yaml | 9 +- .../monitor/unity/collector/pod_allocpage.lua | 189 ++++++++++++++++++ .../tools/monitor/unity/common/dockerinfo.lua | 104 ++++++++++ 4 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/pod_allocpage.lua create mode 100644 source/tools/monitor/unity/common/dockerinfo.lua diff --git a/source/tools/monitor/unity/collector/native/plugincffi.lua b/source/tools/monitor/unity/collector/native/plugincffi.lua index dccb08a4..037ba320 100644 --- a/source/tools/monitor/unity/collector/native/plugincffi.lua +++ b/source/tools/monitor/unity/collector/native/plugincffi.lua @@ -38,6 +38,7 @@ int call(int t, struct unity_lines* lines); void deinit(void); void free(void *p); +int setns(int fd, int nstype); ]] return ffi diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 40143837..bce87887 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -7,6 +7,8 @@ config: mode: specify name: test_specify # mode: hostip +# real_timestamps: true +# unix_socket: "/tmp/sysom_unity.sock" proc_path: /mnt/host/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ # proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ @@ -14,7 +16,7 @@ outline: - /tmp/sysom luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", - "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat","pod_allocpage"] plugins: - so: kmsg @@ -204,3 +206,8 @@ metrics: help: "gpuinfo of system from nvidia-smi" type: "gauge" + #- title: sysak_pod_alloc + #from: pod_alloc + #head: value + #help: "get pod alloc page used" + #type: "gauge" diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua new file mode 100644 index 00000000..7d0b2942 --- /dev/null +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -0,0 +1,189 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liuxinwnei. +--- DateTime: 2023/02/08 17:00 PM +--- + +require("common.class") +fcntl = require("posix.fcntl") +unistd = require("posix.unistd") +dirent = require("posix.dirent") +stdlib = require("posix.stdlib") +cjson = require("cjson") +json = cjson.new() +local CkvProc = require("collector.kvProc") +local CvProc = require("collector.vproc") +local pystring = require("common.pystring") +local dockerinfo = require("common.dockerinfo") + +local CPodAlloc = class("podalloc", CkvProc) + +function CPodAlloc:_init_(proto, pffi, mnt, pFile) + CkvProc._init_(self, proto, pffi, mnt, pFile , "pod_alloc") + self._ffi = require("collector.native.plugincffi") + self.proc_fs, self.sys_fs, self.pods_fs, self.root_fs = dockerinfo:get_hostfs() + self.name_space = {} + self.pod_mem = {} + self.total = 0 +end + +function CPodAlloc:file_exists(file) + local f=io.open(file,"r") + if f ~= nil then + io.close(f) + return true + else + return false + end +end + +function CPodAlloc:switch_ns(pid) + local pid_ns = self.proc_fs .. pid .. "/ns/net" + if not self:file_exists(pid_ns) then return end + + local f = fcntl.open(pid_ns,fcntl.O_RDONLY) + self._ffi.C.setns(f,0) + unistd.close(f) +end + +function CPodAlloc:get_container_info(did) + local res = "unknow" + local podname = did + local podns = did + local cname = did + + res = dockerinfo:get_inspect(did) + restable = json.decode(res) + if #restable > 0 then + restable = restable[1] + end + if restable['Config'] then + local config = restable['Config'] + if config['Labels'] then + local label = config['Labels'] + if label['io.kubernetes.pod.name'] then + podname = label['io.kubernetes.pod.name'] + end + if label['io.kubernetes.container.name'] then + cname = label['io.kubernetes.container.name'] + end + if label['io.kubernetes.pod.namespace'] then + podns = label['io.kubernetes.pod.namespace'] + end + end + if podname == did and restable['Name'] then + cname = restable['Name'] + podname = restable['Name'] + end + elseif restable['status'] then + podname = restable['status']['labels']['io.kubernetes.pod.name'] + cname = restable['status']['labels']['io.kubernetes.container.name'] + podns = restable['status']['labels']['io.kubernetes.pod.namespace'] + end + if pystring:startswith(podname,"/") then podname=string.sub(podname,2,-1) end + if not self.pod_mem[podname] then + self.pod_mem[podname] = {} + self.pod_mem[podname]["allocpage"] = 0 + self.pod_mem[podname]["podns"] = podns + self.pod_mem[podname]["podname"] = podname + end + return podname +end + +function CPodAlloc:get_pidalloc() + local pods = {} + local dockerids = {} + for net,pidn in pairs(self.name_space) do + if pidn == "self" then pidn = "1" end + + self:switch_ns(pidn) + -- local env = posix.getenv() + -- env["PROC_ROOT"] = self.proc_fs + + stdlib.setenv("PROC_ROOT",self.proc_fs) + local pfile = io.popen("ss -anp","r") + io.input(pfile) + for line in io.lines() do + repeat + local proto,recv,task,pid = string.match(line,"(%S*)%s*%S*%s*(%d*).*users:%S*\"(%S*)\",pid=(%d*)") + if not proto or not recv or not task or not pid then break end + if proto ~="tcp" and proto ~="udp" and proto ~="raw" then break end + + recv = tonumber(recv) + + local dockerid = "" + if not dockerids[pid] then + dockerid = dockerinfo:get_dockerid(pid) + if dockerid == "unknow" then break end + dockerids[pid] = dockerid + else + dockerid = dockerids[pid] + end + + local podname = dockerid + if not pods[dockerid] then + podname = self:get_container_info(dockerid) + pods[dockerid] = podname + else + podname = pods[dockerid] + end + if recv < 1024 and podname == dockerid then break end + + if not self.pod_mem[podname] then + self.pod_mem[podname] = {} + self.pod_mem[podname]["allocpage"] = 0 + self.pod_mem[podname]["podns"] = podname + self.pod_mem[podname]["podname"] = podname + end + self.pod_mem[podname]["allocpage"] = self.pod_mem[podname]["allocpage"] + recv + self.total = self.total + recv + until true + end + pfile:close() + self:switch_ns("1") + stdlib.setenv("PROC_ROOT","") + end +end + +function CPodAlloc:scan_namespace() + root = self.proc_fs + for pid in dirent.files(root) do + repeat + if pystring:startswith(pid,".") then break end + if not self:file_exists(self.proc_fs .. pid .. "/comm") then break end + + local proc_ns = self.proc_fs .. pid .. "/ns/net" + if not self:file_exists(proc_ns) then break end + + local slink = unistd.readlink(proc_ns) + if not string.find(slink,"net") then break end + + local inode = string.match(slink,"%[(%S+)%]") + if not inode then break end + + if not self.name_space[inode] then self.name_space[inode] = pystring:strip(pid) end + if not self:file_exists(root .. self.name_space[inode] .. "/comm") then self.name_space[inode] = pystring:strip(pid) end + until true + end +end + +function CPodAlloc:proc(elapsed, lines) + CvProc.proc(self) + self.name_space = {} + self.pod_mem = {} + self.total = 0 + self:scan_namespace() + self:get_pidalloc() + + for k,v in pairs(self.pod_mem) do + local cell = {{name="pod_allocpage", value=v['allocpage']/1024}} + local label = {{name="podname",index=v['podname'],}, {name="namespace",index = v['podns'],},} + self:appendLine(self:_packProto("pod_alloc", label, cell)) + end + local cell = {{name="pod_allocpage_total", value=self.total/1024}} + self:appendLine(self:_packProto("pod_alloc", nil, cell)) + + return self:push(lines) +end + +return CPodAlloc diff --git a/source/tools/monitor/unity/common/dockerinfo.lua b/source/tools/monitor/unity/common/dockerinfo.lua new file mode 100644 index 00000000..798ddbea --- /dev/null +++ b/source/tools/monitor/unity/common/dockerinfo.lua @@ -0,0 +1,104 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liuxinwnei. +--- DateTime: 2023/02/08 17:00 PM +--- + +dockerinfo = {} +local posix = require("posix") +local cjson = require("cjson") +local json = cjson.new() +local pystring = require("common.pystring") + +function file_exists(file) + local f=io.open(file,"r") + if f ~= nil then + io.close(f) + return true + else + return false + end +end + +function dockerinfo:get_hostfs() + local proc_fs="/mnt/host/proc/" + local sys_fs="/mnt/host/sys/" + local pods_fs="/mnt/host/var/lib/kubelet/pods/" + local root_fs = "/mnt/host/" + if file_exists(proc_fs) then + return proc_fs, sys_fs, pods_fs, root_fs + end + proc_fs="/proc/" + sys_fs="/sys/" + pods_fs="/var/lib/kubelet/pods/" + root_fs = "/" + return proc_fs, sys_fs, pods_fs, root_fs +end + +function get_runtimesock() + local root_fs = "" + _, _, _, root_fs = dockerinfo:get_hostfs() + local runtime = "docker" + local runtime_sock = root_fs .. "var/run/docker.sock" + local sock={"var/run/docker.sock","run/containerd/containerd.sock", "var/run/dockershim.sock"} + for _,runtimex in pairs(sock) do + if file_exists(root_fs .. runtimex) then + runtime_sock = root_fs .. runtimex + if not string.find(runtime_sock,"docker.sock") then + runtime = "crictl" + end + end + end + return runtime,runtime_sock +end + +function dockerinfo:get_inspect(did) + local runtime,runtime_sock = get_runtimesock() + if runtime == "docker" then + return get_container_inspect(did) + else + return get_crictl_inspect(did) + end +end + +function dockerinfo:get_dockerid(pid) + local proc_fs = dockerinfo:get_hostfs() + local idstring = "unknow" + if not file_exists(proc_fs .. pid .. "/cgroup") then return idstring end + local cmd = "cat " .. proc_fs .. pid .. "/cgroup 2>/dev/null | grep memory:" + pfile = io.popen(cmd,"r") + local res = pfile:read("*a") + pfile:close() + + if not string.find(res,"kubepods") and not string.find(res,"docker-") then return idstring end + if string.find(res,"docker-") then + idstring = pystring:split(res,"docker-")[2] + elseif string.find(res,"cri-containerd-") then + idstring = pystring:split(res,"cri-containerd-")[2] + else + local tmp = pystring:split(res,"/",10) + idstring = tmp[#tmp] + end + idstring = string.sub(idstring,0,8) + return idstring +end + +function get_container_inspect(did) + local runtime, runtime_sock = get_runtimesock() + local cmd = "curl --silent -XGET --unix-socket " .. runtime_sock .. " http://localhost/containers/" .. did .. "/json 2>/dev/null " + local f = io.popen(cmd,"r") + local res = f:read("*a") + f:close() + return res +end + +function get_crictl_inspect(did) + local runtime, runtime_sock = get_runtimesock() + local cmd = runtime .. " -r " .. runtime_sock .. " inspect " .. did .. " 2>/dev/null " + local f = io.popen(cmd,"r") + local res = f:read("*a") + f:close() + return res +end + +return dockerinfo -- Gitee From 04a686ce24798ed546a02ec6a0217a413cedb90a Mon Sep 17 00:00:00 2001 From: zhilan Date: Tue, 21 Feb 2023 17:27:46 +0800 Subject: [PATCH 28/77] unity: fix no local bugs --- .../monitor/unity/collector/pod_allocpage.lua | 16 ++++++++-------- source/tools/monitor/unity/common/dockerinfo.lua | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua index 7d0b2942..158cba05 100644 --- a/source/tools/monitor/unity/collector/pod_allocpage.lua +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -5,12 +5,12 @@ --- require("common.class") -fcntl = require("posix.fcntl") -unistd = require("posix.unistd") -dirent = require("posix.dirent") -stdlib = require("posix.stdlib") -cjson = require("cjson") -json = cjson.new() +local fcntl = require("posix.fcntl") +local unistd = require("posix.unistd") +local dirent = require("posix.dirent") +local stdlib = require("posix.stdlib") +local cjson = require("cjson") +local json = cjson.new() local CkvProc = require("collector.kvProc") local CvProc = require("collector.vproc") local pystring = require("common.pystring") @@ -53,7 +53,7 @@ function CPodAlloc:get_container_info(did) local cname = did res = dockerinfo:get_inspect(did) - restable = json.decode(res) + local restable = json.decode(res) if #restable > 0 then restable = restable[1] end @@ -146,7 +146,7 @@ function CPodAlloc:get_pidalloc() end function CPodAlloc:scan_namespace() - root = self.proc_fs + local root = self.proc_fs for pid in dirent.files(root) do repeat if pystring:startswith(pid,".") then break end diff --git a/source/tools/monitor/unity/common/dockerinfo.lua b/source/tools/monitor/unity/common/dockerinfo.lua index 798ddbea..829e0cbc 100644 --- a/source/tools/monitor/unity/common/dockerinfo.lua +++ b/source/tools/monitor/unity/common/dockerinfo.lua @@ -66,7 +66,7 @@ function dockerinfo:get_dockerid(pid) local idstring = "unknow" if not file_exists(proc_fs .. pid .. "/cgroup") then return idstring end local cmd = "cat " .. proc_fs .. pid .. "/cgroup 2>/dev/null | grep memory:" - pfile = io.popen(cmd,"r") + local pfile = io.popen(cmd,"r") local res = pfile:read("*a") pfile:close() -- Gitee From 2b7a79166be9cb337da8abd5c09803ae2215084a Mon Sep 17 00:00:00 2001 From: zhilan Date: Wed, 22 Feb 2023 17:09:54 +0800 Subject: [PATCH 29/77] unity: fix bugs in pod_allocpage --- source/tools/monitor/unity/collector/pod_allocpage.lua | 4 ++-- source/tools/monitor/unity/common/dockerinfo.lua | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua index 158cba05..a0226baf 100644 --- a/source/tools/monitor/unity/collector/pod_allocpage.lua +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -9,6 +9,7 @@ local fcntl = require("posix.fcntl") local unistd = require("posix.unistd") local dirent = require("posix.dirent") local stdlib = require("posix.stdlib") +local stat = require("posix.sys.stat") local cjson = require("cjson") local json = cjson.new() local CkvProc = require("collector.kvProc") @@ -28,9 +29,8 @@ function CPodAlloc:_init_(proto, pffi, mnt, pFile) end function CPodAlloc:file_exists(file) - local f=io.open(file,"r") + local f=stat.lstat(file) if f ~= nil then - io.close(f) return true else return false diff --git a/source/tools/monitor/unity/common/dockerinfo.lua b/source/tools/monitor/unity/common/dockerinfo.lua index 829e0cbc..9bbce2ce 100644 --- a/source/tools/monitor/unity/common/dockerinfo.lua +++ b/source/tools/monitor/unity/common/dockerinfo.lua @@ -9,11 +9,11 @@ local posix = require("posix") local cjson = require("cjson") local json = cjson.new() local pystring = require("common.pystring") +local stat = require("posix.sys.stat") function file_exists(file) - local f=io.open(file,"r") + local f=stat.lstat(file) if f ~= nil then - io.close(f) return true else return false @@ -70,10 +70,10 @@ function dockerinfo:get_dockerid(pid) local res = pfile:read("*a") pfile:close() - if not string.find(res,"kubepods") and not string.find(res,"docker-") then return idstring end - if string.find(res,"docker-") then + if not string.find(res,"kubepods") and not string.find(res,"docker%-") then return idstring end + if string.find(res,"docker%-") then idstring = pystring:split(res,"docker-")[2] - elseif string.find(res,"cri-containerd-") then + elseif string.find(res,"cri%-containerd%-") then idstring = pystring:split(res,"cri-containerd-")[2] else local tmp = pystring:split(res,"/",10) -- Gitee From 60d9b4eecd4596160d0e19399ba11aa82aa2f1bf Mon Sep 17 00:00:00 2001 From: zhilan Date: Wed, 22 Feb 2023 17:18:02 +0800 Subject: [PATCH 30/77] unity: pod_alloc: verify slink --- source/tools/monitor/unity/collector/pod_allocpage.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua index a0226baf..1d5f00ef 100644 --- a/source/tools/monitor/unity/collector/pod_allocpage.lua +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -156,6 +156,7 @@ function CPodAlloc:scan_namespace() if not self:file_exists(proc_ns) then break end local slink = unistd.readlink(proc_ns) + if not slink then break end if not string.find(slink,"net") then break end local inode = string.match(slink,"%[(%S+)%]") -- Gitee From 278b96f90d2179d0678ec060619a1c540408960e Mon Sep 17 00:00:00 2001 From: zhilan Date: Thu, 23 Feb 2023 10:55:37 +0800 Subject: [PATCH 31/77] unity: use curl to get hostname --- source/tools/monitor/unity/collector/plugin.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index bce87887..260f81c3 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -4,8 +4,9 @@ config: bind_addr: 0.0.0.0 # bind ip backlog: 32 # listen backlog identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg) - mode: specify - name: test_specify + mode: curl + url: "http://100.100.100.200/latest/meta-data/instance-id" +# name: test_specify # mode: hostip # real_timestamps: true # unix_socket: "/tmp/sysom_unity.sock" -- Gitee From 9c7d218d17feb8c798e9d5a0a2d3bf433259c06b Mon Sep 17 00:00:00 2001 From: zhilan Date: Thu, 23 Feb 2023 16:16:21 +0800 Subject: [PATCH 32/77] unity: Add config to print out real timestamps(ms) --- source/tools/monitor/unity/beaver/export.lua | 39 +++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index b6f3b52e..3b72b9c9 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -11,15 +11,6 @@ require("common.class") local Cexport = class("Cexport") -function Cexport:_init_(instance, fYaml) - self._instance = instance - local ms = system:parseYaml(fYaml) - self._freq = ms.config.freq - self._tDescr = ms.metrics - self._fox = CfoxTSDB.new() - self._fox:_setupRead() -end - local function qFormData(from, tData) local res = {} local len = #tData @@ -40,6 +31,19 @@ local function qFormData(from, tData) return res end +local function packLine_us(title, ls, v, time) + local tLs = {} + for k, v in pairs(ls) do + table.insert(tLs, string.format("%s=\"%s\"", k , v)) + end + local label = "" + if #tLs then + label = pystring:join(",", tLs) + label = "{" .. label .. "}" + end + return string.format("%s%s %.1f %d", title, label, v, time/1000) +end + local function packLine(title, ls, v) local tLs = {} local c = 0 @@ -55,6 +59,21 @@ local function packLine(title, ls, v) return string.format("%s%s %.1f", title, label, v) end +function Cexport:_init_(instance, fYaml) + self._instance = instance + local ms = system:parseYaml(fYaml) + self._freq = ms.config.freq + self._timestamps = ms.config.real_timestamps + if self._timestamps == true then + self.pack_line = packLine_us + else + self.pack_line = packLine + end + self._tDescr = ms.metrics + self._fox = CfoxTSDB.new() + self._fox:_setupRead() +end + function Cexport:export() local qs = {} self._fox:resize() @@ -82,7 +101,7 @@ function Cexport:export() for k, v in pairs(tFrom.values) do labels[line.head] = k c = c + 1 - res[c] = packLine(title, labels, v) + res[c] = self.pack_line(title, labels, v, tFrom.time) end end end -- Gitee From c0cc438643d5b347491f58b66c970569addfdd41 Mon Sep 17 00:00:00 2001 From: zhilan Date: Thu, 23 Feb 2023 16:16:34 +0800 Subject: [PATCH 33/77] unity: fix bugs in beaver/identity.lua to get instance_id from curl --- source/tools/monitor/unity/beaver/identity.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 1c4e93e3..91b96c89 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -41,11 +41,11 @@ function Cidentity:hostip() end function Cidentity:curl() - if self._opts.curl then + if self._opts.url then local ChttpCli = require("httplib.httpCli") local cli = ChttpCli.new() - local res = cli:get(self._opts.curl) + local res = cli:get(self._opts.url) return res.body else return "None" @@ -80,4 +80,4 @@ function Cidentity:id() return self._funcs[self._opts.mode]() end -return Cidentity \ No newline at end of file +return Cidentity -- Gitee From 59e3f187f2c6e903e6aee00272a4b41faa3e9900 Mon Sep 17 00:00:00 2001 From: zhilan Date: Thu, 23 Feb 2023 16:17:40 +0800 Subject: [PATCH 34/77] unity: Add unix socket method --- .../monitor/unity/beaver/localBeaver.lua | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index cb062006..74485a40 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -17,12 +17,17 @@ local function setupServer(fYaml) local port = config["port"] or 8400 local ip = config["bind_addr"] or "0.0.0.0" local backlog = config["backlog"] or 32 - return port, ip, backlog + local unix_socket = config["unix_socket"] + return port, ip, backlog,unix_socket end function CLocalBeaver:_init_(frame, fYaml) - local port, ip, backlog = setupServer(fYaml) - self._bfd = self:_install_fd(port, ip, backlog) + local port, ip, backlog, unix_socket = setupServer(fYaml) + if not unix_socket then + self._bfd = self:_install_fd(port, ip, backlog) + else + self._bfd = self:_install_fd_unisock(backlog, unix_socket) + end self._efd = self:_installFFI() self._cos = {} @@ -107,6 +112,31 @@ local function localBind(fd, tPort) system:posixError(string.format("bind port %d failed.", tPort.port), err, errno) end +function CLocalBeaver:_install_fd_unisock(backlog,unix_socket) + local fd, res, err, errno + unistd.unlink(unix_socket) + fd, err, errno = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) + if fd then -- for socket + local tPort = {family=socket.AF_UNIX, path=unix_socket} + local r, msg = pcall(localBind, fd, tPort) + if r then + res, err, errno = socket.listen(fd, backlog) + if res then -- for listen + return fd + else + unistd.close(fd) + system:posixError("socket listen failed", err, errno) + end + else + print(msg) + unistd.close(fd) + os.exit(1) + end + else -- socket failed + system:posixError("create socket failed", err, errno) + end +end + function CLocalBeaver:_install_fd(port, ip, backlog) local fd, res, err, errno fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) -- Gitee From b3e9aa04c80b911ce992c82ac9153c0c9fb41efb Mon Sep 17 00:00:00 2001 From: zhilan Date: Mon, 27 Feb 2023 16:01:18 +0800 Subject: [PATCH 35/77] unity: Update develop.md --- source/tools/monitor/unity/beaver/guide/develop.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 11945179..230b480f 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -54,6 +54,8 @@ config: # specify: 指定id，需要指定name参数 mode: specify name: test_specify + real_timestamps: true #上报监测数据的真实时间，默认关闭 + unix_socket: "/tmp/sysom_unity.sock" #通过unix_socket方式进行数据传输，默认关闭 proc_path: /mnt/host/ # proc 文件路径，在host侧，为 / 在容器侧，如配置 -v /:/mnt/host 则配置为 /mnt/host outline: # 外部数据入口，适合接入外部数据场景 -- Gitee From 056e49414e04d6df01524fb1590782b628467581 Mon Sep 17 00:00:00 2001 From: zhilan Date: Mon, 27 Feb 2023 16:06:53 +0800 Subject: [PATCH 36/77] unity: shut off pod_allocpage --- source/tools/monitor/unity/collector/plugin.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 260f81c3..c1187760 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -17,7 +17,7 @@ outline: - /tmp/sysom luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", - "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat","pod_allocpage"] + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] plugins: - so: kmsg -- Gitee From 3e0e1c5de0e3845e3321b83ea036e9491909b204 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Mon, 27 Feb 2023 17:03:40 +0800 Subject: [PATCH 37/77] unity: bpfsample: adopt new coolbpf framework Signed-off-by: Shuyi Cheng --- .../collector/plugin/bpfsample/bpfsample.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c index 18eff585..5bd980b1 100644 --- a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c @@ -7,17 +7,29 @@ #include #include "../../../../unity/beeQ/beeQ.h" -DEFINE_SEKL_OBJECT(bpfsample); +struct coolbpf_object *cb = NULL; +int countfd = 0; int init(void *arg) { + cb = coolbpf_object_new(bpfsample); + if (!cb) { + printf("Failed to create coolbpf object\n"); + return -EINVAL; + } + + countfd = coolbpf_object_find_map(cb, "count"); + if (countfd < 0) { + printf("Failed to get count map fd\n"); + return countfd; + } printf("bpfsample plugin install.\n"); - return LOAD_SKEL_OBJECT(bpfsample); + printf("count map fd is %d\n", countfd); + return 0; } int call(int t, struct unity_lines *lines) { - int countfd = bpf_map__fd(bpfsample->maps.count); int default_key = 0; uint64_t count = 0; uint64_t default_count = 0; @@ -37,5 +49,5 @@ int call(int t, struct unity_lines *lines) void deinit(void) { printf("bpfsample plugin uninstall.\n"); - DESTORY_SKEL_BOJECT(bpfsample); + coolbpf_object_destroy(cb); } -- Gitee From b82434901ca985f0c930692bdf3ae31140cfc96f Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Mon, 27 Feb 2023 17:31:44 +0800 Subject: [PATCH 38/77] unity: bpfsample: remove unused macro Signed-off-by: Shuyi Cheng --- .../collector/plugin/bpfsample/bpfsample.h | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h index 9bd981fc..15dcaa72 100644 --- a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h @@ -7,42 +7,6 @@ #include "../plugin_head.h" -#define DEFINE_SEKL_OBJECT(skel_name) \ - struct skel_name##_bpf *skel_name = NULL; - -#define LOAD_SKEL_OBJECT(skel_name) \ - ( \ - { \ - __label__ load_bpf_skel_out; \ - int __ret = 0; \ - skel_name = skel_name##_bpf__open(); \ - if (!skel_name) \ - { \ - printf("failed to open BPF object\n"); \ - __ret = -1; \ - goto load_bpf_skel_out; \ - } \ - __ret = skel_name##_bpf__load(skel_name); \ - if (__ret) \ - { \ - printf("failed to load BPF object: %d\n", __ret); \ - DESTORY_SKEL_BOJECT(skel_name); \ - goto load_bpf_skel_out; \ - } \ - __ret = skel_name##_bpf__attach(skel_name); \ - if (__ret) \ - { \ - printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ - DESTORY_SKEL_BOJECT(skel_name); \ - goto load_bpf_skel_out; \ - } \ - load_bpf_skel_out: \ - __ret; \ - }) - -#define DESTORY_SKEL_BOJECT(skel_name) \ - skel_name##_bpf__destroy(skel_name); - int init(void *arg); int call(int t, struct unity_lines *lines); void deinit(void); -- Gitee From 887fca0276f62f997f5f6bb9fce6d48fca4dfb27 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Tue, 28 Feb 2023 11:15:41 +0800 Subject: [PATCH 39/77] coolbpf: update Signed-off-by: Shuyi Cheng --- source/lib/internal/ebpf/coolbpf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lib/internal/ebpf/coolbpf b/source/lib/internal/ebpf/coolbpf index c9a6c68b..8db5ed33 160000 --- a/source/lib/internal/ebpf/coolbpf +++ b/source/lib/internal/ebpf/coolbpf @@ -1 +1 @@ -Subproject commit c9a6c68b9dd375618cdfd5b5885752e88c2faf60 +Subproject commit 8db5ed3383f9c4521b33514c9b03d31581d65e6c -- Gitee From 7a09d32a1d37223931cfe83243a08960c07f08aa Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 1 Mar 2023 03:37:11 +0000 Subject: [PATCH 40/77] unity/loadavg: Fix the table name to adjust the ymal Signed-off-by: Hailong Liu --- .../monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index d06dd792..df4bbc2a 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -76,7 +76,7 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); - unity_set_table(line, "sched_moni"); + unity_set_table(line, "proc_loadavg"); full_line(line); return 0; } -- Gitee From a6096cb789bc685f8a403cda36847b87798e6a9c Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 1 Mar 2023 12:01:16 +0800 Subject: [PATCH 41/77] ioMonitor: Don't need module requests Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/ioMon/nfPut.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/tools/monitor/ioMonitor/ioMon/nfPut.py b/source/tools/monitor/ioMonitor/ioMon/nfPut.py index 5fcaea96..dbede66c 100755 --- a/source/tools/monitor/ioMonitor/ioMon/nfPut.py +++ b/source/tools/monitor/ioMonitor/ioMon/nfPut.py @@ -14,7 +14,6 @@ __author__ = 'liaozhaoyan' import os import socket -import requests MAX_BUFF = 128 * 1024 -- Gitee From 848f09f3b5b452c70ece985dbfbe0bb778b550f2 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 1 Mar 2023 09:48:41 +0000 Subject: [PATCH 42/77] unity/nosched,irqoff: fix some bugs Signed-off-by: Hailong Liu --- .../plugin/unity_irqoff/unity_irqoff.c | 22 ++++++------ .../plugin/unity_nosched/unity_nosched.c | 35 +++++++++++-------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c index 81f89ea9..0f92e59d 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c @@ -24,7 +24,7 @@ struct env { }; static int nr_cpus; -struct sched_jit_summary summary, *percpu_summary; +struct sched_jit_summary summary, prev; struct bpf_link **sw_mlinks, **hw_mlinks= NULL; DEFINE_SEKL_OBJECT(unity_irqoff); @@ -204,7 +204,8 @@ int init(void *arg) return 0; } - +#define delta(sum, value) \ + sum.value - prev.value int call(int t, struct unity_lines *lines) { struct unity_line *line; @@ -213,14 +214,15 @@ int call(int t, struct unity_lines *lines) line = unity_get_line(lines, 0); unity_set_table(line, "sched_moni_jitter"); unity_set_index(line, 0, "mod", "irqoff"); - unity_set_value(line, 0, "dltnum", summary.num); - unity_set_value(line, 1, "dlttm", summary.total); - unity_set_value(line, 2, "lt10ms", summary.less10ms); - unity_set_value(line, 3, "lt50ms", summary.less50ms); - unity_set_value(line, 4, "lt100ms", summary.less100ms); - unity_set_value(line, 5, "lt500ms", summary.less500ms); - unity_set_value(line, 6, "lt1s", summary.less1s); - unity_set_value(line, 7, "mts", summary.plus1s); + unity_set_value(line, 0, "dltnum", delta(summary, num)); + unity_set_value(line, 1, "dlttm", delta(summary, total)); + unity_set_value(line, 2, "lt10ms", delta(summary, less10ms)); + unity_set_value(line, 3, "lt50ms", delta(summary, less50ms)); + unity_set_value(line, 4, "lt100ms", delta(summary, less100ms)); + unity_set_value(line, 5, "lt500ms", delta(summary, less500ms)); + unity_set_value(line, 6, "lt1s", delta(summary, less1s)); + unity_set_value(line, 7, "mts", delta(summary, plus1s)); + prev = summary; return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c index 2ab26875..77e7f7f4 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -18,7 +18,7 @@ #endif unsigned int nr_cpus; -struct sched_jit_summary summary; +struct sched_jit_summary summary, prev; static void update_summary(struct sched_jit_summary* summary, const struct event *e) { @@ -42,13 +42,15 @@ static void update_summary(struct sched_jit_summary* summary, const struct event void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) { - struct event *e = (struct event *)data; - - e->delay = e->delay/(1000*1000); - if (e->cpu > nr_cpus - 1) + struct event e; + struct event *ev = (struct event *)data; + + e = *ev; + e.delay = e.delay/(1000*1000); + if (e.cpu > nr_cpus - 1) return; - if (e->exit != 0) - update_summary(&summary, e); + if (e.exit != 0) + update_summary(&summary, &e); } @@ -127,6 +129,8 @@ int init(void *arg) return 0; } +#define delta(sum, value) \ + sum.value - prev.value int call(int t, struct unity_lines *lines) { struct unity_line *line; @@ -135,14 +139,15 @@ int call(int t, struct unity_lines *lines) line = unity_get_line(lines, 0); unity_set_table(line, "sched_moni_jitter"); unity_set_index(line, 0, "mod", "noschd"); - unity_set_value(line, 0, "dltnum", summary.num); - unity_set_value(line, 1, "dlttm", summary.total); - unity_set_value(line, 2, "lt10ms", summary.less10ms); - unity_set_value(line, 3, "lt50ms", summary.less50ms); - unity_set_value(line, 4, "lt100ms", summary.less100ms); - unity_set_value(line, 5, "lt500ms", summary.less500ms); - unity_set_value(line, 6, "lt1s", summary.less1s); - unity_set_value(line, 7, "mts", summary.plus1s); + unity_set_value(line, 0, "dltnum", delta(summary,num)); + unity_set_value(line, 1, "dlttm", delta(summary,total)); + unity_set_value(line, 2, "lt10ms", delta(summary,less10ms)); + unity_set_value(line, 3, "lt50ms", delta(summary,less50ms)); + unity_set_value(line, 4, "lt100ms", delta(summary,less100ms)); + unity_set_value(line, 5, "lt500ms", delta(summary,less500ms)); + unity_set_value(line, 6, "lt1s", delta(summary,less1s)); + unity_set_value(line, 7, "mts", delta(summary,plus1s)); + prev = summary; return 0; } -- Gitee From 6034df49ed437bb13c7ef6c07c82316a94b222a7 Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 1 Mar 2023 18:12:42 +0800 Subject: [PATCH 43/77] ioMonitor: Fix log path error in iolatency/iohang Signed-off-by: guangshui.li --- .../monitor/ioMonitor/ioMon/exceptDiagnoseClass.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py index 98f912ce..7a0d5f74 100755 --- a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py +++ b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py @@ -35,7 +35,6 @@ class runDiag(object): return startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) logdir = self.logRootPath+'/iosdiag/hangdetect/'+startTime - file = logdir+'/result.log.seq' outlog = logdir+'/resultCons.log' if not os.path.exists(logdir): try: @@ -45,10 +44,10 @@ class runDiag(object): self.lastDiagTimeDicts['iohang'] = now if devname is not None: os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ - file+' '+devname+' > '+outlog+' &') + logdir+' '+devname+' > '+outlog+' &') else: os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ - file+' > '+outlog+' &') + logdir+' > '+outlog+' &') self.display.start(20, 'iohang', logdir, now, now+60) @@ -61,7 +60,6 @@ class runDiag(object): return startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) logdir = self.logRootPath+'/iosdiag/latency/'+startTime - file = logdir+'/result.log.seq' outlog = logdir+'/resultCons.log' if not os.path.exists(logdir): try: @@ -71,10 +69,10 @@ class runDiag(object): self.lastDiagTimeDicts['iolatency'] = now if devname is not None: os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + - ' -T 45 -f '+file+' '+devname+' > '+outlog+' &') + ' -T 45 -f '+logdir+' '+devname+' > '+outlog+' &') else: os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + - ' -T 45 -f '+file+' > '+outlog+' &') + ' -T 45 -f '+logdir+' > '+outlog+' &') if ioburst: self.display.markIoburst(now) self.display.start(60, 'iolatency', logdir, now, now+60) -- Gitee From 3148e51b3f55eb5a2b4980d63a3b5a07b0d8cb6a Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 1 Mar 2023 18:19:17 +0800 Subject: [PATCH 44/77] ioMonitor: main is ioMonitorMain Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py index 0610135c..ffc13f29 100755 --- a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py @@ -126,7 +126,7 @@ class ioMonCfgClass(object): cmdline = f.read().strip() except Exception: sys.exit(0) - if 'ioMonEntry' in cmdline: + if 'ioMonitorMain' in cmdline: os.system('kill -USR2 '+str(pid)) -- Gitee From 5c83ef5983300c4fa9a634c50e90f2d6022ec767 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 1 Mar 2023 18:45:55 +0800 Subject: [PATCH 45/77] fix bug for proc_mount. --- source/tools/monitor/unity/beaver/beaver.lua | 4 +- source/tools/monitor/unity/beaver/export.lua | 2 +- source/tools/monitor/unity/beaver/frame.lua | 9 +- .../monitor/unity/beaver/localBeaver.lua | 4 +- .../monitor/unity/beaver/query/baseQuery.lua | 245 ++++++++++++++++++ source/tools/monitor/unity/beaver/url_api.lua | 8 +- source/tools/monitor/unity/beeQ/apps.c | 3 +- source/tools/monitor/unity/beeQ/bees.lua | 6 +- source/tools/monitor/unity/beeQ/foxRecv.lua | 4 +- source/tools/monitor/unity/beeQ/pack.sh | 2 + .../monitor/unity/collector/native/sig_stop.c | 4 +- .../tools/monitor/unity/collector/plugin.yaml | 12 +- .../plugin/net_retrans/net_retrans.c | 28 +- .../unity/collector/plugin/plugin_head.h | 25 +- .../monitor/unity/collector/proc_mounts.lua | 44 ++-- .../monitor/unity/collector/proc_uptime.lua | 59 +++++ source/tools/monitor/unity/common/system.lua | 40 +++ .../tools/monitor/unity/test/bees/reload.sh | 3 + source/tools/monitor/unity/test/bees/run.sh | 13 +- source/tools/monitor/unity/test/bpf/dStack.py | 55 ++++ source/tools/monitor/unity/test/fox/query.py | 2 +- .../tools/monitor/unity/test/string/mdEsc.lua | 40 +++ source/tools/monitor/unity/tsdb/foxTSDB.lua | 11 +- 23 files changed, 549 insertions(+), 74 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/query/baseQuery.lua create mode 100644 source/tools/monitor/unity/collector/proc_uptime.lua create mode 100755 source/tools/monitor/unity/test/bees/reload.sh create mode 100644 source/tools/monitor/unity/test/bpf/dStack.py create mode 100644 source/tools/monitor/unity/test/string/mdEsc.lua diff --git a/source/tools/monitor/unity/beaver/beaver.lua b/source/tools/monitor/unity/beaver/beaver.lua index 5774a8d1..2431d99c 100644 --- a/source/tools/monitor/unity/beaver/beaver.lua +++ b/source/tools/monitor/unity/beaver/beaver.lua @@ -15,6 +15,7 @@ local CurlGuide = require("beaver.url_guide") local CurlExportHtml = require("beaver.url_export_html") local CurlExportRaw = require("beaver.url_export_raw") local CLocalBeaver = require("beaver.localBeaver") +local CbaseQuery = require("beaver.query.baseQuery") local lb = nil @@ -24,9 +25,10 @@ function init(fYaml) local web = Cframe.new() CurlIndex.new(web) - CurlApi.new(web) + CurlApi.new(web, fYaml) CurlRpc.new(web) CurlGuide.new(web) + CbaseQuery.new(web, fYaml) local Cidentity = require("beaver.identity") local inst = Cidentity.new(fYaml) diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index b6f3b52e..3fc94823 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -16,7 +16,7 @@ function Cexport:_init_(instance, fYaml) local ms = system:parseYaml(fYaml) self._freq = ms.config.freq self._tDescr = ms.metrics - self._fox = CfoxTSDB.new() + self._fox = CfoxTSDB.new(fYaml) self._fox:_setupRead() end diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 29b98103..2f64132c 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -132,7 +132,7 @@ function Cframe:findObjRes(path) end end -function Cframe:proc(fread) +function Cframe:proc(fread, session) local stream = waitHttpHead(fread) if stream == nil then -- read return stream or error code or nil return nil @@ -140,19 +140,20 @@ function Cframe:proc(fread) local tReq = self:parse(fread, stream) if tReq then + tReq.session = session if self._objs[tReq.path] then local obj = self._objs[tReq.path] local res, keep = obj:call(tReq) - return res, keep + return res, keep, tReq.session end local obj = self:findObjRes(tReq.path) if obj then local res, keep = obj:calls(tReq) - return res, keep + return res, keep, tReq.session end - return self:echo404(), false + return self:echo404(), false, {} end end diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index cb062006..0124b6b9 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -199,8 +199,10 @@ end function CLocalBeaver:_proc(fd) local fread = self:read(fd) + local session = {} + local res, alive while true do - local res, alive = self._frame:proc(fread) + res, alive, session = self._frame:proc(fread, session) if res then local stat = self:write(fd, res) diff --git a/source/tools/monitor/unity/beaver/query/baseQuery.lua b/source/tools/monitor/unity/beaver/query/baseQuery.lua new file mode 100644 index 00000000..ceae638c --- /dev/null +++ b/source/tools/monitor/unity/beaver/query/baseQuery.lua @@ -0,0 +1,245 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/27 11:41 PM +--- + +require("common.class") +local system = require("common.system") +local CfoxTSDB = require("tsdb.foxTSDB") +local ChttpHtml = require("httplib.httpHtml") + +local CbaseQuery = class("baseQuery", ChttpHtml) + +function CbaseQuery:_init_(frame, fYaml) + ChttpHtml._init_(self) + self._urlCb["/query/base"] = function(tReq) return self:base(tReq) end + self._urlCb["/query/baseQ"] = function(tReq) return self:baseQ(tReq) end + self._fox = CfoxTSDB.new(fYaml) + self:_install(frame) +end + +local function packForm1(forms) + forms[1] = '
' +end + +local function packForm2(forms) + table.insert(forms, '\n
') +end + +local function packTimeFormat(forms, session) + session.gmt = session.gmt or "0" + table.insert(forms, '') + if session.gmt == '1' then + table.insert(forms, 'GMT 时间') + table.insert(forms, '本地时间') + else + table.insert(forms, 'GMT 时间') + table.insert(forms, '本地时间') + end + table.insert(forms, '
') +end + +local formTableHead = [[ + + +
+]] +local function packTables(forms, session, tables) + session.selTable = session.selTable or tables[1] + table.insert(forms, formTableHead) + local len = #forms + for i, tbl in ipairs(tables) do + if tbl == session.selTable then + forms[i + len] = string.format('', tbl, tbl) + else + forms[i + len] = string.format('', tbl, tbl) + end + end + table.insert(forms, formTableEnd) +end + +local formTLHead = [[ + + +
+]] +local formTLIndex = {'5', '10', '20', '30', '60', '120', '240', '720', '1440'} +local formTLKV = { + ["5"] = "5m", ["10"] = "10m", ["20"] = "20m", ["30"] = "30m", ["60"] = "1h", + ["120"] = "2h", ["240"] = "4h", ["720"] = "6h", ["1440"] = "24h", +} +local function packTimeLen(forms, session) + session.timeLen = session.timeLen or "30" + table.insert(forms, formTLHead) + for _, k in ipairs(formTLIndex) do + if k == session.timeLen then + table.insert(forms, string.format('', k, formTLKV[k])) + else + table.insert(forms, string.format('', k, formTLKV[k])) + end + end + table.insert(forms, formTLEnd) +end + +local function packForm(session, tables) + local forms = {} + packForm1(forms) + packTimeFormat(forms, session) + packTables(forms, session, tables) + packTimeLen(forms, session) + packForm2(forms) + return table.concat(forms, "\n") +end + +function CbaseQuery:qTables(session, fresh) + fresh = fresh or false + local t = session.qlast or 4 * 6 + if session.tables == nil or fresh then + session.tables = self._fox:qTabelNow(t * 60) + end +end + +function CbaseQuery:base(tReq) + local res = {title="Beaver Query"} + self:qTables(tReq.session) + res.content = packForm(tReq.session, tReq.session.tables) + return res +end + +function CbaseQuery:setSession(queries, session) + if queries.selTable then + session.selTable = queries.selTable + session.gmt = queries.gmt + session.timeLen = queries.timeLen + end +end + +local function escape(s) + if type(s) == "string" then + s = system:escHtml(s) + return system:escMd(s) + end + return "None" +end +local function packDataHead(res, labels, values, logs) + local heads = system:listMerge({"time"}, labels, values, logs) + local show_head = {} + for i, v in ipairs(heads) do + show_head[i] = escape(v) + end + table.insert(res, table.concat({"| ", table.concat(show_head, " | "), " |"})) + + local aligns = {} + table.insert(aligns, "---:") -- for time align left + for _, _ in ipairs(labels) do + table.insert(aligns, ":---") -- for labels align right + end + for _, _ in ipairs(values) do + table.insert(aligns, ":---:") -- for values align center + end + for _, _ in ipairs(logs) do + table.insert(aligns, ":---") -- for values align right + end + table.insert(res, table.concat({"| ", table.concat(aligns, " | "), " |"})) +end + + +local function packDataBody(res, fmt, ms, labels, values, logs) + local len = #res + for i, m in ipairs(ms) do + local datas = {} + local ii = 1 + if fmt then + datas[ii] = os.date("!%x %X", tonumber(m.time) / 1000000) + else + datas[ii] = os.date("%x %X", tonumber(m.time) / 1000000) + end + ii = ii + 1 + + for _, k in ipairs(labels) do + datas[ii] = escape(m.labels[k]) + ii = ii + 1 + end + for _, k in ipairs(values) do + local v = m.values[k] + if v then + datas[ii] = string.format("%7.2f", m.values[k]) + else + datas[ii] = "None" + end + ii = ii + 1 + end + for _, k in ipairs(logs) do + datas[ii] = escape(m.logs[k]) + ii = ii + 1 + end + local data = table.concat({"| ", table.concat(datas, " | "), " |"}) + res[len + i] = data + end +end + +local function packDataTabel(res, ms, tFmt) + if #ms > 0 then + local fmt = false + if tFmt == "1" then + fmt = true + end + + local labels, values, logs = {}, {}, {} + for k, _ in pairs(ms[1].labels) do + table.insert(labels, k) + end + for k, _ in pairs(ms[1].values) do + table.insert(values, k) + end + for k, _ in pairs(ms[1].logs) do + table.insert(logs, k) + end + packDataHead(res, labels, values, logs) + packDataBody(res, fmt, ms, labels, values, logs) + end +end + +function CbaseQuery:baseQ(tReq) + local res = {title="Beaver Query"} + local contents = {} + local session = tReq.session + + if tReq.queries then + self:setSession(tReq.queries, session) + end + + if session.selTable == nil then + contents[1] = "查询表未设置，将跳转会设置页面." + contents[2] = '' + res.content = table.concat(contents, "\n") + return res + end + + local ms = self._fox:qNow(tonumber(session.timeLen) * 60, + {session.selTable}) + table.insert(contents, "# 反馈输入\n") + table.insert(contents, "* 表名: " .. session.selTable) + table.insert(contents, "* 时间戳: " .. session.gmt) + table.insert(contents, "* 时长: " .. session.timeLen) + table.insert(contents, "\n") + + table.insert(contents, "# 显示表格\n") + + packDataTabel(contents, ms, session.gmt) + + table.insert(contents, "[返回](/query/base)") + table.insert(contents, "[刷新](/query/baseQ)") + + res.content = self:markdown(table.concat(contents, "\n")) + return res +end + +return CbaseQuery diff --git a/source/tools/monitor/unity/beaver/url_api.lua b/source/tools/monitor/unity/beaver/url_api.lua index 4eab10dd..e78df154 100644 --- a/source/tools/monitor/unity/beaver/url_api.lua +++ b/source/tools/monitor/unity/beaver/url_api.lua @@ -10,13 +10,13 @@ local ChttpApp = require("httplib.httpApp") local CfoxTSDB = require("tsdb.foxTSDB") local CurlApi = class("urlApi", ChttpApp) -function CurlApi:_init_(frame) +function CurlApi:_init_(frame, fYaml) ChttpApp._init_(self) self._urlCb["/api/sum"] = function(tReq) return self:sum(tReq) end self._urlCb["/api/sub"] = function(tReq) return self:sub(tReq) end self._urlCb["/api/query"] = function(tReq) return self:query(tReq) end self:_install(frame) - self:_setupQs() + self:_setupQs(fYaml) end function CurlApi:sum(tReq) @@ -89,8 +89,8 @@ function CurlApi:qtable(tJson) return self.fox:qTabelNow(secs) end -function CurlApi:_setupQs() - self.fox = CfoxTSDB.new() +function CurlApi:_setupQs(fYaml) + self.fox = CfoxTSDB.new(fYaml) self._q = {} self._q["last"] = function(tJson) return self:qlast(tJson) end self._q["table"] = function(tJson) return self:qtable(tJson) end diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index 2e522bc9..2ccab816 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -70,7 +70,8 @@ static int call_init(lua_State *L, int err_func) { lua_getglobal(L, "init"); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 1, 1, err_func); + lua_pushstring(L, g_yaml_file); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { goto endCall; } diff --git a/source/tools/monitor/unity/beeQ/bees.lua b/source/tools/monitor/unity/beeQ/bees.lua index 70658b04..56e23285 100644 --- a/source/tools/monitor/unity/beeQ/bees.lua +++ b/source/tools/monitor/unity/beeQ/bees.lua @@ -10,9 +10,11 @@ local CfoxRecv = require("beeQ.foxRecv") local unistd = require("posix.unistd") -local fox = CfoxRecv.new() +local fox -function init(tid) +function init(tid, fYaml) + fYaml = fYaml or "../collector/plugin.yaml" + fox = CfoxRecv.new(fYaml) print(string.format("hello beeQ, pid: %d, tid: %d", unistd.getpid(), tid)) return 0 end diff --git a/source/tools/monitor/unity/beeQ/foxRecv.lua b/source/tools/monitor/unity/beeQ/foxRecv.lua index dfb52a39..5f1e88bf 100644 --- a/source/tools/monitor/unity/beeQ/foxRecv.lua +++ b/source/tools/monitor/unity/beeQ/foxRecv.lua @@ -10,8 +10,8 @@ local CfoxTSDB = require("tsdb.foxTSDB") local CfoxRecv = class("CfoxRecv") -function CfoxRecv:_init_() - self._fox = CfoxTSDB.new() +function CfoxRecv:_init_(fYaml) + self._fox = CfoxTSDB.new(fYaml) self._fox:setupWrite() end diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 2b002141..8b67eb5b 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -22,7 +22,9 @@ cp -r /usr/local/share/lua/5.1/* ${DIST}/lua/ mkdir ${APP} mkdir ${APP}/beaver mkdir ${APP}/beaver/native +mkdir ${APP}/beaver/query cp -r beaver/guide ${APP}/beaver/ +cp -r beaver/query ${APP}/beaver/ cp beaver/*.lua ${APP}/beaver/ cp beaver/native/*.lua ${APP}/beaver/native diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index c006072d..45bba2df 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -22,7 +22,7 @@ void plugin_stop(void) { void plugin_thread_stop(pthread_t tid) { if (tid > 0) { printf("send sig stop to thread %lu\n", tid); - pthread_kill(tid, SIGQUIT); + pthread_kill(tid, SIGUSR1); pthread_join(tid, NULL); } } @@ -37,7 +37,7 @@ static void sig_register(void) { action.sa_handler = stop_signal_handler; sigemptyset(&action.sa_mask); action.sa_flags = 0; - sigaction(SIGQUIT, &action, NULL); + sigaction(SIGUSR1, &action, NULL); } static void bump_memlock_rlimit1(void) diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 96bc4339..2c45d5a6 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -9,12 +9,16 @@ config: # mode: hostip proc_path: /mnt/host/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ # proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ + db: + rotate: 7 # tsdb file retention time, unit day + budget: 200 # max query buffer from tsdb. outline: - /tmp/sysom luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", - "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat", + "proc_uptime"] plugins: - so: kmsg @@ -25,9 +29,9 @@ plugins: - so: sample_threads description: "threads example." -# - -# so: bpfsample2 -# description: "bpf threads example." + - + so: bpfsample2 + description: "bpf threads example." - so: proc_schedstat diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c index 0cc8ca67..3062aa61 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -104,10 +104,9 @@ void deinit(void) #define LOG_MAX 256 static char log[LOG_MAX]; -static char * transIP(unsigned long lip) { - struct in_addr addr; - memcpy(&addr, &lip, sizeof(lip)); - return inet_ntoa(addr); +static int transIP(unsigned long lip, char *result, int size) { + inet_ntop(AF_INET, (void *) &lip, result, size); + return 0; } static const char * resetSock(int stack_fd, struct data_t *e){ @@ -167,9 +166,14 @@ static const char * resetActive(int stack_fd, struct data_t *e){ } int proc(int stack_fd, struct data_t *e, struct unity_line *line) { + char sip[32]; + char dip[32]; + + transIP(e->ip_src, sip, 32); + transIP(e->ip_dst, dip, 32); snprintf(log, LOG_MAX, "task:%d|%s, tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ - transIP(e->ip_src), htons(e->sport), \ - transIP(e->ip_src), htons(e->sport), \ + sip, htons(e->sport), \ + dip, htons(e->sport), \ e->sk_state); switch (e->type) { case NET_RETRANS_TYPE_RTO: @@ -177,30 +181,30 @@ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { case NET_RETRANS_TYPE_SYN: case NET_RETRANS_TYPE_SYN_ACK: { - char buf[LOG_MAX]; - snprintf(buf, LOG_MAX, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " + char buf[LOG_MAX - 1]; + snprintf(buf, LOG_MAX - 1, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " "snd_wnd:%d, rcv_wnd:%d, lost_out:%d, packets_out:%d, retrans_out:%d, " "sacked_out:%d, reordering:%d", e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, e->sacked_out, e->reordering ); - strncat(log, buf, LOG_MAX); + strncat(log, buf, LOG_MAX -1); } break; case NET_RETRANS_TYPE_RST: - strncat(log, "noport", LOG_MAX); + strncat(log, "noport", LOG_MAX - 1); break; case NET_RETRANS_TYPE_RST_SK: { const char *type = resetSock(stack_fd, e); - strncat(log, type, LOG_MAX); + strncat(log, type, LOG_MAX - 1); } break; case NET_RETRANS_TYPE_RST_ACTIVE: { const char *type = resetActive(stack_fd, e); - strncat(log, type, LOG_MAX); + strncat(log, type, LOG_MAX - 1); } break; default: diff --git a/source/tools/monitor/unity/collector/plugin/plugin_head.h b/source/tools/monitor/unity/collector/plugin/plugin_head.h index ee00783d..924e2c90 100644 --- a/source/tools/monitor/unity/collector/plugin/plugin_head.h +++ b/source/tools/monitor/unity/collector/plugin/plugin_head.h @@ -5,23 +5,28 @@ #ifndef UNITY_PLUGIN_HEAD_H #define UNITY_PLUGIN_HEAD_H +#define TABLE_SIZE 32 +#define NAME_SIZE 16 +#define INDEX_SIZE 16 + + struct unity_index { - char name[16]; - char index[16]; + char name[NAME_SIZE]; + char index[INDEX_SIZE]; }; struct unity_value { - char name[16]; + char name[NAME_SIZE]; double value; }; struct unity_log { - char name[16]; + char name[NAME_SIZE]; char* log; }; struct unity_line { - char table[32]; + char table[TABLE_SIZE]; struct unity_index indexs[4]; struct unity_value values[32]; struct unity_log logs[1]; @@ -84,7 +89,7 @@ inline struct unity_line * unity_get_line(struct unity_lines * lines, unsigned i } inline int unity_set_table(struct unity_line * line, const char * table) { - strncpy(line->table, table, 32); + strncpy(line->table, table, TABLE_SIZE - 1); return 0; } @@ -93,8 +98,8 @@ inline int unity_set_index(struct unity_line * line, if (i >= 4) { return -ERANGE; } - strncpy(line->indexs[i].name, name, 16); - strncpy(line->indexs[i].index, index, 16); + strncpy(line->indexs[i].name, name, NAME_SIZE - 1); + strncpy(line->indexs[i].index, index, INDEX_SIZE - 1); return 0; } @@ -103,14 +108,14 @@ inline int unity_set_value(struct unity_line * line, if (i >= 32) { return -ERANGE; } - strncpy(line->values[i].name, name, 16); + strncpy(line->values[i].name, name, NAME_SIZE - 1); line->values[i].value = value; return 0; } inline int unity_set_log(struct unity_line * line, const char * name, const char * log) { - strncpy(line->logs[0].name, name, 16); + strncpy(line->logs[0].name, name, NAME_SIZE - 1); line->logs[0].log = strdup(log); return 0; } diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 6ad8644a..1e5d0723 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -81,27 +81,29 @@ end function CprocMounts:_proc() for k, v in pairs(self._mpoints) do local stat = statvfs(k) - local ls = { - { - name = "fs", - index = v, - }, - { - name = "mount", - index = k, - }, - } - local vs = { - { name="f_bsize", value=stat.f_bsize, }, - { name="f_blocks", value=stat.f_blocks, }, - { name="f_bfree", value=stat.f_bfree, }, - { name="f_bavail", value=stat.f_bavail, }, - { name="f_files", value=stat.f_files, }, - { name="f_ffree", value=stat.f_ffree, }, - { name="f_favail", value=stat.f_favail, }, - } - local line = self:_packProto("fs_stat", ls, vs) - self:appendLine(line) + if stat then -- stat may return 0 + local ls = { + { + name = "fs", + index = v, + }, + { + name = "mount", + index = k, + }, + } + local vs = { + { name="f_bsize", value=stat.f_bsize, }, + { name="f_blocks", value=stat.f_blocks, }, + { name="f_bfree", value=stat.f_bfree, }, + { name="f_bavail", value=stat.f_bavail, }, + { name="f_files", value=stat.f_files, }, + { name="f_ffree", value=stat.f_ffree, }, + { name="f_favail", value=stat.f_favail, }, + } + local line = self:_packProto("fs_stat", ls, vs) + self:appendLine(line) + end end end diff --git a/source/tools/monitor/unity/collector/proc_uptime.lua b/source/tools/monitor/unity/collector/proc_uptime.lua new file mode 100644 index 00000000..538b3df2 --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_uptime.lua @@ -0,0 +1,59 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/1 3:13 PM +--- + +require("common.class") +require("uname") +local CvProc = require("collector.vproc") + +local CprocUptime = class("procUptime", CvProc) + +function CprocUptime:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/uptime") + local distro = uname() + self._labels = { + {name = "sysname", index = distro.sysname}, + {name = "nodename", index = distro.nodename}, + {name = "release", index = distro.release}, + {name = "version", index = distro.version}, + {name = "machine", index = distro.machine}, + } + self._release = mnt .. "etc/system-release" +end + +local function readNum(pFile) + local f = io.open(pFile,"r") + local res1, res2 = f:read("*n"), f:read("*n") + f:close() + return res1, res2 +end + +local function readRelease(pFile) + local f = io.open(pFile) + local res = "unknown" + if f then + res = f:read() + f:close() + end + return res +end + +function CprocUptime:proc(elapsed, lines) + CvProc.proc(self) + local uptime, idletime = readNum(self.pFile) + local vs = { + {name = "uptime", value = uptime}, + {name = "idletime", value = idletime}, + {name = "stamp", value = os.time()}, + } + self:appendLine(self:_packProto("uptime", nil, vs)) + local dummyValue = {{name = "dummy", value=1.0}} + self:appendLine(self:_packProto("uname", self._labels, dummyValue)) + local releaseInfo = {{name = "release", index = readRelease(self._release)}} + self:appendLine(self:_packProto("system_release", releaseInfo, dummyValue)) + self:push(lines) +end + +return CprocUptime diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index 68dfa7f5..e2173ded 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -89,6 +89,20 @@ function system:dictCopy(tbl) return cp end +function system:listMerge(...) + local res = {} + local i = 1 + for _, vt in ipairs{...} do + if type(vt) == "table" then + for _, v in ipairs(vt) do + res[i] = v + i = i + 1 + end + end + end + return res +end + function system:hex2ups(hex) return (string.gsub(hex, ".", function (c) return string.format("%02X", string.byte(c)) @@ -111,6 +125,29 @@ function system:hexdump(buf) end end +local htmlRep = { + ["<"] = function() return "<" end, + [">"] = function() return ">" end, + ["&"] = function() return "&" end, + ['"'] = function() return """ end, + ["\t"] = function() return " " end, +} +local function esc_html(s) + return htmlRep[s]() +end +function system:escHtml(s) + local reHtml = '[<>&"\t]' + return string.gsub(s, reHtml, function(s) return esc_html(s) end) +end + +local function esc_md(s) + return "\\" .. s +end +function system:escMd(s) + local reFmt = "[\\`%*_%{%}%[%]%(%)#%+%-%.!|]" + return string.gsub(s, reFmt, function(s) return esc_md(s) end) +end + function system:timeRfc1123(t) t = t or os.time() return os.date("!%a, %d %b %Y %H:%M:%S GMT", t) @@ -119,6 +156,9 @@ end function system:parseYaml(fYaml) local lyaml = require("lyaml") local f = io.open(fYaml,"r") + if not f then + error("file: " .. fYaml .. " is not exist.") + end local s = f:read("*all") f:close() diff --git a/source/tools/monitor/unity/test/bees/reload.sh b/source/tools/monitor/unity/test/bees/reload.sh new file mode 100755 index 00000000..472c8c65 --- /dev/null +++ b/source/tools/monitor/unity/test/bees/reload.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pkill -1 unity-mon \ No newline at end of file diff --git a/source/tools/monitor/unity/test/bees/run.sh b/source/tools/monitor/unity/test/bees/run.sh index 32a1732a..82d67271 100755 --- a/source/tools/monitor/unity/test/bees/run.sh +++ b/source/tools/monitor/unity/test/bees/run.sh @@ -1,5 +1,7 @@ #!/bin/bash +pkill unity-mon + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ @@ -8,10 +10,9 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../beaver/ source /etc/profile cd ../../beeQ -make -if [ $? -ne 0 ];then - echo " make -- Failed : "$? - exit 0 -fi -./unity-mon +yaml_path=$1 +[ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" + +echo $yaml_yaml_path +./unity-mon $yaml_path & diff --git a/source/tools/monitor/unity/test/bpf/dStack.py b/source/tools/monitor/unity/test/bpf/dStack.py new file mode 100644 index 00000000..0a7a82d4 --- /dev/null +++ b/source/tools/monitor/unity/test/bpf/dStack.py @@ -0,0 +1,55 @@ +import os +import sys +import re + +rePid = re.compile(r"^\d+$") + + +def checkState(path, pid): + fStatus = os.path.join(path, pid, "status") + try: + with open(fStatus, "r") as f: + for _, line in enumerate(f): + if line.startswith("State:"): + _, stats = line.split(":", 1) + stats, _ = stats.split("(", 1) + return stats.strip() + except Exception: + return "N" + + +def getCmd(path, pid): + fCmd = os.path.join(path, pid, "cmdline") + try: + with open(fCmd, 'r') as f: + return f.read() + except Exception: + return "unknown" + + +def getKstack(path, pid): + fCmd = os.path.join(path, pid, "stack") + try: + with open(fCmd, 'r') as f: + return f.read() + except Exception: + return "unknown" + + +def walk_pids(path, fil): + for pid in os.listdir(path): + if rePid.match(pid): + tPath = "/proc/%s/task" % pid + for tid in os.listdir(tPath): + if checkState(tPath, tid) == fil: + print("task %s, comm: %s, status: %s" % (pid, getCmd(tPath, pid), fil)) + print(getKstack(tPath, pid)) + + +if __name__ == "__main__": + if len(sys.argv) == 1: + fil = "D" + else: + fil = sys.argv[1] + walk_pids("/proc", fil) + pass diff --git a/source/tools/monitor/unity/test/fox/query.py b/source/tools/monitor/unity/test/fox/query.py index cddb328b..bda9542c 100644 --- a/source/tools/monitor/unity/test/fox/query.py +++ b/source/tools/monitor/unity/test/fox/query.py @@ -17,7 +17,7 @@ def q_table(): def q_by_table(): - post_test({"mode": "last", "time": "100m", "table": ["cpu_total", "cpus"]}) + post_test({"mode": "last", "time": "100m", "table": ["IOMonDiagLog"]}) def q_by_date(): diff --git a/source/tools/monitor/unity/test/string/mdEsc.lua b/source/tools/monitor/unity/test/string/mdEsc.lua new file mode 100644 index 00000000..50eef095 --- /dev/null +++ b/source/tools/monitor/unity/test/string/mdEsc.lua @@ -0,0 +1,40 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/28 11:00 PM +--- + +local reFmt = "[\\`%*_%{%}%[%]%(%)#%+%-%.!|]" +local reHtml = '[<>&"\t]' + +local htmlRep = { + ["<"] = function() return "<" end, + [">"] = function() return ">" end, + ["&"] = function() return "&" end, + ['"'] = function() return """ end, + ["\t"] = function() return " " end, +} + +local function escape(s) + return htmlRep[s]() +end + +local function pEscape(s) + return string.gsub(s, reHtml, function(s) return escape(s) end) +end + +local function escMd(s) + return "\\" .. s +end + +local function pEscMd(s) + return string.gsub(s, reFmt, function(s) return escMd(s) end) +end + +local s = "" +print(pEscape(s)) + +s = "\\`.*_{}[]()#+-.|!" +print(#s) +print(pEscMd(s)) + diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index 807e1ea1..002d8bf0 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -14,11 +14,11 @@ local foxFFI = require("tsdb.native.foxffi") local CfoxTSDB = class("CfoxTSDB") -function CfoxTSDB:_init_() +function CfoxTSDB:_init_(fYaml) self.ffi = foxFFI.ffi self.cffi = foxFFI.cffi self._proto = CprotoData.new(nil) - self._qBudget = 200 + self:setupConf(fYaml) end function CfoxTSDB:_del_() @@ -28,6 +28,13 @@ function CfoxTSDB:_del_() self._man = nil end +function CfoxTSDB:setupConf(fYaml) + local conf = system:parseYaml(fYaml) + local dbConf = conf.db or {budget = 200, rotate=7} + self._qBudget = dbConf.budget or 200 + self._rotate = dbConf.rotate or 7 +end + function CfoxTSDB:get_us() return self.cffi.get_us() end -- Gitee From b35eefb7c4f781c62445f6b6a0e96b9dcbc1f49f Mon Sep 17 00:00:00 2001 From: zhaohang Date: Wed, 1 Mar 2023 11:24:01 +0000 Subject: [PATCH 46/77] =?UTF-8?q?!435=20=E6=9E=84=E5=BB=BA=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E6=B7=BB=E5=8A=A0etc/sysak=20*=20skip=20numainfo=20*?= =?UTF-8?q?=20add=20etc/sysak=20in=20rpm=20build=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rpm/sysak-build-nodep.sh | 5 ++++- source/tools/monitor/unity/collector/plugin.yaml | 16 ++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/rpm/sysak-build-nodep.sh b/rpm/sysak-build-nodep.sh index 89ac80c9..9afe21a5 100755 --- a/rpm/sysak-build-nodep.sh +++ b/rpm/sysak-build-nodep.sh @@ -38,11 +38,13 @@ fi %install mkdir -p \$RPM_BUILD_ROOT/usr/bin +mkdir -p \$RPM_BUILD_ROOT/etc/sysak mkdir -p \$RPM_BUILD_ROOT/usr/local/sysak/log mkdir -p \$RPM_BUILD_ROOT/usr/lib/systemd/system/ /bin/cp -rf $BUILD_DIR/.sysak_components \$RPM_BUILD_ROOT/usr/local/sysak/.sysak_components /bin/cp -rf $BUILD_DIR/sysak \$RPM_BUILD_ROOT/usr/bin/ /bin/cp -f $BUILD_DIR/.sysak_components/tools/monitor/sysakmon.conf \$RPM_BUILD_ROOT/usr/local/sysak +/bin/cp -f $BUILD_DIR/.sysak_components/tools/dist/app/collector/plugin.yaml \$RPM_BUILD_ROOT/etc/sysak/ /bin/cp $SOURCE_DIR/rpm/sysak.service \$RPM_BUILD_ROOT/usr/lib/systemd/system/ /bin/cp $SOURCE_DIR/rpm/sysak_server.conf \$RPM_BUILD_ROOT/usr/local/sysak/ @@ -58,6 +60,7 @@ fi rm -rf /usr/local/sysak %files +/etc/sysak /usr/local/sysak /usr/bin/sysak /usr/lib/systemd/system/sysak.service @@ -83,7 +86,7 @@ main() { export LINUX_VERSION=$(uname -r) - TARGET_LIST="--enable-target-all --enable-static --disable-target-rtrace --disable-target-PingTrace" + TARGET_LIST="--enable-target-all" build_rpm } diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index c1187760..10f5e17b 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -49,9 +49,9 @@ plugins: - so: unity_irqoff description: "irqoff:detect irq turned off and can't response" - - - so: numainfo - description: "collect numainfo" + #- + # so: numainfo + # description: "collect numainfo" # - # so: cpufreq # description: "collect cpufreq" @@ -191,11 +191,11 @@ metrics: head: value help: "net_retrans_count" type: "gauge" - - title: sysak_numainfo - from: numainfo - head: value - help: "numainfo of system from /sys/devices/system/" - type: "gauge" + #- title: sysak_numainfo + # from: numainfo + # head: value + # help: "numainfo of system from /sys/devices/system/" + # type: "gauge" # - title: sysak_proc_cpufreq # from: cpufreq # head: value -- Gitee From 4d19cc1aab68d56d71236b8b7aca6ce9f15eeb43 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 2 Mar 2023 14:26:46 +0800 Subject: [PATCH 47/77] fix dist bug for bpf application. --- source/tools/monitor/unity/beaver/export.lua | 11 +----- .../monitor/unity/beaver/query/baseQuery.lua | 2 +- source/tools/monitor/unity/beeQ/apps.c | 1 - .../monitor/unity/collector/native/fastKsym.c | 7 ++++ .../monitor/unity/collector/native/fastKsym.h | 1 + .../unity/collector/native/procffi.lua | 1 + .../monitor/unity/collector/native/sig_stop.c | 5 +++ .../monitor/unity/collector/outline/outline.c | 2 + .../tools/monitor/unity/collector/plugin.lua | 1 + .../monitor/unity/collector/plugin/Makefile | 2 +- .../monitor/unity/collector/plugin/bpf_head.h | 4 +- .../plugin/net_health/net_health.bpf.c | 13 ++----- .../collector/plugin/net_health/net_health.c | 18 ++++++++- .../plugin/net_retrans/net_retrans.bpf.c | 39 +++++++++---------- .../plugin/net_retrans/net_retrans.c | 20 +++++----- source/tools/monitor/unity/test/bees/stop.sh | 3 ++ 16 files changed, 74 insertions(+), 56 deletions(-) create mode 100755 source/tools/monitor/unity/test/bees/stop.sh diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index da6cb509..a62a29d9 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -12,15 +12,6 @@ require("common.class") local Cexport = class("Cexport") -function Cexport:_init_(instance, fYaml) - self._instance = instance - local ms = system:parseYaml(fYaml) - self._freq = ms.config.freq - self._tDescr = ms.metrics - self._fox = CfoxTSDB.new(fYaml) - self._fox:_setupRead() -end - local function qFormData(from, tData) local res = {} local len = #tData @@ -80,7 +71,7 @@ function Cexport:_init_(instance, fYaml) self.pack_line = packLine end self._tDescr = ms.metrics - self._fox = CfoxTSDB.new() + self._fox = CfoxTSDB.new(fYaml) self._fox:_setupRead() end diff --git a/source/tools/monitor/unity/beaver/query/baseQuery.lua b/source/tools/monitor/unity/beaver/query/baseQuery.lua index ceae638c..db2eab3d 100644 --- a/source/tools/monitor/unity/beaver/query/baseQuery.lua +++ b/source/tools/monitor/unity/beaver/query/baseQuery.lua @@ -226,7 +226,7 @@ function CbaseQuery:baseQ(tReq) local ms = self._fox:qNow(tonumber(session.timeLen) * 60, {session.selTable}) table.insert(contents, "# 反馈输入\n") - table.insert(contents, "* 表名: " .. session.selTable) + table.insert(contents, "* 表名: " .. system:escMd(session.selTable)) table.insert(contents, "* 时间戳: " .. session.gmt) table.insert(contents, "* 时长: " .. session.timeLen) table.insert(contents, "\n") diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index 2ccab816..3c1ed69f 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -161,7 +161,6 @@ int app_recv_proc(void* msg, struct beeQ* q) { q->qarg = L; counter = sighup_counter; } - body = malloc(len); // http://www.lua.org/manual/5.1/manual.html#lua_pushlstring //Pushes the string pointed to by s with size len onto the stack. // Lua makes (or reuses) an internal copy of the given string, diff --git a/source/tools/monitor/unity/collector/native/fastKsym.c b/source/tools/monitor/unity/collector/native/fastKsym.c index ea0e2db8..611bf871 100644 --- a/source/tools/monitor/unity/collector/native/fastKsym.c +++ b/source/tools/monitor/unity/collector/native/fastKsym.c @@ -94,6 +94,8 @@ static int sort_ksym(int fd, int count) { qsort(pmmap, count, sizeof (struct ksym_cell), sym_cmp); + madvise(pmmap, sb.st_size, MADV_DONTNEED); + madvise(pmmap, sb.st_size, MADV_NORMAL); gCell = (struct ksym_cell*)pmmap; return ret; @@ -133,6 +135,11 @@ int ksym_setup(int stack_only) { return ret; } +void ksym_free(void) { + munmap((void *)gCell, sym_cnt * sizeof (struct ksym_cell)); + close(tfd); +} + struct ksym_cell* ksym_search(addr_t key) { int start = 0, end = sym_cnt; int mid; diff --git a/source/tools/monitor/unity/collector/native/fastKsym.h b/source/tools/monitor/unity/collector/native/fastKsym.h index 57bf8be2..31ddebae 100644 --- a/source/tools/monitor/unity/collector/native/fastKsym.h +++ b/source/tools/monitor/unity/collector/native/fastKsym.h @@ -15,6 +15,7 @@ struct ksym_cell { }; int ksym_setup(int stack_only); +void ksym_free(void); struct ksym_cell* ksym_search(addr_t key); #endif //FASTKSYM_FASTKSYM_H diff --git a/source/tools/monitor/unity/collector/native/procffi.lua b/source/tools/monitor/unity/collector/native/procffi.lua index 85a2ed6a..d35a6a74 100644 --- a/source/tools/monitor/unity/collector/native/procffi.lua +++ b/source/tools/monitor/unity/collector/native/procffi.lua @@ -33,6 +33,7 @@ void set_unity_sys(const char *path); int plugin_is_working(void); void plugin_stop(void); void plugin_init(void); +void plugin_deinit(void); ]] return {ffi = ffi, cffi=cffi} diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index 45bba2df..28a5421b 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -5,6 +5,7 @@ #include "sig_stop.h" #include #include +#include #include #include #include "fastKsym.h" @@ -59,3 +60,7 @@ void plugin_init(void) { sig_register(); working = 1; } + +void plugin_deinit(void) { + ksym_free(); +} diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index eae253a4..0dd6d454 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -102,10 +102,12 @@ static int work(lua_State *L) { perror("beaver.lua echo failed."); goto endReturn; } + lua_close(L); return ret; endReturn: endCall: + lua_close(L); return ret; } diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 2c0c5899..70c28589 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -27,6 +27,7 @@ function Cplugin:_del_() local cffi = plugin.cffi cffi.deinit() end + self._sig_cffi.plugin_deinit() end function Cplugin:setProcSys(procFFI, config) diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 881726f6..9c538406 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -5,7 +5,7 @@ OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq gpuinfo +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq gpuinfo all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 99adf37f..f02896f3 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -65,8 +65,8 @@ }) #define DESTORY_SKEL_BOJECT(skel_name) \ - if (perf_thread > 0) \ - kill_perf_thread(perf_thread); \ + if (perf_thread != 0) \ + plugin_thread_stop(perf_thread); \ skel_name##_bpf__destroy(skel_name); #else #define DEFINE_SEKL_OBJECT(skel_name) \ diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c index 5efee8bb..32aacfb7 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -4,24 +4,17 @@ #include #include -BPF_ARRAY(outCnt, u64, 2); +BPF_HASH(outCnt, int, u64, 2); BPF_ARRAY(netHist, u64, 20); -static inline void addCnt(int k, u64 val) { - u64 *pv = bpf_map_lookup_elem(&outCnt, &k); - if (pv) { - __sync_fetch_and_add(pv, val); - } -} - SEC("kprobe/tcp_validate_incoming") int j_tcp_validate_incoming(struct pt_regs *ctx) { struct tcp_sock *tp = (struct tcp_sock *)PT_REGS_PARM1(ctx); u64 ts = BPF_CORE_READ(tp, srtt_us) >> 3; u64 ms = ts / 1000; if (ms > 0) { - addCnt(0, ms); - addCnt(1, 1); + add_hist((struct bpf_map_def *)&outCnt, 0, ms); + add_hist((struct bpf_map_def *)&outCnt, 1, 1); hist10_push((struct bpf_map_def *)&netHist, ms); } return 0; diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c index 231f80bd..de4a5309 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -13,6 +13,8 @@ DEFINE_SEKL_OBJECT(net_health); static int cnt_fd = 0; static int dist_fd = 0; +//#define ZHAOYAN_DEBUG + int init(void *arg) { int ret; @@ -26,11 +28,23 @@ int init(void *arg) static int get_dist(unsigned long *locals) { int i = 0; unsigned long value = 0; + int key, key_next; - for (i = 0; i < DIST_ARRAY_SIZE; i ++) { - coobpf_key_value(dist_fd, &i, &value); + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } +#ifdef ZHAOYAN_DEBUG + for (i = 0; i < NET_DIST_INDEX; i ++) { + printf("%ld, ", locals[i]); } + printf("\n"); +#endif return i; } diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c index 70769cba..d9d9428b 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c @@ -26,7 +26,7 @@ struct liphdr { BPF_PERF_OUTPUT(perf, 1024); BPF_STACK_TRACE(stack, MAX_ENTRY); -BPF_ARRAY(outCnt, u64, NET_RETRANS_TYPE_MAX); +BPF_HASH(outCnt, int, u64, NET_RETRANS_TYPE_MAX); static inline void addCnt(int k, u64 val) { u64 *pv = bpf_map_lookup_elem(&outCnt, &k); @@ -61,7 +61,6 @@ static inline int get_skb_info(struct data_t* pdata, struct sk_buff *skb, u32 ty struct liphdr *piph; struct tcphdr *ptcph; - addCnt(type, 1); pdata->type = type; pdata->sk_state = 0; @@ -109,26 +108,10 @@ static inline void get_task(struct data_t* pdata, struct sock *sk) { get_sock_task(sk, pdata); } -static inline void get_socket_task(struct data_t* e, struct sock *sk) { -// struct socket* psocket; -// struct socket_wq *wq; - - e->pid = 0; - e->comm[0] = '\0'; - -// psocket = BPF_CORE_READ(sk, sk_socket); -// wq = BPF_CORE_READ(psocket, wq); -// if (wq) { -// struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); -// get_list_task(phead, e); -// } -} - static inline int get_info(struct data_t* pdata, struct sock *sk, u32 type) { struct inet_sock *inet = (struct inet_sock *)sk; - addCnt(type, 1); pdata->type = type; pdata->ip_dst = BPF_CORE_READ(sk, __sk_common.skc_daddr); pdata->dport = BPF_CORE_READ(sk, __sk_common.skc_dport); @@ -176,6 +159,7 @@ int j_tcp_enter_loss(struct pt_regs *ctx) return 0; } get_task(&data, sk); + addCnt(NET_RETRANS_TYPE_RTO, 1); get_info(&data, sk, NET_RETRANS_TYPE_RTO); data.stack_id = 0; get_tcp_info(&data, (struct tcp_sock *)sk); @@ -198,6 +182,7 @@ int j_tcp_send_probe0(struct pt_regs *ctx) return 0; } + addCnt(NET_RETRANS_TYPE_ZERO, 1); get_info(&data, sk, NET_RETRANS_TYPE_ZERO); data.stack_id = 0; get_task(&data, sk); @@ -216,13 +201,19 @@ int j_tcp_v4_send_reset(struct pt_regs *ctx) sk = (struct sock *)PT_REGS_PARM1(ctx); if (sk == NULL) { struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); + addCnt(NET_RETRANS_TYPE_RST, 1); get_skb_info(&data, skb, NET_RETRANS_TYPE_RST); get_task(&data, NULL); data.stack_id = 0; } else { + addCnt(NET_RETRANS_TYPE_RST_SK, 1); get_info(&data, sk, NET_RETRANS_TYPE_RST_SK); get_task(&data, sk); + if (data.sk_state == 10) { // for listen cath skb info. + struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); + get_skb_info(&data, skb, NET_RETRANS_TYPE_RST_SK); + } data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); } if (check_ip(&data)) { @@ -237,6 +228,8 @@ int j_tcp_send_active_reset(struct pt_regs *ctx) struct sock *sk; struct data_t data = {}; + addCnt(NET_RETRANS_TYPE_RST_ACTIVE, 1); + sk = (struct sock *)PT_REGS_PARM1(ctx); get_info(&data, sk, NET_RETRANS_TYPE_RST_ACTIVE); data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); @@ -262,8 +255,10 @@ int j_tcp_retransmit_skb(struct pt_regs *ctx){ { struct data_t data = {}; + addCnt(NET_RETRANS_TYPE_SYN, 1); get_info(&data, sk, NET_RETRANS_TYPE_SYN); get_task(&data, sk); + get_tcp_info(&data, (struct tcp_sock *)sk); bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); } return 0; @@ -272,12 +267,16 @@ int j_tcp_retransmit_skb(struct pt_regs *ctx){ SEC("kprobe/tcp_rtx_synack") int j_tcp_rtx_synack(struct pt_regs *ctx) { - struct sock *sk; + struct sock *sk, *sk2; + struct request_sock *req = (struct request_sock *)PT_REGS_PARM2(ctx); struct data_t data = {}; + addCnt(NET_RETRANS_TYPE_SYN_ACK, 1); sk = (struct sock *)PT_REGS_PARM1(ctx); - get_info(&data, sk, NET_RETRANS_TYPE_SYN_ACK); + sk2 = BPF_CORE_READ(req, sk); + get_info(&data, sk2, NET_RETRANS_TYPE_SYN_ACK); get_task(&data, sk); + get_tcp_info(&data, (struct tcp_sock *)sk2); bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c index 3062aa61..84f940a3 100644 --- a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -110,13 +110,13 @@ static int transIP(unsigned long lip, char *result, int size) { } static const char * resetSock(int stack_fd, struct data_t *e){ - unsigned long addr; - int i = 1; //last stack + unsigned long addr[128]; + int i = e->stack_id; //last stack struct ksym_cell* cell; coobpf_key_value(stack_fd, &i, &addr); - if (addr) { - cell = ksym_search(addr); + if (addr[1] > 0) { + cell = ksym_search(addr[1]); if (cell) { if (strcmp(cell->func, "tcp_v4_rcv") == 0) { if (e->sk_state == 12) { @@ -129,6 +129,7 @@ static const char * resetSock(int stack_fd, struct data_t *e){ } else if (strcmp(cell->func, "tcp_v4_do_rcv") == 0) { return "tcp_stat"; } else { + printf("sym: %s\n", cell->func); return "unknown_sock"; } } @@ -171,9 +172,9 @@ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { transIP(e->ip_src, sip, 32); transIP(e->ip_dst, dip, 32); - snprintf(log, LOG_MAX, "task:%d|%s, tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ + snprintf(log, LOG_MAX, "task:%d(%s), tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ sip, htons(e->sport), \ - dip, htons(e->sport), \ + dip, htons(e->dport), \ e->sk_state); switch (e->type) { case NET_RETRANS_TYPE_RTO: @@ -182,9 +183,9 @@ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { case NET_RETRANS_TYPE_SYN_ACK: { char buf[LOG_MAX - 1]; - snprintf(buf, LOG_MAX - 1, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " - "snd_wnd:%d, rcv_wnd:%d, lost_out:%d, packets_out:%d, retrans_out:%d, " - "sacked_out:%d, reordering:%d", + snprintf(buf, LOG_MAX - 1, "rcv_nxt:%u, rcv_wup:%u, snd_nxt:%u, snd_una:%u, copied_seq:%u, " + "snd_wnd:%u, rcv_wnd:%u, lost_out:%u, packets_out:%u, retrans_out:%u, " + "sacked_out:%u, reordering:%u", e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, e->sacked_out, e->reordering @@ -212,6 +213,7 @@ int proc(int stack_fd, struct data_t *e, struct unity_line *line) { } unity_set_table(line, "net_retrans_log"); unity_set_index(line, 0, "type", net_title[e->type]); + printf("%s\n", log); unity_set_log(line, "log", log); return 0; } diff --git a/source/tools/monitor/unity/test/bees/stop.sh b/source/tools/monitor/unity/test/bees/stop.sh new file mode 100755 index 00000000..f16bc5e9 --- /dev/null +++ b/source/tools/monitor/unity/test/bees/stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pkill unity-mon \ No newline at end of file -- Gitee From d682c6e3c4ff02a7839f4e453b3000a3ba0e99cf Mon Sep 17 00:00:00 2001 From: zhilan Date: Fri, 3 Mar 2023 14:37:24 +0800 Subject: [PATCH 48/77] unity: proc_buddyinfo: avoid mem leak --- .../tools/monitor/unity/collector/proc_buddyinfo.lua | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index f2fc8c9b..9521ac33 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -5,18 +5,23 @@ --- require("common.class") -local CkvProc = require("collector.kvProc") local CvProc = require("collector.vproc") local pystring = require("common.pystring") -local CprocBuddyinfo = class("proc_buddyinfo", CkvProc) +local CprocBuddyinfo = class("proc_buddyinfo", CvProc) function CprocBuddyinfo:_init_(proto, pffi, mnt,pFile) - CkvProc._init_(self, proto, pffi, mnt, pFile or "proc/buddyinfo", "buddyinfo") + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/buddyinfo") + self._protoTable = { + line = "buddyinfo", + ls = nil, + vs = {} + } end function CprocBuddyinfo:proc(elapsed, lines) CvProc.proc(self) + self._protoTable.vs = {} local buddyinfo = {} for line in io.lines(self.pFile) do if string.find(line,"Normal") then -- Gitee From fcf0894307a866245e17165a774bd27cab206c51 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sat, 4 Mar 2023 23:22:44 +0800 Subject: [PATCH 49/77] add local modifiers for meminfo. --- source/tools/monitor/unity/Dockerfile | 3 ++- source/tools/monitor/unity/collector/proc_meminfo.lua | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/tools/monitor/unity/Dockerfile b/source/tools/monitor/unity/Dockerfile index dbaf3064..9afeee48 100644 --- a/source/tools/monitor/unity/Dockerfile +++ b/source/tools/monitor/unity/Dockerfile @@ -2,7 +2,7 @@ FROM registry.cn-hangzhou.aliyuncs.com/sysom/lcc MAINTAINER "liaozhaoyan " WORKDIR /root/ RUN source /opt/rh/devtoolset-9/enable && \ - yum install -y make wget lua-devel unzip git && \ + yum install -y make wget lua-devel unzip git numactl-devel && \ mkdir /root/build && \ cd /root/build && \ git clone https://gitee.com/chuyansz/sysak.git && \ @@ -33,6 +33,7 @@ RUN source /opt/rh/devtoolset-9/enable && \ luarocks install sha1 && \ luarocks install md5 && \ luarocks install luaposix 35.1-1 && \ + luarocks install uname && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ && \ cd ../beeQ/ && \ make \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/proc_meminfo.lua b/source/tools/monitor/unity/collector/proc_meminfo.lua index 5a1dd387..fa306d55 100644 --- a/source/tools/monitor/unity/collector/proc_meminfo.lua +++ b/source/tools/monitor/unity/collector/proc_meminfo.lua @@ -33,7 +33,7 @@ function CprocMeminfo:readIomem() reserved = reserved + (tonumber(cells[2], 16)-tonumber(cells[1], 16)) end end - self._protoTable_dict["vs"]["res"],_ = math.modf(reserved/1024) + self._protoTable_dict["vs"]["res"], _ = math.modf(reserved/1024) end function CprocMeminfo:readVmalloc() @@ -61,11 +61,11 @@ function CprocMeminfo:readUsed() end function CprocMeminfo:readHugepage(size,name) - file = "/sys/kernel/mm/hugepages/hugepages-" .. size .. "kB/nr_hugepages" + local file = "/sys/kernel/mm/hugepages/hugepages-" .. size .. "kB/nr_hugepages" local f=io.open(file,"r") if f ~= nil then - pages = tonumber(f:read("*a")) + local pages = tonumber(f:read("*a")) io.close(f) self._protoTable_dict["vs"][name]=pages * size end @@ -91,7 +91,7 @@ function CprocMeminfo:proc(elapsed, lines) for line in io.lines(self.pFile) do self:readKV(line) end - tmp_dict = self._protoTable_dict.vs + local tmp_dict = self._protoTable_dict.vs local cell = {name="total", value=tmp_dict["MemTotal"]+tmp_dict["res"]} table.insert(self._protoTable["vs"], cell) -- Gitee From 35c589d5aaafe6f67e822192c742d5c611e51955 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sat, 4 Mar 2023 23:50:52 +0800 Subject: [PATCH 50/77] use posix.sys.utsname to instead of uname module --- .../tools/monitor/unity/beaver/identity.lua | 13 +++--- .../monitor/unity/collector/proc_uptime.lua | 45 ++++++++++++------- .../monitor/unity/test/posix/t_utsname.lua | 12 +++++ 3 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 source/tools/monitor/unity/test/posix/t_utsname.lua diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 91b96c89..220b6066 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -57,15 +57,16 @@ function Cidentity:hostname() end function Cidentity:file() + local res = "None" if self._opts.path then local file = io.open(self._opts.path, "r") - io.input(file) - local res = io.read() - io.close(file) - return res - else - return "None" + if file then + io.input(file) + res = io.read() + io.close(file) + end end + return res end function Cidentity:specify() diff --git a/source/tools/monitor/unity/collector/proc_uptime.lua b/source/tools/monitor/unity/collector/proc_uptime.lua index 538b3df2..437d13dd 100644 --- a/source/tools/monitor/unity/collector/proc_uptime.lua +++ b/source/tools/monitor/unity/collector/proc_uptime.lua @@ -5,28 +5,36 @@ --- require("common.class") -require("uname") +local utsname = require("posix.sys.utsname") local CvProc = require("collector.vproc") local CprocUptime = class("procUptime", CvProc) function CprocUptime:_init_(proto, pffi, mnt, pFile) CvProc._init_(self, proto, pffi, mnt, pFile or "proc/uptime") - local distro = uname() - self._labels = { - {name = "sysname", index = distro.sysname}, - {name = "nodename", index = distro.nodename}, - {name = "release", index = distro.release}, - {name = "version", index = distro.version}, - {name = "machine", index = distro.machine}, - } + local distro, s, errno = utsname.uname() + if distro then + self._labels = { + {name = "sysname", index = distro.sysname}, + {name = "nodename", index = distro.nodename}, + {name = "release", index = distro.release}, + {name = "version", index = distro.version}, + {name = "machine", index = distro.machine}, + } + else + error(string.format("read uname get %s, errno %d"), s, errno) + end self._release = mnt .. "etc/system-release" + self._counter = 0 end local function readNum(pFile) local f = io.open(pFile,"r") - local res1, res2 = f:read("*n"), f:read("*n") - f:close() + local res1, res2 = -1, -1 + if f then + res1, res2 = f:read("*n"), f:read("*n") + f:close() + end return res1, res2 end @@ -49,10 +57,17 @@ function CprocUptime:proc(elapsed, lines) {name = "stamp", value = os.time()}, } self:appendLine(self:_packProto("uptime", nil, vs)) - local dummyValue = {{name = "dummy", value=1.0}} - self:appendLine(self:_packProto("uname", self._labels, dummyValue)) - local releaseInfo = {{name = "release", index = readRelease(self._release)}} - self:appendLine(self:_packProto("system_release", releaseInfo, dummyValue)) + + local totalTime = elapsed * self._counter + if totalTime >= 60 * 60 then -- report by hour + local dummyValue = {{name = "dummy", value=1.0}} + self:appendLine(self:_packProto("uname", self._labels, dummyValue)) + local releaseInfo = {{name = "release", index = readRelease(self._release)}} + self:appendLine(self:_packProto("system_release", releaseInfo, dummyValue)) + self._counter = 0 + else + self._counter = self._counter + 1 + end self:push(lines) end diff --git a/source/tools/monitor/unity/test/posix/t_utsname.lua b/source/tools/monitor/unity/test/posix/t_utsname.lua new file mode 100644 index 00000000..b4b809df --- /dev/null +++ b/source/tools/monitor/unity/test/posix/t_utsname.lua @@ -0,0 +1,12 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/4 11:29 PM +--- + +package.path = package.path .. ";../../?.lua;" +local utsname = require("posix.sys.utsname") +local system = require("common.system") + +local uts = utsname.uname() +print(system:dump(uts)) \ No newline at end of file -- Gitee From 0b1cf318d9243d2509881fa3d4d579c4cfd0b007 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sun, 5 Mar 2023 00:21:08 +0800 Subject: [PATCH 51/77] add test suite for posix time. --- .../tools/monitor/unity/httplib/httpHtml.lua | 2 +- .../tools/monitor/unity/test/posix/t_time.lua | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/test/posix/t_time.lua diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index 34a9a9ea..91db3e88 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -29,7 +29,7 @@ function ChttpHtml:_init_(frame) } end -local function loadFile(fPpath) +local function loadFile(fPpath) -- conform file already exist. local f = io.open(fPpath,"r") local s = f:read("*all") f:close() diff --git a/source/tools/monitor/unity/test/posix/t_time.lua b/source/tools/monitor/unity/test/posix/t_time.lua new file mode 100644 index 00000000..a5d9c85d --- /dev/null +++ b/source/tools/monitor/unity/test/posix/t_time.lua @@ -0,0 +1,37 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/4 11:59 PM +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local ptime = require("posix.time") + +local function calcSleep(hope, now) + if hope.tv_nsec >= now.tv_nsec then + return {tv_sec = hope.tv_sec - now.tv_sec, + tv_nsec = hope.tv_nsec - now.tv_nsec} + else + return {tv_sec = hope.tv_sec - now.tv_sec - 1, + tv_nsec = 1e9 + hope.tv_nsec - now.tv_nsec} + end +end + +local unit = 5 +local tStart = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) +while true do + local now = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) + local hope = {tv_sec = tStart.tv_sec + unit, tv_nsec = tStart.tv_sec} + local diff = calcSleep(hope, now) + assert(diff.tv_sec >= 0) + print(system:dump(diff)) + local _, s, errno, _ = ptime.nanosleep(diff) + if errno then + print(string.format("new sleep stop. %d, %s", errno, s)) + break + end + tStart = hope +end + + -- Gitee From a2fc6482b72fb51a9204eddbede41bc9ae8a4db7 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sun, 5 Mar 2023 12:21:59 +0800 Subject: [PATCH 52/77] use signal USER1 to reload signals. --- source/tools/monitor/unity/beeQ/apps.c | 107 ++---------------- source/tools/monitor/unity/beeQ/bees.c | 27 ++++- .../tools/monitor/unity/beeQ/collectors.lua | 37 ++++-- source/tools/monitor/unity/beeQ/lib/beeQ.c | 2 +- .../monitor/unity/collector/native/sig_stop.c | 4 +- .../monitor/unity/collector/outline/outline.c | 4 +- .../monitor/unity/collector/outline/outline.h | 2 +- source/tools/monitor/unity/tsdb/foxTSDB.lua | 1 - 8 files changed, 63 insertions(+), 121 deletions(-) diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index 3c1ed69f..7bcebd76 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -12,7 +12,6 @@ #include #define gettidv1() syscall(__NR_gettid) -static int sample_period = 0; extern char *g_yaml_file; static int lua_traceback(lua_State *L) @@ -226,7 +225,7 @@ int collector_qout(lua_State *L) { return 1; // return a value. } -static lua_State * app_collector_init(void* q, void* proto_q) { +static int app_collector_work(void* q, void* proto_q) { int ret; int err_func; lua_Number lret; @@ -248,7 +247,7 @@ static lua_State * app_collector_init(void* q, void* proto_q) { lua_register(L, "collector_qout", collector_qout); // call init. - lua_getglobal(L, "init"); + lua_getglobal(L, "work"); lua_pushlightuserdata(L, q); lua_pushlightuserdata(L, proto_q); lua_pushstring(L, g_yaml_file); @@ -268,118 +267,30 @@ static lua_State * app_collector_init(void* q, void* proto_q) { if (lret < 0) { errno = -EINVAL; ret = -1; - perror("collectors.lua init failed."); + perror("collectors.lua work failed."); goto endReturn; } - sample_period = lret; - printf("setup sample period %ds\n", sample_period); - return L; + lua_close(L); + return lret; endReturn: endCall: endLoad: lua_close(L); endNew: - return NULL; -} - -static int app_collector_work(lua_State **pL, void* q, void* proto_q) { - int ret; - int err_func; - lua_Number lret; - static int counter = 0; - - lua_State *L = *pL; - - if (counter != sighup_counter) { // check counter for signal. - lua_close(L); - - L = app_collector_init(q, proto_q); - if (L == NULL) { - exit(1); - } - *pL = L; - counter = sighup_counter; - } - - err_func = lua_gettop(L); - lua_getglobal(L, "work"); - lua_pushinteger(L, sample_period); - ret = lua_pcall(L, 1, 1, err_func); - if (ret) { - lua_check_ret(ret); - goto endCall; - } - - if (!lua_isnumber(L, -1)) { // check - errno = -EINVAL; - perror("function collectors.lua work must return a number."); - goto endReturn; - } - lret = lua_tonumber(L, -1); - lua_pop(L, 1); - if (lret < 0) { - errno = -EINVAL; - ret = -1; - perror("collectors.lua work failed."); - goto endReturn; - } - - return ret; - endReturn: - endCall: - return ret; -} - -#include -#include -typedef long bee_time_t; -static bee_time_t local_time(void) { - int ret; - struct timespec tp; - - ret = clock_gettime(CLOCK_MONOTONIC, &tp); - if (ret == 0) { - return tp.tv_sec * 1000000 + tp.tv_nsec / 1000; - } else { - perror("get clock failed."); - exit(1); - return 0; - } + return -1; } int app_collector_run(struct beeQ* q, void* arg) { int ret = 0; - lua_State *L; - lua_State **pL; struct beeQ* proto_que = (struct beeQ* )arg; - L = app_collector_init(q, proto_que); - if (L == NULL) { - ret = -1; - goto endInit; - } - pL = &L; - while (1) { - bee_time_t t1, t2, delta; - t1 = local_time(); - ret = app_collector_work(pL, q, proto_que); + ret = app_collector_work(q, proto_que); if (ret < 0) { - goto endLoop; - } - t2 = local_time(); - - delta = t1 + sample_period * 1000000 - t2; - - if (delta > 0) { - usleep(delta); + perror("collect work run failed."); + break; } } - - lua_close(L); - return 0; - endLoop: - endInit: return ret; } diff --git a/source/tools/monitor/unity/beeQ/bees.c b/source/tools/monitor/unity/beeQ/bees.c index c962fe73..d28c435b 100644 --- a/source/tools/monitor/unity/beeQ/bees.c +++ b/source/tools/monitor/unity/beeQ/bees.c @@ -15,14 +15,22 @@ volatile int sighup_counter = 0; char *g_yaml_file = NULL; +static pthread_t pid_collector = 0; +static pthread_t pid_outline = 0; void sig_handler(int num) { printf("receive the signal %d.\n", num); - if (num == SIGHUP) { - sighup_counter ++; - } else { - exit(1); + switch (num) { + case SIGHUP: + sighup_counter ++; + pthread_kill(pid_collector, SIGUSR1); + pthread_kill(pid_outline, SIGUSR1); + break; + case SIGUSR1: // to stop + break; + default: + exit(1); } } @@ -36,6 +44,7 @@ int main(int argc, char *argv[]) { } signal(SIGHUP, sig_handler); + signal(SIGUSR1, sig_handler); signal(SIGINT, sig_handler); q = beeQ_init(RUN_QUEUE_SIZE, @@ -49,9 +58,15 @@ int main(int argc, char *argv[]) { if (proto_que == NULL) { exit(1); } - beeQ_send_thread(q, proto_que, app_collector_run); + pid_collector = beeQ_send_thread(q, proto_que, app_collector_run); + if (pid_collector == 0) { + exit(1); + } - outline_init(q, g_yaml_file); + pid_outline = outline_init(q, g_yaml_file); + if (pid_outline == 0) { + exit(1); + } beaver_init(g_yaml_file); fprintf(stderr, "loop exit."); diff --git a/source/tools/monitor/unity/beeQ/collectors.lua b/source/tools/monitor/unity/beeQ/collectors.lua index 192d8e4f..6ce95d05 100644 --- a/source/tools/monitor/unity/beeQ/collectors.lua +++ b/source/tools/monitor/unity/beeQ/collectors.lua @@ -7,8 +7,7 @@ package.path = package.path .. ";../?.lua;" local Cloop = require("collector.loop") local system = require("common.system") - -workLoop = nil +local ptime = require("posix.time") local function setupFreq(fYaml) local conf = system:parseYaml(fYaml) @@ -25,14 +24,32 @@ local function setupFreq(fYaml) end end -function init(que, proto_q, yaml) - local fYaml = yaml or "../collector/plugin.yaml" - local work = Cloop.new(que, proto_q, fYaml) - workLoop = work - return setupFreq(fYaml) +local function calcSleep(hope, now) + if hope.tv_nsec >= now.tv_nsec then + return {tv_sec = hope.tv_sec - now.tv_sec, + tv_nsec = hope.tv_nsec - now.tv_nsec} + else + return {tv_sec = hope.tv_sec - now.tv_sec - 1, + tv_nsec = 1e9 + hope.tv_nsec - now.tv_nsec} + end end -function work(t) - workLoop:work(t) - return 0 +function work(que, proto_q, yaml) + local fYaml = yaml or "../collector/plugin.yaml" + local w = Cloop.new(que, proto_q, fYaml) + local unit = setupFreq(fYaml) + local tStart = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) + while true do + local now = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) + local hope = {tv_sec = tStart.tv_sec + unit, tv_nsec = tStart.tv_sec} + local diff = calcSleep(hope, now) + assert(diff.tv_sec >= 0) + w:work(unit) + local _, s, errno, _ = ptime.nanosleep(diff) + if errno then -- interrupt by signal + print(string.format("new sleep stop. %d, %s", errno, s)) + return 0 + end + tStart = hope + end end diff --git a/source/tools/monitor/unity/beeQ/lib/beeQ.c b/source/tools/monitor/unity/beeQ/lib/beeQ.c index 74e91a6b..a60afa85 100644 --- a/source/tools/monitor/unity/beeQ/lib/beeQ.c +++ b/source/tools/monitor/unity/beeQ/lib/beeQ.c @@ -298,5 +298,5 @@ pthread_t beeQ_send_thread(struct beeQ *q, void *sarg, int (*cb)(struct beeQ *q, failThread: free(msg); failMalloc: - return res; + return 0; } diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index 28a5421b..7c5e284f 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -23,7 +23,7 @@ void plugin_stop(void) { void plugin_thread_stop(pthread_t tid) { if (tid > 0) { printf("send sig stop to thread %lu\n", tid); - pthread_kill(tid, SIGUSR1); + pthread_kill(tid, SIGUSR2); pthread_join(tid, NULL); } } @@ -38,7 +38,7 @@ static void sig_register(void) { action.sa_handler = stop_signal_handler; sigemptyset(&action.sa_mask); action.sa_flags = 0; - sigaction(SIGUSR1, &action, NULL); + sigaction(SIGUSR2, &action, NULL); } static void bump_memlock_rlimit1(void) diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 0dd6d454..11c22d1a 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -135,8 +135,8 @@ static int outline_run(struct beeQ* q, void* arg) { return ret; } -int outline_init(struct beeQ* pushQ, char *fYaml) { - pthread_t tid; +pthread_t outline_init(struct beeQ* pushQ, char *fYaml) { + pthread_t tid = 0; tid = beeQ_send_thread(pushQ, fYaml, outline_run); return tid; diff --git a/source/tools/monitor/unity/collector/outline/outline.h b/source/tools/monitor/unity/collector/outline/outline.h index 4f82428b..2bbe30ea 100644 --- a/source/tools/monitor/unity/collector/outline/outline.h +++ b/source/tools/monitor/unity/collector/outline/outline.h @@ -8,6 +8,6 @@ #include "../../beeQ/beeQ.h" #include #include -int outline_init(struct beeQ* pushQ, char *fYaml); +pthread_t outline_init(struct beeQ* pushQ, char *fYaml); #endif //UNITY_OUTLINE_H diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index 002d8bf0..c43794e3 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -154,7 +154,6 @@ function CfoxTSDB:rotateDb() print("delete " .. "./" .. f) pcall(unistd.unlink, "./" .. f) end - --pcall(unistd.unlink, "../" .. f) end end end -- Gitee From 74359e42102acfaa08ab68f1c06e3b7e296bcff6 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 6 Mar 2023 00:59:14 +0800 Subject: [PATCH 53/77] use full plugin. --- .../monitor/unity/beaver/native/Makefile | 2 +- source/tools/monitor/unity/beeQ/Makefile | 4 +- source/tools/monitor/unity/beeQ/apps.c | 4 +- source/tools/monitor/unity/beeQ/bees.c | 1 + .../tools/monitor/unity/beeQ/collectors.lua | 89 +++++++++++++++++- .../unity/collector/interface/Makefile | 19 ++++ .../{native => interface}/fastKsym.c | 0 .../{native => interface}/fastKsym.h | 0 .../{native => interface}/sig_stop.c | 2 +- .../{native => interface}/sig_stop.h | 0 .../{native => interface}/unity_interface.c | 2 +- .../{native => interface}/unity_interface.h | 2 + .../monitor/unity/collector/native/Makefile | 2 +- .../unity/collector/native/ffi_unity_api.c | 27 ++++++ .../unity/collector/native/ffi_unity_api.h | 14 +++ .../unity/collector/native/procffi.lua | 12 +-- .../monitor/unity/collector/outline/outline.c | 2 +- .../tools/monitor/unity/collector/plugin.lua | 10 +- .../unity/collector/plugin/plugin_head.h | 6 +- .../unity/collector/plugin/proto_sender.c | 2 +- source/tools/monitor/unity/test/bees/run.sh | 18 +++- .../tools/monitor/unity/test/posix/copySo.lua | 91 +++++++++++++++++++ .../monitor/unity/test/posix/src/libtest.so | 1 + .../monitor/unity/test/posix/src/libtest.so.2 | 0 .../monitor/unity/test/posix/src/libtest.txt | 0 25 files changed, 278 insertions(+), 32 deletions(-) create mode 100644 source/tools/monitor/unity/collector/interface/Makefile rename source/tools/monitor/unity/collector/{native => interface}/fastKsym.c (100%) rename source/tools/monitor/unity/collector/{native => interface}/fastKsym.h (100%) rename source/tools/monitor/unity/collector/{native => interface}/sig_stop.c (95%) rename source/tools/monitor/unity/collector/{native => interface}/sig_stop.h (100%) rename source/tools/monitor/unity/collector/{native => interface}/unity_interface.c (99%) rename source/tools/monitor/unity/collector/{native => interface}/unity_interface.h (72%) create mode 100644 source/tools/monitor/unity/collector/native/ffi_unity_api.c create mode 100644 source/tools/monitor/unity/collector/native/ffi_unity_api.h create mode 100644 source/tools/monitor/unity/test/posix/copySo.lua create mode 100644 source/tools/monitor/unity/test/posix/src/libtest.so create mode 100644 source/tools/monitor/unity/test/posix/src/libtest.so.2 create mode 100644 source/tools/monitor/unity/test/posix/src/libtest.txt diff --git a/source/tools/monitor/unity/beaver/native/Makefile b/source/tools/monitor/unity/beaver/native/Makefile index 583c418a..48c42266 100644 --- a/source/tools/monitor/unity/beaver/native/Makefile +++ b/source/tools/monitor/unity/beaver/native/Makefile @@ -13,7 +13,7 @@ $(SO): $(OBJS) $(CC) -o $@ $(OBJS) $(LDFLAG) install: $(SO) - cp $(SO) ../../collector/native/ + cp $(SO) ../../beeQ/lib clean: rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/beeQ/Makefile b/source/tools/monitor/unity/beeQ/Makefile index 1ad7ee4d..693514e9 100644 --- a/source/tools/monitor/unity/beeQ/Makefile +++ b/source/tools/monitor/unity/beeQ/Makefile @@ -2,11 +2,11 @@ LIB= -lpthread -ldl CC=gcc CFLAG := -g -I../beaver -I../collector/outline -LDFLAG := -g -lm -ldl -lrt -lpthread -lluajit-5.1 -L./lib/ -lbeeQ -L../beaver -lbeaver -L../collector/outline/ -loutline -L../collector/plugin/ -lproto_sender -L../collector/native/ -lprocffi +LDFLAG := -g -lm -ldl -lrt -lpthread -lluajit-5.1 -L./lib/ -lbeeQ -L../beaver -lbeaver -lcollectorApi -L../collector/outline/ -loutline -L../collector/plugin/ -lproto_sender PRG=unity-mon OBJ=apps.o bees.o -DEPMOD=lib ../beaver ../collector/native ../collector/outline ../collector/plugin ../tsdb/native +DEPMOD=lib ../beaver ../collector/native ../collector/interface ../collector/outline ../collector/plugin ../tsdb/native $(PRG): $(DEPMOD) $(OBJ) $(CC) $(LIB) -o $@ $(OBJ) $(LDFLAG) diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index 7bcebd76..e1ea2932 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -108,7 +108,7 @@ static lua_State * app_recv_init(void) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); - ret = lua_load_do_file(L, "bees.lua"); + ret = lua_load_do_file(L, "../beeQ/bees.lua"); if (ret) { goto endLoad; } @@ -239,7 +239,7 @@ static int app_collector_work(void* q, void* proto_q) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); - ret = lua_load_do_file(L, "collectors.lua"); + ret = lua_load_do_file(L, "../beeQ/collectors.lua"); if (ret) { goto endLoad; } diff --git a/source/tools/monitor/unity/beeQ/bees.c b/source/tools/monitor/unity/beeQ/bees.c index d28c435b..f6174e49 100644 --- a/source/tools/monitor/unity/beeQ/bees.c +++ b/source/tools/monitor/unity/beeQ/bees.c @@ -30,6 +30,7 @@ void sig_handler(int num) case SIGUSR1: // to stop break; default: + printf("signal %d exit.\n", num); exit(1); } } diff --git a/source/tools/monitor/unity/beeQ/collectors.lua b/source/tools/monitor/unity/beeQ/collectors.lua index 6ce95d05..e92a06f5 100644 --- a/source/tools/monitor/unity/beeQ/collectors.lua +++ b/source/tools/monitor/unity/beeQ/collectors.lua @@ -5,10 +5,93 @@ --- package.path = package.path .. ";../?.lua;" -local Cloop = require("collector.loop") +local dirent = require("posix.dirent") +local unistd = require("posix.unistd") +local stat = require("posix.sys.stat") +local bit = require("bit") local system = require("common.system") local ptime = require("posix.time") +local srcPath = "../collector/native/" +local dstPath = "../collector/lib/" + +local function listSrc(path) + local res = {} + local files = dirent.files(path) + for f in files do + if string.find(f, "%.so") then + table.insert(res, f) + end + end + return res +end + +local function checkDst() + if unistd.access(dstPath) then + local pstat = stat.stat(dstPath) + if stat.S_ISDIR(pstat.st_mode) == 0 then + error(string.format("dst %s is no a dictionary", dstPath)) + end + else + print("mkdir " .. dstPath) + local _, s, errno = stat.mkdir(dstPath) + if errno then + error(string.format("mkdir %s failed ,report %s. %d", dstPath), s, errno) + end + end +end + +local function checkSo(fPath) + local fSrc = srcPath .. fPath + local fDst = dstPath .. fPath + + if unistd.access(fDst) then + local sSrc = stat.stat(fSrc) + local sDst = stat.stat(fDst) + + if sSrc.st_mtime > sDst.st_mtime then -- modified + return true + else + return false + end + else -- exit + return true + end +end + +local function copySo(fPath) + local fSrc = srcPath .. fPath + local fDst = dstPath .. fPath + + local sFile, err = io.open(fSrc,"rb") + if err then + error(string.format("open file %s report %s."), fSrc, err) + end + local stream = sFile:read("*a") + sFile:close() + + local dFile, err = io.open(fDst,"wb") + if err then + error(string.format("open file %s report %s."), fDst, err) + end + dFile:write(stream) + dFile:close() + + stat.chmod(fDst, bit.bor(stat.S_IRWXU, stat.S_IRGRP, stat.S_IROTH)) +end + +local function checkSos() + print(unistd.getcwd()) + checkDst() + local so_s = listSrc(srcPath) + for _, so in ipairs(so_s) do + if checkSo(so) then + print("need copy " .. so) + copySo(so) + end + end +end + local function setupFreq(fYaml) local conf = system:parseYaml(fYaml) if conf then @@ -36,15 +119,17 @@ end function work(que, proto_q, yaml) local fYaml = yaml or "../collector/plugin.yaml" + checkSos() + local Cloop = require("collector.loop") local w = Cloop.new(que, proto_q, fYaml) local unit = setupFreq(fYaml) local tStart = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) while true do + w:work(unit) local now = ptime.clock_gettime(ptime.CLOCK_MONOTONIC) local hope = {tv_sec = tStart.tv_sec + unit, tv_nsec = tStart.tv_sec} local diff = calcSleep(hope, now) assert(diff.tv_sec >= 0) - w:work(unit) local _, s, errno, _ = ptime.nanosleep(diff) if errno then -- interrupt by signal print(string.format("new sleep stop. %d, %s", errno, s)) diff --git a/source/tools/monitor/unity/collector/interface/Makefile b/source/tools/monitor/unity/collector/interface/Makefile new file mode 100644 index 00000000..4b0dcf38 --- /dev/null +++ b/source/tools/monitor/unity/collector/interface/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := unity_interface.o sig_stop.o fastKsym.o +SO := libcollectorApi.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../beeQ/lib + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/native/fastKsym.c b/source/tools/monitor/unity/collector/interface/fastKsym.c similarity index 100% rename from source/tools/monitor/unity/collector/native/fastKsym.c rename to source/tools/monitor/unity/collector/interface/fastKsym.c diff --git a/source/tools/monitor/unity/collector/native/fastKsym.h b/source/tools/monitor/unity/collector/interface/fastKsym.h similarity index 100% rename from source/tools/monitor/unity/collector/native/fastKsym.h rename to source/tools/monitor/unity/collector/interface/fastKsym.h diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/interface/sig_stop.c similarity index 95% rename from source/tools/monitor/unity/collector/native/sig_stop.c rename to source/tools/monitor/unity/collector/interface/sig_stop.c index 7c5e284f..769e743f 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/interface/sig_stop.c @@ -22,7 +22,7 @@ void plugin_stop(void) { void plugin_thread_stop(pthread_t tid) { if (tid > 0) { - printf("send sig stop to thread %lu\n", tid); + printf("send sig user2 to thread %lu\n", tid); pthread_kill(tid, SIGUSR2); pthread_join(tid, NULL); } diff --git a/source/tools/monitor/unity/collector/native/sig_stop.h b/source/tools/monitor/unity/collector/interface/sig_stop.h similarity index 100% rename from source/tools/monitor/unity/collector/native/sig_stop.h rename to source/tools/monitor/unity/collector/interface/sig_stop.h diff --git a/source/tools/monitor/unity/collector/native/unity_interface.c b/source/tools/monitor/unity/collector/interface/unity_interface.c similarity index 99% rename from source/tools/monitor/unity/collector/native/unity_interface.c rename to source/tools/monitor/unity/collector/interface/unity_interface.c index 0c6d3139..41876791 100644 --- a/source/tools/monitor/unity/collector/native/unity_interface.c +++ b/source/tools/monitor/unity/collector/interface/unity_interface.c @@ -36,4 +36,4 @@ char *get_unity_proc(void) { char *get_unity_sys(void) { return unity_sys; -} +} \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/native/unity_interface.h b/source/tools/monitor/unity/collector/interface/unity_interface.h similarity index 72% rename from source/tools/monitor/unity/collector/native/unity_interface.h rename to source/tools/monitor/unity/collector/interface/unity_interface.h index 5bb0b670..b1bd0635 100644 --- a/source/tools/monitor/unity/collector/native/unity_interface.h +++ b/source/tools/monitor/unity/collector/interface/unity_interface.h @@ -5,6 +5,8 @@ #ifndef UNITY_UNITY_INTERFACE_H #define UNITY_UNITY_INTERFACE_H +void set_unity_proc(const char *path); +void set_unity_sys(const char *path); char *get_unity_proc(void); char *get_unity_sys(void); diff --git a/source/tools/monitor/unity/collector/native/Makefile b/source/tools/monitor/unity/collector/native/Makefile index d185cf7f..b738b35e 100644 --- a/source/tools/monitor/unity/collector/native/Makefile +++ b/source/tools/monitor/unity/collector/native/Makefile @@ -1,7 +1,7 @@ CC := gcc CFLAG := -g -fpic LDFLAG := -g -fpic -shared -OBJS := procffi.o sig_stop.o unity_interface.o fastKsym.o +OBJS := procffi.o ffi_unity_api.o SO := libprocffi.so all: $(SO) diff --git a/source/tools/monitor/unity/collector/native/ffi_unity_api.c b/source/tools/monitor/unity/collector/native/ffi_unity_api.c new file mode 100644 index 00000000..35d9a69a --- /dev/null +++ b/source/tools/monitor/unity/collector/native/ffi_unity_api.c @@ -0,0 +1,27 @@ +// +// Created by 廖肇燕 on 2023/3/5. +// + +#include "ffi_unity_api.h" +#include "../interface/unity_interface.h" +#include "../interface/sig_stop.h" + +void ffi_set_unity_proc(const char *path) { + set_unity_proc(path); +} + +void ffi_set_unity_sys(const char *path) { + set_unity_sys(path); +} + +void ffi_plugin_init(void) { + plugin_init(); +} + +void ffi_plugin_stop(void) { + plugin_stop(); +} + +void ffi_plugin_deinit(void) { + plugin_deinit(); +} diff --git a/source/tools/monitor/unity/collector/native/ffi_unity_api.h b/source/tools/monitor/unity/collector/native/ffi_unity_api.h new file mode 100644 index 00000000..758d8f4b --- /dev/null +++ b/source/tools/monitor/unity/collector/native/ffi_unity_api.h @@ -0,0 +1,14 @@ +// +// Created by 廖肇燕 on 2023/3/5. +// + +#ifndef UNITY_FFI_UNITY_API_H +#define UNITY_FFI_UNITY_API_H + +void ffi_set_unity_proc(const char *path); +void ffi_set_unity_sys(const char *path); +void ffi_plugin_init(void); +void ffi_plugin_stop(void); +void ffi_plugin_deinit(void); + +#endif //UNITY_FFI_UNITY_API_H diff --git a/source/tools/monitor/unity/collector/native/procffi.lua b/source/tools/monitor/unity/collector/native/procffi.lua index d35a6a74..d7e64a3b 100644 --- a/source/tools/monitor/unity/collector/native/procffi.lua +++ b/source/tools/monitor/unity/collector/native/procffi.lua @@ -27,13 +27,11 @@ int var_input_long(const char * line, struct var_long *p); int var_input_string(const char * line, struct var_string *p); int var_input_kvs(const char * line, struct var_kvs *p); -void set_unity_proc(const char *path); -void set_unity_sys(const char *path); - -int plugin_is_working(void); -void plugin_stop(void); -void plugin_init(void); -void plugin_deinit(void); +void ffi_set_unity_proc(const char *path); +void ffi_set_unity_sys(const char *path); +void ffi_plugin_init(void); +void ffi_plugin_stop(void); +void ffi_plugin_deinit(void); ]] return {ffi = ffi, cffi=cffi} diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 11c22d1a..90ec914f 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -56,7 +56,7 @@ static lua_State * pipe_init(void* q, char *fYaml) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); - ret = lua_load_do_file(L, "outline.lua"); + ret = lua_load_do_file(L, "../beeQ/outline.lua"); if (ret) { goto endLoad; } diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 70c28589..57823c0f 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -15,27 +15,27 @@ function Cplugin:_init_(proto, procffi, que, proto_q, fYaml) self:setProcSys(procffi, res.config) self._sig_cffi = procffi["cffi"] - self._sig_cffi.plugin_init() + self._sig_cffi.ffi_plugin_init() self._ffi = require("collector.native.plugincffi") self:setup(res.plugins, proto_q) end function Cplugin:_del_() - self._sig_cffi.plugin_stop() + self._sig_cffi.ffi_plugin_stop() for _, plugin in ipairs(self._plugins) do local cffi = plugin.cffi cffi.deinit() end - self._sig_cffi.plugin_deinit() + self._sig_cffi.ffi_plugin_deinit() end function Cplugin:setProcSys(procFFI, config) local proc = config["proc_path"] or "/" local sys = config["sys_path"] or "/" - procFFI.cffi.set_unity_proc(procFFI.ffi.string(proc)) - procFFI.cffi.set_unity_sys(procFFI.ffi.string(sys)) + procFFI.cffi.ffi_set_unity_proc(procFFI.ffi.string(proc)) + procFFI.cffi.ffi_set_unity_sys(procFFI.ffi.string(sys)) end function Cplugin:setup(plugins, proto_q) diff --git a/source/tools/monitor/unity/collector/plugin/plugin_head.h b/source/tools/monitor/unity/collector/plugin/plugin_head.h index 924e2c90..84974116 100644 --- a/source/tools/monitor/unity/collector/plugin/plugin_head.h +++ b/source/tools/monitor/unity/collector/plugin/plugin_head.h @@ -42,9 +42,9 @@ struct unity_lines { #include #include #include "../../beeQ/beeQ.h" -#include "../native/sig_stop.h" -#include "../native/unity_interface.h" -#include "../native/fastKsym.h" +#include "../interface/sig_stop.h" +#include "../interface/unity_interface.h" +#include "../interface/fastKsym.h" inline struct unity_lines *unity_new_lines(void) __attribute__((always_inline)); inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); diff --git a/source/tools/monitor/unity/collector/plugin/proto_sender.c b/source/tools/monitor/unity/collector/plugin/proto_sender.c index a05211c1..552e6068 100644 --- a/source/tools/monitor/unity/collector/plugin/proto_sender.c +++ b/source/tools/monitor/unity/collector/plugin/proto_sender.c @@ -60,7 +60,7 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { luaL_openlibs(L); err_func = lua_reg_errFunc(L); - ret = lua_load_do_file(L, "proto_send.lua"); + ret = lua_load_do_file(L, "../beeQ/proto_send.lua"); if (ret) { goto endLoad; } diff --git a/source/tools/monitor/unity/test/bees/run.sh b/source/tools/monitor/unity/test/bees/run.sh index 82d67271..37055941 100755 --- a/source/tools/monitor/unity/test/bees/run.sh +++ b/source/tools/monitor/unity/test/bees/run.sh @@ -3,13 +3,21 @@ pkill unity-mon export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../collector/native/ -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../beaver/ +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../lib/ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../collector/lib/ source /etc/profile -cd ../../beeQ +cd ../../beeQ || exit 1 +[ ! -d ../lib ] && mkdir ../lib +cp ./lib/*.so ../lib +cp ../tsdb/native/*.so ../lib +rm -rf ../bin +[ ! -d ../bin ] && mkdir ../bin +cp unity-mon ../bin + +cd ../bin || exit 1 yaml_path=$1 [ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" diff --git a/source/tools/monitor/unity/test/posix/copySo.lua b/source/tools/monitor/unity/test/posix/copySo.lua new file mode 100644 index 00000000..042fcfba --- /dev/null +++ b/source/tools/monitor/unity/test/posix/copySo.lua @@ -0,0 +1,91 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/5 12:58 PM +--- + +package.path = package.path .. ";../../?.lua;" +local system = require("common.system") +local dirent = require("posix.dirent") +local unistd = require("posix.unistd") +local stat = require("posix.sys.stat") +local pystring = require("common.pystring") + +local srcPath = "src/" +local dstPath = "dst/" + +local function listSrc(path) + local res = {} + local files = dirent.files(path) + for f in files do + if string.find(f, "%.so") then + table.insert(res, f) + end + end + return res +end + +local function checkDst() + if unistd.access(dstPath) then + local pstat = stat.stat(dstPath) + if stat.S_ISDIR(pstat.st_mode) == 0 then + error(string.format("dst %s is no a dictionary", dstPath)) + end + else + print("mkdir " .. dstPath) + local _, s, errno = stat.mkdir(dstPath) + if errno then + error(string.format("mkdir %s failed ,report %s. %d", dstPath), s, errno) + end + end +end + +local function checkSo(fPath) + local fSrc = srcPath .. fPath + local fDst = dstPath .. fPath + + if unistd.access(fDst) then + local sSrc = stat.stat(fSrc) + local sDst = stat.stat(fDst) + + if sSrc.st_mtime > sDst.st_mtime then -- modified + return true + else + return false + end + else -- exit + return true + end +end + +local function copySo(fPath) + local fSrc = srcPath .. fPath + local fDst = dstPath .. fPath + + local sFile, err = io.open(fSrc,"rb") + if err then + error(string.format("open file %s report %s."), fSrc, err) + end + local stream = sFile:read("*a") + sFile:close() + + local dFile, err = io.open(fDst,"wb") + if err then + error(string.format("open file %s report %s."), fDst, err) + end + dFile:write(stream) + dFile:close() +end + +local function checkSos() + checkDst() + local so_s = listSrc(srcPath) + for _, so in ipairs(so_s) do + if checkSo(so) then + print("need copy " .. so) + copySo(so) + end + end +end + +checkSos() diff --git a/source/tools/monitor/unity/test/posix/src/libtest.so b/source/tools/monitor/unity/test/posix/src/libtest.so new file mode 100644 index 00000000..2af1bb78 --- /dev/null +++ b/source/tools/monitor/unity/test/posix/src/libtest.so @@ -0,0 +1 @@ +just a test for so. \ No newline at end of file diff --git a/source/tools/monitor/unity/test/posix/src/libtest.so.2 b/source/tools/monitor/unity/test/posix/src/libtest.so.2 new file mode 100644 index 00000000..e69de29b diff --git a/source/tools/monitor/unity/test/posix/src/libtest.txt b/source/tools/monitor/unity/test/posix/src/libtest.txt new file mode 100644 index 00000000..e69de29b -- Gitee From 2978fadc88fccef2b2c3f0addd53ee467355f76b Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 6 Mar 2023 11:35:45 +0800 Subject: [PATCH 54/77] set foxtsdb rotate day by yaml --- .../monitor/unity/collector/proc_uptime.lua | 18 +++++++++++++++++- source/tools/monitor/unity/tsdb/foxTSDB.lua | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/unity/collector/proc_uptime.lua b/source/tools/monitor/unity/collector/proc_uptime.lua index 437d13dd..b190994a 100644 --- a/source/tools/monitor/unity/collector/proc_uptime.lua +++ b/source/tools/monitor/unity/collector/proc_uptime.lua @@ -38,6 +38,21 @@ local function readNum(pFile) return res1, res2 end +local function readUname() + local distro, s, errno = utsname.uname() + if distro then + return { + {name = "sysname", index = distro.sysname}, + {name = "nodename", index = distro.nodename}, + {name = "release", index = distro.release}, + {name = "version", index = distro.version}, + {name = "machine", index = distro.machine}, + } + else + error(string.format("read uname get %s, errno %d"), s, errno) + end +end + local function readRelease(pFile) local f = io.open(pFile) local res = "unknown" @@ -61,7 +76,8 @@ function CprocUptime:proc(elapsed, lines) local totalTime = elapsed * self._counter if totalTime >= 60 * 60 then -- report by hour local dummyValue = {{name = "dummy", value=1.0}} - self:appendLine(self:_packProto("uname", self._labels, dummyValue)) + local labels = readUname() + self:appendLine(self:_packProto("uname", labels, dummyValue)) local releaseInfo = {{name = "release", index = readRelease(self._release)}} self:appendLine(self:_packProto("system_release", releaseInfo, dummyValue)) self._counter = 0 diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index c43794e3..536d1384 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -136,7 +136,7 @@ function CfoxTSDB:rotateDb() local unistd = require("posix.unistd") local usec = self._man.now - local sec = 7 * 24 * 60 * 60 + local sec = self._rotate * 24 * 60 * 60 local foxTime = self:getDateFrom_us(usec - sec * 1e6) local level = foxTime.year * 10000 + foxTime.mon * 100 + foxTime.mday -- Gitee From 03eac81dd815daaae2faf84bb13ea41a903f7e35 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 6 Mar 2023 11:38:53 +0800 Subject: [PATCH 55/77] change CbaseQuery:qTables default value. --- source/tools/monitor/unity/beaver/query/baseQuery.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/beaver/query/baseQuery.lua b/source/tools/monitor/unity/beaver/query/baseQuery.lua index db2eab3d..8fb9ebf6 100644 --- a/source/tools/monitor/unity/beaver/query/baseQuery.lua +++ b/source/tools/monitor/unity/beaver/query/baseQuery.lua @@ -100,7 +100,7 @@ end function CbaseQuery:qTables(session, fresh) fresh = fresh or false - local t = session.qlast or 4 * 6 + local t = session.qlast or 4 * 60 if session.tables == nil or fresh then session.tables = self._fox:qTabelNow(t * 60) end -- Gitee From 924db211b53e173ba634bbca6ae9bb89600b2081 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 7 Mar 2023 14:46:34 +0800 Subject: [PATCH 56/77] change guide.md, and add --- source/tools/monitor/unity/beaver/export.lua | 2 + .../monitor/unity/beaver/guide/develop.md | 3 -- .../tools/monitor/unity/beaver/guide/guide.md | 13 +++--- .../monitor/unity/beaver/guide/metrics.md | 28 +++++++++++++ source/tools/monitor/unity/beaver/index.lua | 1 + .../monitor/unity/collector/proc_arp.lua | 42 +++++++++++++++++++ 6 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/guide/metrics.md create mode 100644 source/tools/monitor/unity/collector/proc_arp.lua diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index a62a29d9..9fad5977 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -105,6 +105,8 @@ function Cexport:export() res[c] = self.pack_line(title, labels, v, tFrom.time) end end + c = c + 1 + res[c] = "" end end local lines = pystring:join("\n", res) diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 230b480f..03acc3bc 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -678,6 +678,3 @@ int deinit(void *arg) return DESTORY_SKEL_BOJECT(bpf_sample2); } ``` - - - diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index 654a28c5..f2d83799 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,7 @@ # 目录 -1. [开发手册](/guide/guide.md) -2. [插件化与热更新](/guide/hotplugin.md) -3. [面向对象设计](/guide/oop.md) -4. [字符串处理](/guide/pystring.md) -5. [页面开发](/guide/webdevel.md) -6. [proc和probe记录表](/guide/proc_probe.md) -7. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file +1. [开发手册](/guide/develop.md) +2. [proc和probe记录表](/guide/proc_probe.md) +3. [metric 指标数据说明](/guide/metrics.md) +4. [在lua 中使用pystring](/guide/pystring.md) +5. [bpf\ map 开发](/guide/bpf.md) +6. [bpf perf 开发](/guide/bpf_perf.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md new file mode 100644 index 00000000..260343cd --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -0,0 +1,28 @@ +# 指标说明 + +这里记录所有采集到的监控指标说明和来源，方便监控系统集成。 + +## 通用指标 + +------------- + +### uptime 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| uptime | 秒 | 从系统启动到现在的时间 | | collector/proc_uptime.lua | +| idletime | 秒 | 系统总空闲的时间 | | collector/proc_uptime.lua | +| stamp | 秒 | 系统时间戳 | unix 时间 | collector/proc_uptime.lua | + +### uname 表 + +每小时获取一次 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| nodename | - | uname -r | | collector/proc_uptime.lua | +| version | - | uname -r | | collector/proc_uptime.lua | +| release | - | uname -r | | collector/proc_uptime.lua | +| machine | - | uname -r | | collector/proc_uptime.lua | +| sysname | - | uname -r | | collector/proc_uptime.lua | + diff --git a/source/tools/monitor/unity/beaver/index.lua b/source/tools/monitor/unity/beaver/index.lua index b6884e0b..270ac24f 100644 --- a/source/tools/monitor/unity/beaver/index.lua +++ b/source/tools/monitor/unity/beaver/index.lua @@ -53,6 +53,7 @@ function CurlIndex:show(tReq) ### Tips  This page is rendered directly via markdown, for [guide](/guide/guide.md) + local data [query entry](/query/base) ]] local content2 = string.format("\n thread id is:%d\n", unistd.getpid()) local title = "welcome to visit SysAk Agent server." diff --git a/source/tools/monitor/unity/collector/proc_arp.lua b/source/tools/monitor/unity/collector/proc_arp.lua new file mode 100644 index 00000000..b3808cff --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_arp.lua @@ -0,0 +1,42 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/7 2:30 PM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local CprocArp = class("procStatm", CvProc) + +function CprocArp:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/net/arp") +end + +function CprocStatm:proc(elapsed, lines) + local c = 0 + CvProc.proc(self) + local arps = {} + for line in io.lines(self.pFile) do + if c > 0 then + local cells = pystring:split(line) + if arps[cells[6]] then + arps[cells[6]] = arps[cells[6]] + 1 + else + arps[cells[6]] = 1 + end + end + c = c + 1 + end + + local i = 1 + local values = {} + for k, v in pairs(arps) do + values[i] = { + name = k, + value = v, + } + end + self:appendLine(self:_packProto("arp", nil, values)) +end \ No newline at end of file -- Gitee From c711175e9972a3b501667d3e7f3c892c4e0bb69f Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 7 Mar 2023 15:52:13 +0800 Subject: [PATCH 57/77] add proc/arp --- source/tools/monitor/unity/beaver/export.lua | 4 ++-- .../tools/monitor/unity/collector/kvProc.lua | 2 +- .../monitor/unity/collector/pod_allocpage.lua | 2 +- .../monitor/unity/collector/proc_arp.lua | 23 ++++++++++++------- .../unity/collector/proc_buddyinfo.lua | 2 +- .../unity/collector/proc_diskstats.lua | 2 +- .../monitor/unity/collector/proc_meminfo.lua | 2 +- .../monitor/unity/collector/proc_mounts.lua | 2 +- .../monitor/unity/collector/proc_netdev.lua | 2 +- .../unity/collector/proc_snmp_stat.lua | 2 +- .../monitor/unity/collector/proc_sockstat.lua | 2 +- .../monitor/unity/collector/proc_stat.lua | 2 +- .../monitor/unity/collector/proc_statm.lua | 2 +- 13 files changed, 28 insertions(+), 21 deletions(-) diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index 9fad5977..5a97c6f3 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -105,10 +105,10 @@ function Cexport:export() res[c] = self.pack_line(title, labels, v, tFrom.time) end end - c = c + 1 - res[c] = "" end end + c = c + 1 + res[c] = "" local lines = pystring:join("\n", res) return lines end diff --git a/source/tools/monitor/unity/collector/kvProc.lua b/source/tools/monitor/unity/collector/kvProc.lua index 6f3bc3e4..f5ac8fac 100644 --- a/source/tools/monitor/unity/collector/kvProc.lua +++ b/source/tools/monitor/unity/collector/kvProc.lua @@ -46,7 +46,7 @@ function CkvProc:proc(elapsed, lines) self:readKV(line) end self:appendLine(self._protoTable) - return self:push(lines) + self:push(lines) end return CkvProc \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/pod_allocpage.lua b/source/tools/monitor/unity/collector/pod_allocpage.lua index 1d5f00ef..23a2f4de 100644 --- a/source/tools/monitor/unity/collector/pod_allocpage.lua +++ b/source/tools/monitor/unity/collector/pod_allocpage.lua @@ -184,7 +184,7 @@ function CPodAlloc:proc(elapsed, lines) local cell = {{name="pod_allocpage_total", value=self.total/1024}} self:appendLine(self:_packProto("pod_alloc", nil, cell)) - return self:push(lines) + self:push(lines) end return CPodAlloc diff --git a/source/tools/monitor/unity/collector/proc_arp.lua b/source/tools/monitor/unity/collector/proc_arp.lua index b3808cff..7d677f87 100644 --- a/source/tools/monitor/unity/collector/proc_arp.lua +++ b/source/tools/monitor/unity/collector/proc_arp.lua @@ -8,13 +8,13 @@ require("common.class") local pystring = require("common.pystring") local CvProc = require("collector.vproc") -local CprocArp = class("procStatm", CvProc) +local CprocArp = class("procArp", CvProc) function CprocArp:_init_(proto, pffi, mnt, pFile) CvProc._init_(self, proto, pffi, mnt, pFile or "proc/net/arp") end -function CprocStatm:proc(elapsed, lines) +function CprocArp:proc(elapsed, lines) local c = 0 CvProc.proc(self) local arps = {} @@ -30,13 +30,20 @@ function CprocStatm:proc(elapsed, lines) c = c + 1 end - local i = 1 - local values = {} for k, v in pairs(arps) do - values[i] = { - name = k, + local ls = {} + ls = { + name = "dev", + index = k, + } + local vs = {} + vs = { + name = "count", value = v, } + self:appendLine(self:_packProto("arp", {ls}, {vs})) end - self:appendLine(self:_packProto("arp", nil, values)) -end \ No newline at end of file + self:push(lines) +end + +return CprocArp \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index 9521ac33..e324c0d5 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -51,7 +51,7 @@ function CprocBuddyinfo:proc(elapsed, lines) end self:appendLine(self._protoTable) - return self:push(lines) + self:push(lines) end return CprocBuddyinfo diff --git a/source/tools/monitor/unity/collector/proc_diskstats.lua b/source/tools/monitor/unity/collector/proc_diskstats.lua index 45badf6b..a8535414 100644 --- a/source/tools/monitor/unity/collector/proc_diskstats.lua +++ b/source/tools/monitor/unity/collector/proc_diskstats.lua @@ -127,7 +127,7 @@ function CprocDiskstats:proc(elapsed, lines) self:_proc(line, elapsed) end self:checkLastDisks() - return self:push(lines) + self:push(lines) end return CprocDiskstats diff --git a/source/tools/monitor/unity/collector/proc_meminfo.lua b/source/tools/monitor/unity/collector/proc_meminfo.lua index fa306d55..2902cd84 100644 --- a/source/tools/monitor/unity/collector/proc_meminfo.lua +++ b/source/tools/monitor/unity/collector/proc_meminfo.lua @@ -156,7 +156,7 @@ function CprocMeminfo:proc(elapsed, lines) table.insert(self._protoTable["vs"], cell) self:appendLine(self._protoTable) - return self:push(lines) + self:push(lines) end diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 1e5d0723..2134f8a6 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -112,7 +112,7 @@ function CprocMounts:proc(elapsed, lines) CvProc.proc(self) self:_proc() - return self:push(lines) + self:push(lines) end return CprocMounts \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/proc_netdev.lua b/source/tools/monitor/unity/collector/proc_netdev.lua index 2682bfdb..d546ad00 100644 --- a/source/tools/monitor/unity/collector/proc_netdev.lua +++ b/source/tools/monitor/unity/collector/proc_netdev.lua @@ -91,7 +91,7 @@ function CprocNetdev:proc(elapsed, lines) i = i + 1 end self:checkLastIfNames() - return self:push(lines) + self:push(lines) end return CprocNetdev diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index 917cd7d4..a9e79e10 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -109,7 +109,7 @@ function CprocSnmpStat:proc(elapsed, lines) self:_proc(self.pFile .. "proc/net/snmp", now) self:_proc(self.pFile .. "proc/net/netstat", now) self:check(now) - return self:push(lines) + self:push(lines) end return CprocSnmpStat diff --git a/source/tools/monitor/unity/collector/proc_sockstat.lua b/source/tools/monitor/unity/collector/proc_sockstat.lua index bf7eb80d..3a076947 100644 --- a/source/tools/monitor/unity/collector/proc_sockstat.lua +++ b/source/tools/monitor/unity/collector/proc_sockstat.lua @@ -38,7 +38,7 @@ function CprocSockStat:proc(elapsed, lines) end end self:appendLine(self:_packProto("sock_stat", nil, vs)) - return self:push(lines) + self:push(lines) end return CprocSockStat diff --git a/source/tools/monitor/unity/collector/proc_stat.lua b/source/tools/monitor/unity/collector/proc_stat.lua index a7096896..c165a282 100644 --- a/source/tools/monitor/unity/collector/proc_stat.lua +++ b/source/tools/monitor/unity/collector/proc_stat.lua @@ -185,7 +185,7 @@ function CprocStat:proc(elapsed, lines) end end self:appendLine(self:_packProto("stat_counters", nil, counter, nil)) - return self:push(lines) + self:push(lines) end return CprocStat diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index 9a9fff79..f1ad61e3 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -33,7 +33,7 @@ function CprocStatm:proc(elapsed, lines) self:appendLine(self:_packProto("self_statm", nil, vs)) end - return self:push(lines) + self:push(lines) end return CprocStatm -- Gitee From 45c9fad25728cab44551e3289e5d2c091be086e8 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Tue, 7 Mar 2023 16:47:15 +0800 Subject: [PATCH 58/77] proc_loadavg: don't expand 100 times for value Signed-off-by: Hailong Liu --- .../unity/collector/plugin/proc_loadavg/proc_loadavg.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index df4bbc2a..e113f55f 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -60,9 +60,9 @@ int full_line(struct unity_line *uline) st_load.nr_running--; } //unity_set_index(uline1[cpu], 0, "load", cpu_name); - unity_set_value(uline, 0, "load1", st_load.load_avg_1*100); - unity_set_value(uline, 1, "load5", st_load.load_avg_5*100); - unity_set_value(uline, 2, "load15", st_load.load_avg_15*100); + unity_set_value(uline, 0, "load1", st_load.load_avg_1); + unity_set_value(uline, 1, "load5", st_load.load_avg_5); + unity_set_value(uline, 2, "load15", st_load.load_avg_15); unity_set_value(uline, 3, "runq", st_load.nr_running); unity_set_value(uline, 4, "plit", st_load.nr_threads); -- Gitee From 017bedff66ae4104a07d93aca91292d182fcb77e Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 8 Mar 2023 08:23:17 +0800 Subject: [PATCH 59/77] add softirqs. --- .../monitor/unity/beaver/query/baseQuery.lua | 1 + .../monitor/unity/collector/proc_arp.lua | 6 +- .../monitor/unity/collector/proc_softirqs.lua | 58 +++++++++++++++++++ .../unity/collector/proc_softnet_stat.lua | 53 +++++++++++++++++ source/tools/monitor/unity/test/fox/query.py | 6 +- 5 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 source/tools/monitor/unity/collector/proc_softirqs.lua create mode 100644 source/tools/monitor/unity/collector/proc_softnet_stat.lua diff --git a/source/tools/monitor/unity/beaver/query/baseQuery.lua b/source/tools/monitor/unity/beaver/query/baseQuery.lua index 8fb9ebf6..109f67c1 100644 --- a/source/tools/monitor/unity/beaver/query/baseQuery.lua +++ b/source/tools/monitor/unity/beaver/query/baseQuery.lua @@ -128,6 +128,7 @@ local function escape(s) end return "None" end + local function packDataHead(res, labels, values, logs) local heads = system:listMerge({"time"}, labels, values, logs) local show_head = {} diff --git a/source/tools/monitor/unity/collector/proc_arp.lua b/source/tools/monitor/unity/collector/proc_arp.lua index 7d677f87..d38c6d32 100644 --- a/source/tools/monitor/unity/collector/proc_arp.lua +++ b/source/tools/monitor/unity/collector/proc_arp.lua @@ -31,13 +31,11 @@ function CprocArp:proc(elapsed, lines) end for k, v in pairs(arps) do - local ls = {} - ls = { + local ls = { name = "dev", index = k, } - local vs = {} - vs = { + local vs = { name = "count", value = v, } diff --git a/source/tools/monitor/unity/collector/proc_softirqs.lua b/source/tools/monitor/unity/collector/proc_softirqs.lua new file mode 100644 index 00000000..d8b1f489 --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_softirqs.lua @@ -0,0 +1,58 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/7 10:08 PM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local Csoftirqs = class("softirqs", CvProc) + +function Csoftirqs:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/softirqs") +end + +function Csoftirqs:proc(elapsed, lines) + local c = 0 + CvProc.proc(self) + local sirqs = {} + for line in io.lines(self.pFile) do + if c > 0 then + local title, irq_str = unpack(pystring:split(line, ":", 1)) + title = string.lower(pystring:strip(title)) + sirqs[title] = pystring:split(pystring:strip(irq_str)) + end + c = c + 1 + end + + local per_sirqs = {} + for k, irqs in pairs(sirqs) do + for i, v in ipairs(irqs) do + if per_sirqs[i] == nil then + per_sirqs[i] = {} + end + per_sirqs[i][k] = tonumber(v) + end + end + + for i, irq in ipairs(per_sirqs) do + local ls = { + name = "cpu", index = "cpu" .. (i - 1) + } + local c = 1 + local values = {} + for k, v in pairs(irq) do + values[c] = { + name = k, + value = v, + } + c = c + 1 + end + self:appendLine(self:_packProto("per_sirqs", {ls}, values)) + end + self:push(lines) +end + +return Csoftirqs diff --git a/source/tools/monitor/unity/collector/proc_softnet_stat.lua b/source/tools/monitor/unity/collector/proc_softnet_stat.lua new file mode 100644 index 00000000..6aa952ea --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_softnet_stat.lua @@ -0,0 +1,53 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/7 8:43 PM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local CsoftnetStat = class("softnetStat", CvProc) + +-- refer to https://insights-core.readthedocs.io/en/latest/shared_parsers_catalog/softnet_stat.html +function CsoftnetStat:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/net/softnet_stat") +end + +function CsoftnetStat:proc(elapsed, lines) + local c = 0 + CvProc.proc(self) + local softnets = {} + for line in io.lines(self.pFile) do + local cell = {} + local cells = pystring:split(line) + cell.packet_process = tonumber(cells[1], 16) + cell.packet_drop = tonumber(cells[2], 16) + cell.time_squeeze = tonumber(cells[3], 16) + cell.cpu_collision = tonumber(cells[9], 16) + cell.received_rps = tonumber(cells[10], 16) + cell.flow_limit_count = tonumber(cells[11], 16) + c = c + 1 + softnets[c] = cell + end + + for i, cpus in ipairs(softnets) do + local ls = { + name = "cpu", index = tostring(i) + } + local c = 1 + local values = {} + for k, v in pairs(cpus) do + values[c] = { + name = k, + value = v, + } + c = c + 1 + end + self:appendLine(self:_packProto("softnets", {ls}, values)) + end + self:push(lines) +end + +return CsoftnetStat diff --git a/source/tools/monitor/unity/test/fox/query.py b/source/tools/monitor/unity/test/fox/query.py index bda9542c..41d6e556 100644 --- a/source/tools/monitor/unity/test/fox/query.py +++ b/source/tools/monitor/unity/test/fox/query.py @@ -17,7 +17,7 @@ def q_table(): def q_by_table(): - post_test({"mode": "last", "time": "100m", "table": ["IOMonDiagLog"]}) + post_test({"mode": "last", "time": "5m", "table": ["per_sirqs"]}) def q_by_date(): @@ -34,7 +34,7 @@ def q_by_date(): if __name__ == "__main__": - post_test({"mode": "last", "time": "4m"}) + # post_test({"mode": "last", "time": "4m"}) # q_table() - # q_by_table() + q_by_table() # q_by_date() -- Gitee From 26d351e7fb31b140c96a8245d309b95349a49e66 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 8 Mar 2023 14:30:46 +0800 Subject: [PATCH 60/77] add proc interrupts. --- .../unity/collector/proc_interrupts.lua | 69 +++++++++++++++++++ .../tools/monitor/unity/test/posix/nr_cpu.lua | 9 +++ .../tools/monitor/unity/test/yaml/descr.yaml | 4 +- .../tools/monitor/unity/test/yaml/pods.yaml | 10 +++ .../monitor/unity/test/yaml/systemYaml.lua | 14 ++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/proc_interrupts.lua create mode 100644 source/tools/monitor/unity/test/posix/nr_cpu.lua create mode 100644 source/tools/monitor/unity/test/yaml/pods.yaml create mode 100644 source/tools/monitor/unity/test/yaml/systemYaml.lua diff --git a/source/tools/monitor/unity/collector/proc_interrupts.lua b/source/tools/monitor/unity/collector/proc_interrupts.lua new file mode 100644 index 00000000..9b66c344 --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_interrupts.lua @@ -0,0 +1,69 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/8 11:07 AM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +local unistd = require("posix.unistd") + +local Cinterrupts = class("interrupts", CvProc) + +function Cinterrupts:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/interrupts") + self._cpus = unistd.sysconf(84) +end + +function Cinterrupts:proc(elapsed, lines) + local c = 0 + CvProc.proc(self) + local ints = {} + for line in io.lines(self.pFile) do + if c > 0 then + ints[c] = pystring:split(pystring:strip(line)) + end + c = c + 1 + end + + local per_irqs = {} + for _, int in ipairs(ints) do + local nums = #int + if nums > self._cpus then + local title = int[1] + local head + if string.match(title, "%d+:") then + head = int[nums] + else + head = string.lower(string.sub(title, 1, -2)) + end + + for i=1, self._cpus do + if not per_irqs[i] then + per_irqs[i] = {} + end + per_irqs[i][head] = tonumber(int[i + 1]) + end + end + end + + for i, irq in ipairs(per_irqs) do + local ls = { + name = "cpu", index = "cpu" .. (i - 1) + } + local c = 1 + local values = {} + for k, v in pairs(irq) do + values[c] = { + name = k, + value = v, + } + c = c + 1 + end + self:appendLine(self:_packProto("interrupts", {ls}, values)) + end + self:push(lines) +end + +return Cinterrupts diff --git a/source/tools/monitor/unity/test/posix/nr_cpu.lua b/source/tools/monitor/unity/test/posix/nr_cpu.lua new file mode 100644 index 00000000..682a6415 --- /dev/null +++ b/source/tools/monitor/unity/test/posix/nr_cpu.lua @@ -0,0 +1,9 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/8 12:48 PM +--- + +local unistd = require("posix.unistd") + +print(unistd.sysconf(84)) \ No newline at end of file diff --git a/source/tools/monitor/unity/test/yaml/descr.yaml b/source/tools/monitor/unity/test/yaml/descr.yaml index ec4e3ec5..619a4a08 100644 --- a/source/tools/monitor/unity/test/yaml/descr.yaml +++ b/source/tools/monitor/unity/test/yaml/descr.yaml @@ -11,4 +11,6 @@ metrics: index: [] field: [user, nice, sys, idle, iowait, hardirq, softirq, steal, guest, guestnice] help: "cpu usage info for per-cpu." - type: "gauge" \ No newline at end of file + type: "gauge" +config: + path: pods.yaml \ No newline at end of file diff --git a/source/tools/monitor/unity/test/yaml/pods.yaml b/source/tools/monitor/unity/test/yaml/pods.yaml new file mode 100644 index 00000000..795e330b --- /dev/null +++ b/source/tools/monitor/unity/test/yaml/pods.yaml @@ -0,0 +1,10 @@ +pods: + - hello + - wolrd + - pod2 + - pod1 + +confs: + - path: /sys/cgroup + depth: 3 + - path: \ No newline at end of file diff --git a/source/tools/monitor/unity/test/yaml/systemYaml.lua b/source/tools/monitor/unity/test/yaml/systemYaml.lua new file mode 100644 index 00000000..3c7410b2 --- /dev/null +++ b/source/tools/monitor/unity/test/yaml/systemYaml.lua @@ -0,0 +1,14 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/8 11:19 AM +--- + +package.path = package.path .. ";../../?.lua;" + +local system = require("common.system") + +local res = system:parseYaml("descr.yaml") +local path = res.config.path +local pods = system:parseYaml(path) +print(system:dump(pods.pods)) -- Gitee From e0d068f5204762d6e8978895a8fbcf890c54fcf5 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 8 Mar 2023 15:33:04 +0800 Subject: [PATCH 61/77] add proc cgroups. --- .../monitor/unity/collector/proc_cgroups.lua | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 source/tools/monitor/unity/collector/proc_cgroups.lua diff --git a/source/tools/monitor/unity/collector/proc_cgroups.lua b/source/tools/monitor/unity/collector/proc_cgroups.lua new file mode 100644 index 00000000..7c95636b --- /dev/null +++ b/source/tools/monitor/unity/collector/proc_cgroups.lua @@ -0,0 +1,40 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/8 2:32 PM +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local Ccgroups = class("cgroups", CvProc) + +function Ccgroups:_init_(proto, pffi, mnt, pFile) + CvProc._init_(self, proto, pffi, mnt, pFile or "proc/cgroups") +end + +function Ccgroups:proc(elapsed, lines) + local c = 0 + CvProc.proc(self) + local values = {} + local ls = { + name = "type", + index = "num_cgroups", + } + + for line in io.lines(self.pFile) do + if c > 0 then + local cell = pystring:split(line) + values[c - 1] = { + name = cell[1], + value = tonumber(cell[3]) + } + end + c = c + 1 + end + self:appendLine(self:_packProto("cgroups", {ls}, values)) + self:push(lines) +end + +return Ccgroups -- Gitee From 0b39d6ac13891f1b43034a8d29cf6758cffbfc91 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 8 Mar 2023 09:08:35 +0000 Subject: [PATCH 62/77] unity/jitter: change the way of expression Signed-off-by: Hailong Liu --- .../plugin/unity_irqoff/unity_irqoff.c | 10 ++++------ .../plugin/unity_nosched/unity_nosched.c | 18 ++++++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c index 0f92e59d..fd443057 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c @@ -216,12 +216,10 @@ int call(int t, struct unity_lines *lines) unity_set_index(line, 0, "mod", "irqoff"); unity_set_value(line, 0, "dltnum", delta(summary, num)); unity_set_value(line, 1, "dlttm", delta(summary, total)); - unity_set_value(line, 2, "lt10ms", delta(summary, less10ms)); - unity_set_value(line, 3, "lt50ms", delta(summary, less50ms)); - unity_set_value(line, 4, "lt100ms", delta(summary, less100ms)); - unity_set_value(line, 5, "lt500ms", delta(summary, less500ms)); - unity_set_value(line, 6, "lt1s", delta(summary, less1s)); - unity_set_value(line, 7, "mts", delta(summary, plus1s)); + unity_set_value(line, 2, "gt50ms", delta(summary, less100ms)); + unity_set_value(line, 3, "gt100ms", delta(summary, less500ms)); + unity_set_value(line, 4, "gt500ms", delta(summary, less1s)); + unity_set_value(line, 5, "gt1s", delta(summary, plus1s)); prev = summary; return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c index 77e7f7f4..e8915d72 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -30,13 +30,13 @@ static void update_summary(struct sched_jit_summary* summary, const struct event } else if (e->delay < 50) { summary->less50ms++; } else if (e->delay < 100) { - summary->less100ms++; + summary->less100ms++; /* gt50 */ } else if (e->delay < 500) { - summary->less500ms++; + summary->less500ms++; /* gt100 */ } else if (e->delay < 1000) { - summary->less1s++; + summary->less1s++; /* gt500 */ } else { - summary->plus1s++; + summary->plus1s++; /* gt1s */ } } @@ -141,12 +141,10 @@ int call(int t, struct unity_lines *lines) unity_set_index(line, 0, "mod", "noschd"); unity_set_value(line, 0, "dltnum", delta(summary,num)); unity_set_value(line, 1, "dlttm", delta(summary,total)); - unity_set_value(line, 2, "lt10ms", delta(summary,less10ms)); - unity_set_value(line, 3, "lt50ms", delta(summary,less50ms)); - unity_set_value(line, 4, "lt100ms", delta(summary,less100ms)); - unity_set_value(line, 5, "lt500ms", delta(summary,less500ms)); - unity_set_value(line, 6, "lt1s", delta(summary,less1s)); - unity_set_value(line, 7, "mts", delta(summary,plus1s)); + unity_set_value(line, 2, "gt50ms", delta(summary,less100ms)); + unity_set_value(line, 3, "gt100ms", delta(summary,less500ms)); + unity_set_value(line, 4, "gt500ms", delta(summary,less1s)); + unity_set_value(line, 5, "gt1s", delta(summary,plus1s)); prev = summary; return 0; } -- Gitee From f3adff1882ce38a6419739b8c93236d6382595f1 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Thu, 9 Mar 2023 01:50:30 +0000 Subject: [PATCH 63/77] proc_schedstat: Take and output the original datas only Signed-off-by: Hailong Liu --- .../plugin/proc_schedstat/proc_schedstat.c | 132 ++++-------------- 1 file changed, 31 insertions(+), 101 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index 5d1075f8..b74309c5 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -21,8 +21,7 @@ struct sched_stats { long nr_cpus; char *real_proc_path; struct unity_line **lines1; -struct sched_stats curr_min, curr_max; -struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; +struct sched_stats *schstats; int init(void * arg) { @@ -39,7 +38,7 @@ int init(void * arg) errno = 0; lines1 = NULL; - schstats = schstats2 = delta = curr = oldp = NULL; + schstats = NULL; nr_cpus = sysconf(_SC_NPROCESSORS_CONF); if (nr_cpus < 0) { ret = errno; @@ -51,49 +50,30 @@ int init(void * arg) schstats = calloc(sizeof(struct sched_stats), nr_cpus); if (!schstats) { ret = errno; + free(real_proc_path); printf("WARN: proc_schedstat install FAIL calloc 1\n"); return ret; } - schstats2 = calloc(sizeof(struct sched_stats), nr_cpus); - if (!schstats2) { - ret = errno; - printf("WARN: proc_schedstat install FAIL calloc 2\n"); - return ret; - } - delta = calloc(sizeof(struct sched_stats), nr_cpus); - if (!delta) { - ret = errno; - printf("WARN: proc_schedstat install FAIL calloc 3\n"); - return ret; - } - curr = schstats; - oldp = schstats2; lines1 = calloc(sizeof(struct unity_line *), nr_cpus); if (!lines1) { ret = errno; + free(real_proc_path); + free(schstats); printf("WARN: proc_schedstat install FAIL calloc 4\n"); return ret; } - curr_min.delay = -1ULL; - curr_max.delay = 0; printf("proc_schedstat plugin install.\n"); return 0; } -static void gen_delta(struct sched_stats *curr, struct sched_stats *old, struct sched_stats *delta) -{ - delta->pcount = curr->pcount - old->pcount; - delta->delay = curr->delay - old->delay; -} - -int full_line(struct unity_line **uline1, struct unity_line *uline2) +int full_line(struct unity_line **uline1) { int n, i, ret, idx; long cpu; FILE *fp; char line[128], cpu_name[8]; - struct sched_stats *st, *tmp; + struct sched_stats st; unsigned long long value[8] = {0}; unsigned long long sum_delay, sum_cnt; @@ -111,71 +91,38 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) while (fgets(line, LINE_LEN, fp) != NULL) { if (!strncmp(line, "cpu", 3)) { n = sscanf(line+3, "%ld", &cpu); - if (n == 1 && cpu >= 0 && cpu < nr_cpus) { - st = &curr[cpu]; - } else { + if ((n != 1) || (cpu < 0) || (cpu >= nr_cpus)) { printf("WARN:sscanf/cpu fails... n=%d,cpu=%ld, nr_cpu=%ld\n", n, cpu, nr_cpus); printf("line=[%s]\n", line+3); continue; } memset(cpu_name, 0, sizeof(cpu_name)); n = sscanf(line, "%s %llu %llu %llu %llu %llu %llu %llu %llu %llu", - cpu_name, &st->yld_count, &value[0], &value[1], &value[2], + cpu_name, &st.yld_count, &value[0], &value[1], &value[2], &value[3], &value[4], &value[5], &value[6], &value[7]); if (n == 9) { - st->sched_count = value[0]; - st->sched_goidle = value[1]; - st->ttwu_count = value[2]; - st->ttwu_local = value[3]; - st->rq_cpu_time = value[4]; - st->delay = value[5]; - st->pcount = value[6]; + st.sched_count = value[0]; + st.sched_goidle = value[1]; + st.ttwu_count = value[2]; + st.ttwu_local = value[3]; + st.rq_cpu_time = value[4]; + st.delay = value[5]; + st.pcount = value[6]; } else if (n == 10) { - st->sched_count = value[1]; - st->sched_goidle = value[2]; - st->ttwu_count = value[3]; - st->ttwu_local = value[4]; - st->rq_cpu_time = value[5]; - st->delay = value[6]; - st->pcount = value[7]; + st.sched_count = value[1]; + st.sched_goidle = value[2]; + st.ttwu_count = value[3]; + st.ttwu_local = value[4]; + st.rq_cpu_time = value[5]; + st.delay = value[6]; + st.pcount = value[7]; } - gen_delta(st, &oldp[cpu], &delta[cpu]); -#if PROC_SCH_DEBUG - printf("%s: pcount=%llu, delay=%llu\n", - cpu_name, delta[cpu].pcount, delta[cpu].delay); -#endif unity_set_index(uline1[cpu], 0, "cpu", cpu_name); - unity_set_value(uline1[cpu], 0, "pcount", delta[cpu].pcount); - unity_set_value(uline1[cpu], 1, "delay", delta[cpu].delay); - if (delta[cpu].delay > curr_max.delay) { - curr_max.delay = delta[cpu].delay; - curr_max.pcount = delta[cpu].pcount; - strcpy(curr_max.cpu_name, cpu_name); - } - if (delta[cpu].delay < curr_min.delay) { - curr_min.delay = delta[cpu].delay; - curr_min.pcount = delta[cpu].pcount; - strcpy(curr_min.cpu_name, cpu_name); - } + unity_set_value(uline1[cpu], 0, "pcount", st.pcount); + unity_set_value(uline1[cpu], 1, "delay", st.delay); } } - /* The avg of ALL cpus */ - for (i = 0; i < nr_cpus; i++) { - sum_cnt += delta[i].pcount; - sum_delay += delta[i].delay; - } -#if PROC_SCH_DEBUG - printf("avg: pcount=%llu, delay=%llu\n", - sum_cnt/nr_cpus, sum_delay/nr_cpus); -#endif - unity_set_index(uline2, 0, "summary", "avg"); - unity_set_value(uline2, 0, "pcount", sum_cnt/nr_cpus); - unity_set_value(uline2, 1, "delay", sum_delay/nr_cpus); - - tmp = curr; - curr = oldp; - oldp = tmp; if (fp) fclose(fp); } @@ -183,40 +130,23 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) int call(int t, struct unity_lines* lines) { int i = 0; static double value = 0.0; - struct unity_line *line2, *line3, *line4; - unity_alloc_lines(lines, nr_cpus+3); + unity_alloc_lines(lines, nr_cpus); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); unity_set_table(lines1[i], "proc_schedstat"); } - line2 = unity_get_line(lines, nr_cpus); - unity_set_table(line2, "proc_schedstat"); - full_line(lines1, line2); - - line3 = unity_get_line(lines, nr_cpus+1); - unity_set_table(line3, "proc_schedstat"); - unity_set_index(line3, 0, "max", curr_max.cpu_name); - unity_set_value(line3, 0, "pcount", curr_max.delay); - unity_set_value(line3, 1, "delay", curr_max.pcount); - - line4 = unity_get_line(lines, nr_cpus+2); - unity_set_table(line4, "proc_schedstat"); - unity_set_index(line4, 0, "min", curr_min.cpu_name); - unity_set_value(line4, 0, "pcount", curr_min.delay); - unity_set_value(line4, 1, "delay", curr_min.pcount); + full_line(lines1); return 0; } void deinit(void) { - if (schstats) - free(schstats); - if (schstats2) - free(schstats2); - if (delta) - free(delta); if (lines1) free(lines1); + if (schstats) + free(schstats); + if (real_proc_path) + free(real_proc_path); printf("proc_schedstat plugin uninstall\n"); } -- Gitee From c1847749fc1c490053fba3acdbfcb5cb939d65b4 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 9 Mar 2023 15:05:26 +0800 Subject: [PATCH 64/77] add net metrics. --- .../monitor/unity/beaver/guide/metrics.md | 75 +++++++++++++++++-- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index 260343cd..85a91f24 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -10,19 +10,78 @@ | 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | | :--- | ---: | :---- | :---- | :--- | -| uptime | 秒 | 从系统启动到现在的时间 | | collector/proc_uptime.lua | -| idletime | 秒 | 系统总空闲的时间 | | collector/proc_uptime.lua | -| stamp | 秒 | 系统时间戳 | unix 时间 | collector/proc_uptime.lua | +| uptime | 秒 | 从系统启动到现在的时间 | | collector/proc\_uptime.lua | +| idletime | 秒 | 系统总空闲的时间 | | collector/proc\_uptime.lua | +| stamp | 秒 | 系统时间戳 | unix 时间 | collector/proc\_uptime.lua | ### uname 表 每小时获取一次 +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +|:---------| ---: | :---- | :---- | :--- | +| nodename | - | uname -r | | collector/proc\_uptime.lua | +| version | - | uname -r | | collector/proc\_uptime.lua | +| release | - | uname -r | | collector/proc\_uptime.lua | +| machine | - | uname -r | | collector/proc\_uptime.lua | +| sysname | - | uname -r | | collector/proc\_uptime.lua | + +## 网络指标 + +----------- + +### arp + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| count | 个 | 网卡名 | 网卡上对应arp表数量 | collector/proc\_arp.lua | + +### networks + +这是网卡流量统计信息，已做差值处理 + | 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | | :--- | ---: | :---- | :---- | :--- | -| nodename | - | uname -r | | collector/proc_uptime.lua | -| version | - | uname -r | | collector/proc_uptime.lua | -| release | - | uname -r | | collector/proc_uptime.lua | -| machine | - | uname -r | | collector/proc_uptime.lua | -| sysname | - | uname -r | | collector/proc_uptime.lua | +| if\_ocompressed | 个 | network\_name 网卡名 | 发送时，设备驱动程序发送或接收的压缩数据包数 | collector/proc\_netdev.lua | +| if\_ocarrier | 个 | network\_name 网卡名 | 发送时，由于carrier错误而丢弃的数据包数 | collector/proc\_netdev.lua | +| if\_ocolls | 个 | network\_name 网卡名 | 发送时，冲突信息包的数目 | collector/proc\_netdev.lua | +| if\_ofifo | 个 | network\_name 网卡名 | 发送时，FIFO缓冲区错误的数量 | collector/proc\_netdev.lua | +| if\_obytes | Byte | network\_name 网卡名 | 发送时，数据的总字节数 | collector/proc\_netdev.lua | +| if\_odrop | 个 | network\_name 网卡名 | 发送时，设备驱动程序丢弃的数据包总数 | collector/proc\_netdev.lua | +| if\_oerrs | 个 | network\_name 网卡名 | 发送时，错误的总数 | collector/proc\_netdev.lua | +| if\_opackets | 个 | network\_name 网卡名 | 发送时，数据包总数 | collector/proc\_netdev.lua | +| if\_icompressed | 个 | network\_name 网卡名 | 接收时，设备驱动程序发送或接收的压缩数据包数 | collector/proc\_netdev.lua | +| if\_ierrs | 个 | network\_name 网卡名 | 接收时，错误的总数 | collector/proc\_netdev.lua | +| if\_ififo | 个 | network\_name 网卡名 | 接收时，FIFO缓冲区错误的数量 | collector/proc\_netdev.lua | +| if\_iframe | 个 | network\_name 网卡名 | 接收时，分组帧错误的数量 | collector/proc\_netdev.lua | +| if\_ipackets | 个 | network\_name 网卡名 | 接收时，数据包总数 | collector/proc\_netdev.lua | +| if\_idrop | 个 | network\_name 网卡名 | 接收时，设备驱动程序丢弃的数据包总数 | collector/proc\_netdev.lua | +| if\_imulticast | 个 | network\_name 网卡名 | 接收时，多播帧数 | collector/proc\_netdev.lua | +| if\_ibytes | 个 | network\_name 网卡名 | 接收时，数据字节总数 | collector/proc\_netdev.lua | +### pkt_status + +这里统计所有包状态，详细可以通过 pkt_logs 获取 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| abort | 次 | | 协议栈断言失效次数 | collector/proc\_snmp\_stat.lua | +| overflow | 次 | | 协议栈溢出次数 | collector/proc\_snmp\_stat.lua | +| err | 次 | | 协议栈错误次数 | collector/proc\_snmp\_stat.lua | +| paws | 次 | | 协议栈PAWS回绕次数 | collector/proc\_snmp\_stat.lua | +| fail | 次 | | 协议栈failure次数 | collector/proc\_snmp\_stat.lua | +| retrans | 次 | | 协议栈溢出次数 | collector/proc\_snmp\_stat.lua | +| drop | 次 | | 协议栈丢包次数 | collector/proc\_snmp\_stat.lua | + +### softnets + +This parser parses the stats from network devices. These stats includes events per cpu\(in row\), number of packets processed i.e packet_process \(first column\), number of packet drops packet\_drops \(second column\), time squeeze eg net\_rx\_action performed time_squeeze\(third column\), cpu collision eg collision occur while obtaining device lock while transmitting cpu\_collision packets \(eighth column\), received_rps number of times cpu woken up received\_rps \(ninth column\), number of times reached flow limit count flow_limit_count \(tenth column\), backlog status \(eleventh column\), core id \(twelfth column\). + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| packet\_process | 个 | cpu，对应CPU号 | 所在核收包个数 | collector/proc\_softnet\_stat.lua | +| packet\_drop | 个 | cpu，对应CPU号 | 所在核丢包个数 | collector/proc\_softnet\_stat.lua | +| cpu\_collision | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | +| received\_rps | 个 | cpu，对应CPU号 | number of times cpu woken up received_rps. | collector/proc\_softnet\_stat.lua | +| time\_squeeze | 个 | cpu，对应CPU号 | net_rx_action. | collector/proc\_softnet\_stat.lua | +| flow\_limit\_count | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | -- Gitee From 1a5bbfd41adbdb3cd146230845318196249f7ef5 Mon Sep 17 00:00:00 2001 From: Bixuan Cui Date: Thu, 9 Mar 2023 16:54:34 +0800 Subject: [PATCH 65/77] metrics: add cgroups Signed-off-by: Bixuan Cui --- .../monitor/unity/beaver/guide/metrics.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index 85a91f24..d3ecf423 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -85,3 +85,21 @@ This parser parses the stats from network devices. These stats includes events p | received\_rps | 个 | cpu，对应CPU号 | number of times cpu woken up received_rps. | collector/proc\_softnet\_stat.lua | | time\_squeeze | 个 | cpu，对应CPU号 | net_rx_action. | collector/proc\_softnet\_stat.lua | | flow\_limit\_count | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | + +### cgroups 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| type | - | subsys类型 | | collector/proc_cgroups.lua | +| blkio | 个 | blkio cgroup 数量 | | collector/proc_cgroups.lua | +| freezer | 个 | freezer cgroup数量 | | collector/proc_cgroups.lua | +| devices | 个 | devices cgroup数量 | | collector/proc_cgroups.lua | +| hugetlb | 个 | hugetlb cgroup数量 | | collector/proc_cgroups.lua | +| pids | 个 | blkio cgroup 数量 | | collector/proc_cgroups.lua | +| rdma | 个 | rdma cgroup数量 | | collector/proc_cgroups.lua | +| net_prio | 个 | net_prio cgroup数量 | | collector/proc_cgroups.lua | +| net_cls | 个 | net_cls cgroup数量 | | collector/proc_cgroups.lua | +| cpu | 个 | cpu cgroup 数量 | | collector/proc_cgroups.lua | +| cpuacct | 个 | cpuacct cgroup数量 | | collector/proc_cgroups.lua | +| perf_event | 个 | perf_event cgroup数量 | | collector/proc_cgroups.lua | +| memory | 个 | memory cgroup数量 | | collector/proc_cgroups.lua | -- Gitee From 9ea5000356d9e7c3e4bd07cfcd91bace6ba5af90 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 9 Mar 2023 23:02:33 +0800 Subject: [PATCH 66/77] fix metrics for markdown. --- source/tools/monitor/unity/beaver/guide/metrics.md | 4 ++-- source/tools/monitor/unity/collector/proc_uptime.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index 85a91f24..94d15263 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -75,7 +75,7 @@ ### softnets -This parser parses the stats from network devices. These stats includes events per cpu\(in row\), number of packets processed i.e packet_process \(first column\), number of packet drops packet\_drops \(second column\), time squeeze eg net\_rx\_action performed time_squeeze\(third column\), cpu collision eg collision occur while obtaining device lock while transmitting cpu\_collision packets \(eighth column\), received_rps number of times cpu woken up received\_rps \(ninth column\), number of times reached flow limit count flow_limit_count \(tenth column\), backlog status \(eleventh column\), core id \(twelfth column\). +This parser parses the stats from network devices. These stats includes events per cpu\(in row\), number of packets processed i.e packet_process \(first column\), number of packet drops packet\_drops \(second column\), time squeeze eg net\_rx\_action performed time_squeeze\(third column\), cpu collision eg collision occur while obtaining device lock while transmitting cpu\_collision packets \(eighth column\), received_rps number of times cpu woken up received\_rps \(ninth column\), number of times reached flow limit count flow\_limit\_count \(tenth column\), backlog status \(eleventh column\), core id \(twelfth column\). | 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | | :--- | ---: | :---- | :---- | :--- | @@ -83,5 +83,5 @@ This parser parses the stats from network devices. These stats includes events p | packet\_drop | 个 | cpu，对应CPU号 | 所在核丢包个数 | collector/proc\_softnet\_stat.lua | | cpu\_collision | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | | received\_rps | 个 | cpu，对应CPU号 | number of times cpu woken up received_rps. | collector/proc\_softnet\_stat.lua | -| time\_squeeze | 个 | cpu，对应CPU号 | net_rx_action. | collector/proc\_softnet\_stat.lua | +| time\_squeeze | 个 | cpu，对应CPU号 | net\_rx\_action. | collector/proc\_softnet\_stat.lua | | flow\_limit\_count | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | diff --git a/source/tools/monitor/unity/collector/proc_uptime.lua b/source/tools/monitor/unity/collector/proc_uptime.lua index b190994a..4d9d778b 100644 --- a/source/tools/monitor/unity/collector/proc_uptime.lua +++ b/source/tools/monitor/unity/collector/proc_uptime.lua @@ -25,7 +25,7 @@ function CprocUptime:_init_(proto, pffi, mnt, pFile) error(string.format("read uname get %s, errno %d"), s, errno) end self._release = mnt .. "etc/system-release" - self._counter = 0 + self._counter = 60 * 60 end local function readNum(pFile) @@ -74,7 +74,7 @@ function CprocUptime:proc(elapsed, lines) self:appendLine(self:_packProto("uptime", nil, vs)) local totalTime = elapsed * self._counter - if totalTime >= 60 * 60 then -- report by hour + if totalTime >= 10 * 60 then -- report by hour local dummyValue = {{name = "dummy", value=1.0}} local labels = readUname() self:appendLine(self:_packProto("uname", labels, dummyValue)) -- Gitee From bac26800968ca93f8874911a26a625602807f281 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 9 Mar 2023 23:03:04 +0800 Subject: [PATCH 67/77] delete guide code. --- source/tools/monitor/unity/guide/oop/base.lua | 22 ---------------- source/tools/monitor/unity/guide/oop/one.lua | 20 --------------- .../tools/monitor/unity/guide/oop/three.lua | 24 ------------------ source/tools/monitor/unity/guide/oop/tobj.lua | 25 ------------------- source/tools/monitor/unity/guide/oop/two.lua | 22 ---------------- 5 files changed, 113 deletions(-) delete mode 100644 source/tools/monitor/unity/guide/oop/base.lua delete mode 100644 source/tools/monitor/unity/guide/oop/one.lua delete mode 100644 source/tools/monitor/unity/guide/oop/three.lua delete mode 100644 source/tools/monitor/unity/guide/oop/tobj.lua delete mode 100644 source/tools/monitor/unity/guide/oop/two.lua diff --git a/source/tools/monitor/unity/guide/oop/base.lua b/source/tools/monitor/unity/guide/oop/base.lua deleted file mode 100644 index b118f203..00000000 --- a/source/tools/monitor/unity/guide/oop/base.lua +++ /dev/null @@ -1,22 +0,0 @@ ---- ---- Generated by EmmyLua(https://github.com/EmmyLua) ---- Created by liaozhaoyan. ---- DateTime: 2022/12/16 10:23 AM ---- -require("class") - -local Cbase = class("base") - -function Cbase:_init_(name) - self.name = name -end - -function Cbase:hello() - return self.name -end - -function Cbase:_del_() - print("base del..." .. self.name) -end - -return Cbase diff --git a/source/tools/monitor/unity/guide/oop/one.lua b/source/tools/monitor/unity/guide/oop/one.lua deleted file mode 100644 index 1e432f0c..00000000 --- a/source/tools/monitor/unity/guide/oop/one.lua +++ /dev/null @@ -1,20 +0,0 @@ ---- ---- Generated by EmmyLua(https://github.com/EmmyLua) ---- Created by liaozhaoyan. ---- DateTime: 2022/12/16 10:20 AM ---- - -require("class") -local Cbase = require("base") - -Cone = class("one", Cbase) - -function Cone:_init_(name) - Cbase._init_(self, name) -end - -function Cone:say() - print("one say " .. self.name) -end - -return Cone diff --git a/source/tools/monitor/unity/guide/oop/three.lua b/source/tools/monitor/unity/guide/oop/three.lua deleted file mode 100644 index cae5dd73..00000000 --- a/source/tools/monitor/unity/guide/oop/three.lua +++ /dev/null @@ -1,24 +0,0 @@ ---- ---- Generated by EmmyLua(https://github.com/EmmyLua) ---- Created by liaozhaoyan. ---- DateTime: 2022/12/16 1:07 PM ---- - -require("class") -local Cone = require("one") - -CThree = class("three", Cone) - -function CThree:_init_(name) - Cone._init_(self, name) - self._child = Cone.new("child") -end - - -function CThree:say() - print("three say " .. self.name) - print("child say.") - self._child:say() -end - -return CThree \ No newline at end of file diff --git a/source/tools/monitor/unity/guide/oop/tobj.lua b/source/tools/monitor/unity/guide/oop/tobj.lua deleted file mode 100644 index 356c9ece..00000000 --- a/source/tools/monitor/unity/guide/oop/tobj.lua +++ /dev/null @@ -1,25 +0,0 @@ ---- ---- Generated by EmmyLua(https://github.com/EmmyLua) ---- Created by liaozhaoyan. ---- DateTime: 2022/12/16 10:35 AM ---- -package.path = package.path .. ";../../common/?.lua;" - -local Cone = require("one") -local Ctwo = require("two") -local CThree = require("three") - -local one = Cone.new("1one") -local two = Ctwo.new("2two") -local three = CThree.new("3three") - -assert(one:hello() == "1one") -assert(two:hello() == "2two") -assert(three:hello() == "3three") - -one:say() -two:say() -three:say() - -one = nil -two:say() diff --git a/source/tools/monitor/unity/guide/oop/two.lua b/source/tools/monitor/unity/guide/oop/two.lua deleted file mode 100644 index 8c8de5b0..00000000 --- a/source/tools/monitor/unity/guide/oop/two.lua +++ /dev/null @@ -1,22 +0,0 @@ ---- ---- Generated by EmmyLua(https://github.com/EmmyLua) ---- Created by liaozhaoyan. ---- DateTime: 2022/12/16 10:33 AM ---- ---- -require("class") -local Cone = require("one") - -CTwo = class("two", Cone) - -function CTwo:_init_(name) - Cone._init_(self, name) -end - -function CTwo:say() - print("two say " .. self.name) - print("super") - Cone.say(self) -end - -return CTwo \ No newline at end of file -- Gitee From 2978eef0bbd32e4b613d80fa2cfdb86f2d0f3727 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Fri, 10 Mar 2023 08:00:42 +0800 Subject: [PATCH 68/77] add lua docker socket. --- .../tools/monitor/unity/httplib/dockerApi.lua | 489 ++++++++++++++++++ .../tools/monitor/unity/test/curl/docker.lua | 13 + 2 files changed, 502 insertions(+) create mode 100644 source/tools/monitor/unity/httplib/dockerApi.lua create mode 100644 source/tools/monitor/unity/test/curl/docker.lua diff --git a/source/tools/monitor/unity/httplib/dockerApi.lua b/source/tools/monitor/unity/httplib/dockerApi.lua new file mode 100644 index 00000000..983006a2 --- /dev/null +++ b/source/tools/monitor/unity/httplib/dockerApi.lua @@ -0,0 +1,489 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/10 12:00 AM +--- + +local client = require 'http.client' +local headers = require 'http.headers' +local util = require 'http.util' + +local cjson = require 'cjson.safe' +local basexx = require 'basexx' + +local handle_response_body = function (body) + if type(body) == 'string' then + local res = cjson.decode(body) + return res ~= nil and res or body + else + return nil + end +end + +local validate_instance = function (instance) + if type(instance) ~= 'table' then + return nil, "not a table" + end + + if instance.host == nil then + return nil, "missing host" + end + + if instance.path == nil then + return nil, "missing path" + end + + if instance.version == nil then + return nil, "missing version" + end + + return true +end + +local perform_request = function (instance, method, endpoint, query, authority, body) + local response_body + local response_headers + local err, errn + local wh_failure, wb_failure + local connection, stream + local instance_check + + instance_check, err = validate_instance(instance) + + if instance_check == nil then + return instance_check, err + end + + if endpoint == nil then + return endpoint, "endpoint not defined" + end + + if type(endpoint) ~= 'string' then + return nil, "endpoint should be a string" + end + + connection, err, errn = client.connect { + host = instance.host, + path = instance.path, + version = 1.1, + sendname = true, + port = 80, + tls = false + } + + -- error while making connection + + if connection == nil then + return connection, err, errn + end + + stream = connection:new_stream() + + -- error while creating stream + + if stream == nil then + return stream, err, errn + end + + -- prepare headers + + local h = headers.new() + + h:append(':method', method or 'GET') + + -- HTTP 1.1 seems to require this header + + h:append(':authority', '') + + h:append(':path', string.format( + '/%s%s%s', + instance.version, + endpoint, + (type(query) == 'table') and '?' .. util.dict_to_query(query) or '' + )) + + h:append('content-type', body and 'application/json' or 'text/plain') + h:append('user-agent', 'lua-docker') + + -- docker uses a custom authority header + + if authority then + local json_authority, e = cjson.encode(authority) + if json_authority == nil then + return nil, e, nil + end + local base64_encoded_authority = basexx.to_base64(json_authority) + h:append('X-Registry-Auth', base64_encoded_authority) + end + + local encoded_body, e + + if body ~= nil then + if type(body) == 'table' then + encoded_body, e = cjson.encode(body) + if encoded_body == nil then + return nil, e, nil + end + else + encoded_body = tostring(body) + end + h:append('content-length', tostring(#encoded_body)) + end + + -- write data to stream + + local end_after_headers = true + + if body then end_after_headers = false end + + wh_failure, err, errn = stream:write_headers(h, end_after_headers) + + -- error while writing headers to stream + + if wh_failure == nil then + return wh_failure, err, errn + end + + if body then + wb_failure, err, errn = stream:write_body_from_string(encoded_body) + + -- error while writing body to stream + + if wb_failure == nil then + return wb_failure, err, errn + end + end + + -- read response + + response_headers, err, errn = stream:get_headers() + + -- error getting response headers + + if response_headers == nil then + return response_headers, err, errn + end + + + response_body, err, errn = stream:get_body_as_string() + + -- error getting response body + + if response_body == nil then + return response_body, err, errn + end + + -- successfull response + + return { + body = handle_response_body(response_body), + headers = response_headers, + status = tonumber(response_headers:get(':status')) + } +end + +local loop_through_entity_endpoints = function (endpoint_data, group, target_table) + for k, v in pairs(endpoint_data) do + target_table[k] = function (self, name_or_id, query, authority, body) + return perform_request( + self, v.method, + string.format( + '/%s/%s%s', group, name_or_id, + v.endpoint and ('/' .. v.endpoint) or '' + ), + query, + authority, + body + ) + end + end +end + +-- @todo handle streaming responses +-- example: functions which return logs +-- also provide a streaming variant +-- those endpoints have a bool follow +-- query parameter set to true + +return { + new = function (host, path, version) + local d = { + host = host or 'localhost', + path = path or '/var/run/docker.sock', + version = version or 'v1.38', + + custom = perform_request, + + get_version = function (self) + return perform_request(self, 'GET', '/version') + end, + + list_containers = function (self, query) + return perform_request(self, 'GET', '/containers/json', query) + end, + + create_container = function (self, query, body) + return perform_request(self, 'POST', '/containers/create', query, nil, body) + end, + + update_container = function (self, name_or_id, body) + return perform_request( + self, 'POST', + string.format('/containers/%s/%s', name_or_id, 'update'), + nil, nil, body + ) + end, + + delete_stopped_containers = function (self, query) + return perform_request(self, 'POST', '/containers/prune', query) + end, + + -- @todo missing endpoints: + -- export_container + -- get_container_stats + -- attach_to_container + -- attach_to_container_ws + -- extract_archive_to_container_dir + + list_images = function (self, query) + return perform_request(self, 'GET', '/images/json', query) + end, + + delete_builder_cache = function (self) + return perform_request(self, 'POST', '/build/prune') + end, + + create_image = function (self, query, auth, body) + return perform_request(self, 'POST', '/images/create', query, auth, body) + end, + + search_image = function (self, query) + return perform_request(self, 'GET', '/images/search', query) + end, + + delete_unused_images = function (self, query) + return perform_request(self, 'POST', '/images/prune', query) + end, + + create_image_from_container = function (self, query, body) + return perform_request(self, 'POST', '/commit', query, nil, body) + end, + + -- @todo missing endpoints: + -- build_image + -- export_image + -- export_images + -- import_images + + list_networks = function (self, query) + return perform_request(self, 'GET', '/networks', query) + end, + + create_network = function (self, body) + return perform_request(self, 'POST', '/networks/create', nil, nil, body) + end, + + delete_unused_networks = function (self, query) + return perform_request(self, 'POST', '/networks/prune', query) + end, + + list_volumes = function (self, query) + return perform_request(self, 'GET', '/volumes', query) + end, + + create_volume = function (self, body) + return perform_request(self, 'POST', '/volumes/create', nil, nil, body) + end, + + delete_unused_volumes = function (self, query) + return perform_request(self, 'POST', '/volumes/prune', query) + end, + + inspect_swarm = function (self) + return perform_request(self, 'GET', '/swarm') + end, + + initialize_swarm = function (self, body) + return perform_request(self, 'POST', '/swarm/init', nil, nil, body) + end, + + join_swarm = function (self, body) + return perform_request(self, 'POST', '/swarm/join', nil, nil, body) + end, + + leave_swarm = function (self, query) + return perform_request(self, 'POST', '/swarm/leave', query) + end, + + update_swarm = function (self, query, body) + return perform_request(self, 'POST', '/swarm/update', query, nil, body) + end, + + get_swarm_unlockkey = function (self) + return perform_request(self, 'GET', '/swarm/unlockkey') + end, + + unlock_swarm_manager = function (self, body) + return perform_request(self, 'POST', '/swarm/unlock', nil, nil, body) + end, + + list_nodes = function (self, query) + return perform_request(self, 'GET', '/nodes', query) + end, + + list_services = function (self, query) + return perform_request(self, 'GET', '/services', query) + end, + + create_service = function (self, auth, body) + return perform_request(self, 'POST', '/services/create', nil, auth, body) + end, + + list_tasks = function (self, query) + return perform_request(self, 'GET', '/tasks', query) + end, + + list_secrets = function (self, query) + return perform_request(self, 'GET', '/secrets', query) + end, + + create_secret = function (self, body) + return perform_request(self, 'POST', '/secrets/create', nil, nil, body) + end, + + list_configs = function (self, query) + return perform_request(self, 'GET', '/configs', query) + end, + + create_config = function (self, body) + return perform_request(self, 'POST', '/configs/create', nil, nil, body) + end, + + list_plugins = function (self, query) + return perform_request(self, 'GET', '/plugins', query) + end, + + get_plugin_privileges = function (self, query) + return perform_request(self, 'GET', '/plugins/privileges', query) + end, + + install_plugin = function (self, query, auth, body) + return perform_request(self, 'POST', '/plugins/pull', query, auth, body) + end, + + create_plugin = function (self, query, body) + return perform_request(self, 'POST', '/plugins/create', query, nil, body) + end, + + check_auth_config = function (self, body) + return perform_request(self, 'POST', '/auth', nil, nil, body) + end, + + get_system_info = function (self) + return perform_request(self, 'GET', '/info') + end, + + ping_server = function (self) + return perform_request(self, 'GET', '/_ping') + end, + + -- @todo missing endpoints: + -- monitor_events + + get_usage = function (self) + return perform_request(self, 'GET', '/system/df') + end, + } + + loop_through_entity_endpoints({ + ['list_container_processes'] = { method = 'GET', endpoint = 'top' }, + ['inspect_container'] = { method = 'GET', endpoint = 'json' }, + ['get_container_logs'] = { method = 'GET', endpoint = 'logs' }, + ['get_container_fs_changes'] = { method = 'GET', endpoint = 'changes' }, + ['resize_container_tty'] = { method = 'POST', endpoint = 'resize' }, + ['start_container'] = { method = 'POST', endpoint = 'start' }, + ['stop_container'] = { method = 'POST', endpoint = 'stop' }, + ['restart_container'] = { method = 'POST', endpoint = 'restart' }, + ['kill_container'] = { method = 'POST', endpoint = 'kill' }, + ['rename_container'] = { method = 'POST', endpoint = 'rename' }, + ['pause_container'] = { method = 'POST', endpoint = 'pause' }, + ['resume_container'] = { method = 'POST', endpoint = 'unpause' }, + ['wait_for_container'] = { method = 'POST', endpoint = 'wait' }, + ['remove_container'] = { method = 'DELETE' }, + ['get_container_resource_info'] = { method = 'HEAD', endpoint = 'archive' }, + ['get_container_resource_archive'] = { method = 'GET', endpoint = 'archive' }, + ['create_exec_instance'] = { method = 'POST', endpoint = 'exec' }, + }, 'containers', d) + + loop_through_entity_endpoints({ + ['inspect_image'] = { method = 'GET', endpoint = 'json' }, + ['get_image_history'] = { method = 'GET', endpoint = 'history' }, + ['push_image'] = { method = 'POST', endpoint = 'push' }, + ['tag_image'] = { method = 'POST', endpoint = 'tag' }, + ['remove_image'] = { method = 'DELETE' }, + }, 'images', d) + + loop_through_entity_endpoints({ + ['inspect_network'] = { method = 'GET' }, + ['remove_network'] = { method = 'DELETE' }, + ['connect_container_to_network'] = { method = 'POST', endpoint = 'connect' }, + ['disconnect_container_from_network'] = { method = 'POST', endpoint = 'disconnect' }, + }, 'networks', d) + + loop_through_entity_endpoints({ + ['inspect_volume'] = { method = 'GET' }, + ['remove_volume'] = { method = 'DELETE' }, + }, 'volumes', d) + + loop_through_entity_endpoints({ + ['start_exec_instance'] = { method = 'POST', endpoint = 'start' }, + ['resize_exec_instance'] = { method = 'POST', endpoint = 'resize' }, + ['inspect_exec_instance'] = { method = 'GET', endpoint = 'json' }, + }, 'exec', d) + + loop_through_entity_endpoints({ + ['inspect_node'] = { method = 'GET' }, + ['delete_node'] = { method = 'DELETE' }, + ['update_node'] = { method = 'POST', endpoint = 'update' }, + }, 'nodes', d) + + loop_through_entity_endpoints({ + ['inspect_service'] = { method = 'GET' }, + ['delete_service'] = { method = 'DELETE' }, + ['update_service'] = { method = 'POST', endpoint = 'update' }, + ['get_service_logs'] = { method = 'GET', endpoint = 'logs' }, + }, 'services', d) + + loop_through_entity_endpoints({ + ['inspect_task'] = { method = 'GET' }, + }, 'tasks', d) + + loop_through_entity_endpoints({ + ['inspect_secret'] = { method = 'GET' }, + ['delete_secret'] = { method = 'DELETE' }, + ['update_secret'] = { method = 'POST', endpoint = 'update' }, + }, 'secrets', d) + + loop_through_entity_endpoints({ + ['inspect_config'] = { method = 'GET' }, + ['delete_config'] = { method = 'DELETE' }, + ['update_config'] = { method = 'POST', endpoint = 'update' }, + }, 'configs', d) + + loop_through_entity_endpoints({ + ['inspect_plugin'] = { method = 'GET', endpoint = 'json' }, + ['remove_plugin'] = { method = 'DELETE' }, + ['enable_plugin'] = { method = 'POST', endpoint = 'enable' }, + ['disable_plugin'] = { method = 'POST', endpoint = 'disable' }, + ['upgrade_plugin'] = { method = 'POST', endpoint = 'upgrade' }, + ['push_plugin'] = { method = 'POST', endpoint = 'push' }, + ['configure_plugin'] = { method = 'POST', endpoint = 'set' }, + }, 'plugins', d) + + loop_through_entity_endpoints({ + ['get_registry_image_info'] = { method = 'GET', endpoint = 'json' }, + }, 'distribution', d) + + return d + end +} diff --git a/source/tools/monitor/unity/test/curl/docker.lua b/source/tools/monitor/unity/test/curl/docker.lua new file mode 100644 index 00000000..dd349539 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/docker.lua @@ -0,0 +1,13 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/10 12:01 AM +--- + +package.path = package.path .. ";../../?.lua;" + +local api = require("httplib.dockerApi") +local system = require("common.system") + +local d = api.new('localhost', '/mnt/host/var/run/docker.sock') +print(system:dump(d:list_containers({ all = 'true' }))) -- Gitee From f00486f48ee18ae9c8d361eca72b2cc07884748c Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Fri, 10 Mar 2023 08:24:31 +0800 Subject: [PATCH 69/77] add m4 and lua http for docker. --- source/tools/monitor/unity/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/unity/Dockerfile b/source/tools/monitor/unity/Dockerfile index 9afeee48..c60554cb 100644 --- a/source/tools/monitor/unity/Dockerfile +++ b/source/tools/monitor/unity/Dockerfile @@ -2,7 +2,7 @@ FROM registry.cn-hangzhou.aliyuncs.com/sysom/lcc MAINTAINER "liaozhaoyan " WORKDIR /root/ RUN source /opt/rh/devtoolset-9/enable && \ - yum install -y make wget lua-devel unzip git numactl-devel && \ + yum install -y make wget lua-devel unzip git numactl-devel m4 && \ mkdir /root/build && \ cd /root/build && \ git clone https://gitee.com/chuyansz/sysak.git && \ @@ -33,7 +33,7 @@ RUN source /opt/rh/devtoolset-9/enable && \ luarocks install sha1 && \ luarocks install md5 && \ luarocks install luaposix 35.1-1 && \ - luarocks install uname && \ + luarocks install http && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ && \ cd ../beeQ/ && \ make \ No newline at end of file -- Gitee From d97c3e629bd74f72b6daa8a04f866d937ab68587 Mon Sep 17 00:00:00 2001 From: Bixuan Cui Date: Thu, 9 Mar 2023 18:00:54 +0800 Subject: [PATCH 70/77] metrics: add interrupts/mounts/softirqs/self_statm Signed-off-by: Bixuan Cui --- .../monitor/unity/beaver/guide/metrics.md | 76 +++++++++++++++---- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index d3ecf423..36ee4696 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -90,16 +90,66 @@ This parser parses the stats from network devices. These stats includes events p | 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | | :--- | --- | :---- | :---- | :--- | -| type | - | subsys类型 | | collector/proc_cgroups.lua | -| blkio | 个 | blkio cgroup 数量 | | collector/proc_cgroups.lua | -| freezer | 个 | freezer cgroup数量 | | collector/proc_cgroups.lua | -| devices | 个 | devices cgroup数量 | | collector/proc_cgroups.lua | -| hugetlb | 个 | hugetlb cgroup数量 | | collector/proc_cgroups.lua | -| pids | 个 | blkio cgroup 数量 | | collector/proc_cgroups.lua | -| rdma | 个 | rdma cgroup数量 | | collector/proc_cgroups.lua | -| net_prio | 个 | net_prio cgroup数量 | | collector/proc_cgroups.lua | -| net_cls | 个 | net_cls cgroup数量 | | collector/proc_cgroups.lua | -| cpu | 个 | cpu cgroup 数量 | | collector/proc_cgroups.lua | -| cpuacct | 个 | cpuacct cgroup数量 | | collector/proc_cgroups.lua | -| perf_event | 个 | perf_event cgroup数量 | | collector/proc_cgroups.lua | -| memory | 个 | memory cgroup数量 | | collector/proc_cgroups.lua | +| type | - | subsys类型 | | collector/proc\_cgroups.lua | +| blkio | 个 | blkio cgroup 数量 | | collector/proc\_cgroups.lua | +| freezer | 个 | freezer cgroup数量 | | collector/proc\_cgroups.lua | +| devices | 个 | devices cgroup数量 | | collector/proc\_cgroups.lua | +| hugetlb | 个 | hugetlb cgroup数量 | | collector/proc\_cgroups.lua | +| pids | 个 | blkio cgroup 数量 | | collector/proc\_cgroups.lua | +| rdma | 个 | rdma cgroup数量 | | collector/proc\_cgroups.lua | +| net\_prio | 个 | net_prio cgroup数量 | | collector/proc\_cgroups.lua | +| net\_cls | 个 | net_cls cgroup数量 | | collector/proc\_cgroups.lua | +| cpu | 个 | cpu cgroup 数量 | | collector/proc\_cgroups.lua | +| cpuacct | 个 | cpuacct cgroup数量 | | collector/proc\_cgroups.lua | +| perf\_event | 个 | perf_event cgroup数量 | | collector/proc\_cgroups.lua | +| memory | 个 | memory cgroup数量 | | collector/proc\_cgroups.lua | + +### interrupts 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| cpu | - | CPU ID | | collector/proc\_interrupts.lua | +| 中断名称 | 次 | 中断触发次数 | | collector/proc\_interrupts.lua | + +### mounts 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| fs | - | sysfs | | collector/proc\_mounts.lua | +| mount | - | 挂载目录 | | collector/proc\_mounts.lua | +| f\_bsize | - | Filesystem block size | | collector/proc\_mounts.lua | +| f\_blocks | - | Size of fs in f_frsize units | | collector/proc\_mounts.lua | +| f\_bfree | - | Number of free blocks | | collector/proc\_mounts.lua | +| f\_bavail | - | Number of free blocks for unprivileged users | | collector/proc\_mounts.lua | +| f\_files | - | Number of inodes | | collector/proc\_mounts.lua | +| f\_ffree | - | Number of free inodes | | collector/proc\_mounts.lua | +| f\_favail | - | Number of free inodes for unprivileged users | | collector/proc\_mounts.lua | + +### softirqs 表 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| cpu | - | CPU ID | | collector/proc\_softirqs.lua | +| HI | 次 | HI软中断触发次数 | | collector/proc\_softirqs.lua | +| TIMER | 次 | TIMER软中断触发次数 | | collector/proc\_softirqs.lua | +| NET\_TX | 次 | NET\_TX软中断触发次数 | | collector/proc\_softirqs.lua | +| NET\_RX | 次 | NET\_RX软中断触发次数 | | collector/proc\_softirqs.lua | +| BLOCK | 次 | BLOCK软中断触发次数 | | collector/proc\_softirqs.lua | +| IRQ_POLL | 次 | IRQ\_POLL软中断触发次数 | | collector/proc\_softirqs.lua | +| TASKLET | 次 | TASKLET软中断触发次数 | | collector/proc\_softirqs.lua | +| SCHED | 次 | SCHED软中断触发次数 | | collector/proc\_softirqs.lua | +| HRTIMER | 次 | HRTIMER软中断触发次数 | | collector/proc\_softirqs.lua | +| RCU | 次 | RCU软中断触发次数 | | collector/proc\_softirqs.lua | + +### self_statm 表 +统计监控进程的statm信息 + +| 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | --- | :---- | :---- | :--- | +| size | - | total program size | | collector/proc\_statm.lua | +| resident | - | resident set size | | collector/proc\_statm.lua | +| shared | - | number of resident shared pages | | collector/proc\_statm.lua | +| text | - | text (code) | | collector/proc\_statm.lua | +| lib | - | library | | collector/proc\_statm.lua | +| data | - | data + stack | | collector/proc\_statm.lua | +| dt | - | dirty pages | | collector/proc\_statm.lua | -- Gitee From afe785cae1fdf9a2ad72c6241fa6047f5ba15e60 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Fri, 10 Mar 2023 10:14:09 +0800 Subject: [PATCH 71/77] add conPlugin model. --- .../monitor/unity/collector/conPlugin.lua | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 source/tools/monitor/unity/collector/conPlugin.lua diff --git a/source/tools/monitor/unity/collector/conPlugin.lua b/source/tools/monitor/unity/collector/conPlugin.lua new file mode 100644 index 00000000..3709afa1 --- /dev/null +++ b/source/tools/monitor/unity/collector/conPlugin.lua @@ -0,0 +1,19 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/10 8:26 AM +--- + +require("common.class") +local system = require("common.system") +local CconPlugin = class("conPlugin") + +function CconPlugin:_init_(proto, procffi, que, proto_q, fYaml) + self._proto = proto + self._que = que + self._plugins = setup() +end + +function CconPlugin:proc(elapsed, lines) + +end \ No newline at end of file -- Gitee From ea3b42882c19e493c68f0d17550444e182dc6268 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Fri, 10 Mar 2023 23:59:48 +0800 Subject: [PATCH 72/77] add demo for pod list. --- .../tools/monitor/unity/beaver/guide/develop.md | 2 +- source/tools/monitor/unity/collector/plugin.yaml | 2 ++ source/tools/monitor/unity/test/curl/pods.lua | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/test/curl/pods.lua diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 03acc3bc..2a2feff7 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -17,7 +17,7 @@ git clone -b unity https://gitee.com/anolis/sysak.git ## 2.2 拉起容器 ``` -docker run -v /root/1ext/code/:/root/code -v /:/mnt/host:ro -v /var/run/docker.sock:/var/run/docker.sock --net=host --name unity --privileged -itd registry.cn-hangzhou.aliyuncs.com/sysom/sysom:v1.0 /bin/sh +docker run -v /root/1ext/code/:/root/code -v /:/mnt/host:ro --net=host --pid=host --name unity --privileged -itd registry.cn-hangzhou.aliyuncs.com/sysom/sysom:v1.0 /bin/sh ``` docker 参数说明： diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index dfff38e6..2b9821f6 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -23,6 +23,8 @@ luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat", "proc_uptime"] + + plugins: - so: kmsg description: "collect dmesg info." diff --git a/source/tools/monitor/unity/test/curl/pods.lua b/source/tools/monitor/unity/test/curl/pods.lua new file mode 100644 index 00000000..17ffb373 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/pods.lua @@ -0,0 +1,15 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/3/10 11:50 PM +--- + +package.path = package.path .. ";../../?.lua;" + +local ChttpCli = require("httplib.httpCli") +local system = require("common.system") + +local cli = ChttpCli.new() +local res = cli:get("http://127.0.0.1:10255/pods") +local obj = cli:jdecode(res.body) +print(system:dump(obj.items)) -- Gitee From c922c9935e9b7f231733e0b864c9096967e0c847 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sun, 12 Mar 2023 23:25:25 +0800 Subject: [PATCH 73/77] add string function for pystring. --- .../tools/monitor/unity/common/pystring.lua | 295 +++++++++++++++++- .../tools/monitor/unity/httplib/dockerApi.lua | 10 +- source/tools/monitor/unity/test/string/py.lua | 135 +++++++- 3 files changed, 417 insertions(+), 23 deletions(-) diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index 4c0e6f7c..a499ee85 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -40,12 +40,9 @@ local function newStack() return stack end -local function checkDelimiter(ch) - local s = "().%+-*?[]^$" - if ch == " " then - return "%s" - end - for c in string.gmatch(s, ".") do +local luaReReserve = "().%+-*?[]^$" +local function checkLuaReReserve(ch) + for c in string.gmatch(luaReReserve, ".") do if c == ch then return "%" .. ch end @@ -58,7 +55,11 @@ local function setupDelimiter(delimiter) local i = 0 for c in string.gmatch(delimiter, ".") do i = i + 1 - rt[i] = checkDelimiter(c) + if c == " " then + rt[i] = "%s" + else + rt[i] = checkLuaReReserve(c) + end end return table.concat(rt) end @@ -73,8 +74,26 @@ local function setupPatten(s) return patten end +local function _setupRepl(repl) + local rt = {} + local i = 0 + for c in string.gmatch(repl, ".") do + i = i + 1 + rt[i] = checkLuaReReserve(c) + end + return table.concat(rt) +end + +local function setupRepl(s) + if s == nil then + error("repl must be a string.") + else + return _setupRepl(s) + end +end + function pystring:shift(s, n) -- position for right, negative for left - local len = string.len(s) + local len = #s if len == 0 then return s end @@ -94,21 +113,92 @@ function pystring:shift(s, n) -- position for right, negative for left end end +function pystring:islower(s) + local match = string.match(s, "[%l%s%p]+") + if not match then + return false + end + return #match == #s +end + +function pystring:isupper(s) + local match = string.match(s, "[%u%s%p]+") + if not match then + return false + end + return #match == #s +end + +function pystring:isdigit(s) + local match = string.match(s, "%d+") + if not match then + return false + end + return #match == #s +end + +function pystring:ishex(s) + local match = string.match(s, "%x+") + if not match then + return false + end + return #match == #s +end + +function pystring:isalnum(s) + local match = string.match(s, "%w+") + if not match then + return false + end + return #match == #s +end + +function pystring:istilte(s) + local match = string.match(s, "%u%l*") + if not match then + return false + end + return #match == #s +end + +function pystring:isfloat(s) + local dotCnt = 0 + local ascDot = string.byte(".") + local asc0, asc9 = string.byte("0"), string.byte("9") + for i = 1, #s do + local ch = s:byte(i) + if ch == ascDot then + dotCnt = dotCnt + 1 + if dotCnt > 1 then + return false + end + elseif ch > asc9 or ch < asc0 then + return false + end + end + return true +end + function pystring:lower(s) return string.lower(s) end +function pystring:casefold(s) + return string.lower(s) +end + function pystring:upper(s) return string.upper(s) end function pystring:swapcase(s) local swaps = {} + local ascA, ascZ, asc_a, asc_z = string.byte('A'), string.byte('Z'), string.byte('a'), string.byte('z') for i=1, #s do local ch = string.byte(s, i) - if ch >= 65 and ch <= 90 then + if ch >= ascA and ch <= ascZ then swaps[i] = string.char(ch + 32) - elseif ch >= 97 and ch <= 122 then + elseif ch >= asc_a and ch <= asc_z then swaps[i] = string.char(ch - 32) else swaps[i] = string.char(ch) @@ -126,6 +216,17 @@ function pystring:capitalize(s) return string.upper(s1) .. s2 end +function pystring:title(s) + if #s < 1 then + return s + end + local ss = pystring:split(s, " ") + for i = 1, #ss do + ss[i] = pystring:capitalize(ss[i]) + end + return table.concat(ss, " ") +end + function pystring:capwords(s) local lines = pystring:split(s, "\n") local rLines = {} @@ -140,6 +241,55 @@ function pystring:capwords(s) return table.concat(rLines, "\n") end +function pystring:ljust(s, len, ch) + ch = ch or " " + if #ch ~= 1 then + error("pad string master a single word, not " .. ch) + end + local delta = len - #s + if delta > 0 then + local pad = string.rep(ch, delta) + return pad .. s + else + return s + end +end + +function pystring:rjust(s, len, ch) + ch = ch or " " + if #ch ~= 1 then + error("pad string master a single word, not " .. ch) + end + local delta = len - #s + if delta > 0 then + local pad = string.rep(ch, delta) + return s .. pad + else + return s + end +end + +function pystring:center(s, len, ch) + ch = ch or " " + if #ch ~= 1 then + error("pad string master a single word, not " .. ch) + end + local delta = len - #s + if delta > 0 then + local left = math.floor(delta / 2) + local right = delta - left + + local res = {string.rep(ch, left), s, string.rep(ch, right)} + return table.concat(res) + else + return s + end +end + +function pystring:zfill(s, len) + return pystring:ljust(s, len, "0") +end + function pystring:split(s, delimiter, n) local result = {} if not delimiter or delimiter == "" then -- for blank, gsub multi blank to single @@ -160,18 +310,48 @@ function pystring:split(s, delimiter, n) nums = nums + 1 if nums >= n then c = c + 1 - result[c] = string.sub(s, beg, string.len(s)) + result[c] = string.sub(s, beg, #s) break end else c = c + 1 - result[c] = string.sub(s, beg, string.len(s)) + result[c] = string.sub(s, beg, #s) break end end return result end +function pystring:partition(s, del) + local result = {} + del = del or " " + local delimiter = setupDelimiter(del) + local iBeg, iEnd = string.find(s, delimiter) + if iBeg then + result[1] = string.sub(s, 1, iBeg - 1) + result[2] = del + result[3] = string.sub(s, iEnd + 1) + return result + else + return nil + end +end + +function pystring:partition(s, del) + local result = {} + del = del or " " + local delimiter = setupDelimiter(del) + local iBeg, iEnd = string.find(s, delimiter) + if iBeg then + result[1] = string.sub(s, 1, iBeg - 1) + result[2] = del + result[3] = string.sub(s, iEnd + 1) + return result + else + return nil + end +end + function pystring:reverseTable(t) local n = #t for i = 1, n / 2 do @@ -182,7 +362,7 @@ end function pystring:rsplit(s, delimiter, n) local result = {} local n = n or 2 ^ 63 - 1 - local len = string.len(s) + 1 + local len = #s + 1 local rs = string.reverse(s) local rDel = string.reverse(delimiter or " ") rDel = setupDelimiter(rDel) @@ -213,6 +393,29 @@ function pystring:rsplit(s, delimiter, n) return result end +function pystring:rpartition(s, del) + local result = {} + del = del or " " + local rs = string.reverse(s) + local rDel = string.reverse(del) + local delimiter = setupDelimiter(rDel) + local len = #s + + local iBeg, iEnd = string.find(rs, delimiter) + if iBeg then + result[1] = string.sub(s, 1, len - iBeg + 1 - #del) + result[2] = del + result[3] = string.sub(s, len - iEnd + 1 + #del) + return result + else + return nil + end +end + +function pystring:splitlines(s) + return pystring:split(s, '\n') +end + function pystring:lstrip(s, chars) local patten = "^" .. setupPatten(chars) .. "+" local _, ends = string.find(s, patten) @@ -243,15 +446,73 @@ function pystring:join(delim, strings) end function pystring:startswith(s1, s2) - return string.sub(s1,1,string.len(s2)) == s2 + return string.sub(s1,1, #s2) == s2 end function pystring:endswith(s1, s2) - return s2=='' or string.sub(s1,-string.len(s2)) == s2 + return s2 == '' or string.sub(s1,-#s2) == s2 +end + +function pystring:find(s1, s2, start, stop) + start = start or 1 + stop = stop or -1 + s1 = string.sub(s1, start, stop) + local res = string.find(s1, s2, 1, false) + return res or -1 +end + +function pystring:rfind(s1, s2, start, stop) + start = start or 1 + stop = stop or -1 + s1 = string.sub(s1, start, stop) + + local len = #s1 + local lFind = #s2 + local rs1, rs2 = string.reverse(s1), string.reverse(s2) + local i = string.find(rs1, rs2, 1, false) + if i then + return len - i - lFind + 1 + else + return -1 + end +end + +function pystring:index(s1, s2, start, stop) + local res = pystring:find(s1, s2, start, stop) + if res < 0 then + error(s2 .. " is not in " .. s1) + end + return res +end + +function pystring:rindex(s1, s2, start, stop) + local res = pystring:rfind(s1, s2, start, stop) + if res < 0 then + error(s2 .. " is not in " .. s1) + end + return res +end + +function pystring:count(s, find) + local i = 0 + local patten = setupPatten(find) + for _ in string.gmatch(s, patten) do + i = i + 1 + end + return i +end + +function pystring:replace(s, find, repl, n) + local patten = setupPatten(find) + repl = setupRepl(repl) + + return string.gsub(s, patten, repl, n) end -function pystring:find(s1, s2) - return string.find(s1, s2, 1, false) +function pystring:expandtabs(s, tabs) + tabs = tabs or 4 + local repl = string.rep(" ", tabs) + return string.gsub(s, "\t", repl) end return pystring diff --git a/source/tools/monitor/unity/httplib/dockerApi.lua b/source/tools/monitor/unity/httplib/dockerApi.lua index 983006a2..ee7c395b 100644 --- a/source/tools/monitor/unity/httplib/dockerApi.lua +++ b/source/tools/monitor/unity/httplib/dockerApi.lua @@ -4,12 +4,12 @@ --- DateTime: 2023/3/10 12:00 AM --- -local client = require 'http.client' -local headers = require 'http.headers' -local util = require 'http.util' +local client = require("http.client") +local headers = require("http.headers") +local util = require("http.util") -local cjson = require 'cjson.safe' -local basexx = require 'basexx' +local cjson = require("cjson.safe") +local basexx = require("basexx") local handle_response_body = function (body) if type(body) == 'string' then diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index a4d17e12..2c6e5606 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -35,6 +35,9 @@ assert(ret[3] == "language") ret = pystring:rsplit("hello*lua *language", "*", 1) assert(#ret == 2) assert(ret[1] == "hello*lua ") +ret = pystring:rsplit("hello*lua *lua language", "lua", 1) +assert(#ret == 2) +assert(ret[1] == "hello*lua *") -- 多字符串分割 ret = pystring:split("hello*lua *language", "*l") @@ -43,6 +46,46 @@ assert(ret[1] == "hello") assert(ret[2] == "ua ") assert(ret[3] == "anguage") +-- partition +ret = pystring:partition("hello lua") +assert(#ret == 3) +assert(ret[1] == "hello") +assert(ret[2] == " ") +assert(ret[3] == "lua") +ret = pystring:partition("hello*lua", "*") +assert(#ret == 3) +assert(ret[1] == "hello") +assert(ret[2] == "*") +assert(ret[3] == "lua") +ret = pystring:partition("hello lua language") +assert(#ret == 3) +assert(ret[1] == "hello") +assert(ret[2] == " ") +assert(ret[3] == "lua language") +ret = pystring:partition("hello lua language", "lua") +assert(#ret == 3) +assert(ret[1] == "hello ") +assert(ret[2] == "lua") +assert(ret[3] == " language") +ret = pystring:partition("hello*lua") +assert(ret == nil) + +-- rpartition +ret = pystring:rpartition("hello lua language") +assert(ret[1] == "hello lua") +assert(ret[2] == " ") +assert(ret[3] == "language") +ret = pystring:rpartition("hello lua lua language", "lua") +assert(ret[1] == "hello lua ") +assert(ret[2] == "lua") +assert(ret[3] == " language") + +-- splitlines +ret = pystring:splitlines("hello\nlua\nlanguage") +assert(ret[1] == "hello") +assert(ret[2] == "lua") +assert(ret[3] == "language") + -- strip掉左右空格 assert(pystring:strip("\t hello world. \t\n") == "hello world.") @@ -72,10 +115,21 @@ assert(pystring:endswith("hello world", "world")) -- find assert(pystring:find("hello world.", "hello") == 1) +assert(pystring:find("hello world.", "hEllo") == -1) + +-- rfind +assert(pystring:rfind("hello world hello.", "hello") == 12) +assert(pystring:rfind("hello world hello.", "hEllo") == -1) + +-- count +assert(pystring:count("hello world hello.", "hello") == 2) +assert(pystring:count("hello world hello.", "hEllo") == 0) +assert(pystring:count("hello world hello.", " ") == 2) -- shift assert(pystring:shift("abcd", 1) == "dabc") assert(pystring:shift("abcd", -1) == "bcda") +assert(pystring:shift("abcd", -2) == "cdab") -- swapcase assert(pystring:swapcase("Hello, World!") == "hELLO, wORLD!") @@ -85,6 +139,85 @@ assert(pystring:capitalize("hello") == "Hello") assert(pystring:capitalize("") == "") assert(pystring:capitalize("H") == "H") +-- title +assert(pystring:title("hello") == "Hello") +assert(pystring:title("") == "") +assert(pystring:title("hello world.") == "Hello World.") +assert(pystring:title("hello world.") == "Hello World.") + -- capwords assert(pystring:capwords("hello world.") == "Hello World.") -assert(pystring:capwords("hello world.\nhere you are.") == "Hello World.\nHere You Are.") \ No newline at end of file +assert(pystring:capwords("hello world.\nhere you are.") == "Hello World.\nHere You Are.") + +-- islower +assert(pystring:islower("hello") == true) +assert(pystring:islower("Hello") == false) +assert(pystring:islower("hello world!") == true) + +-- isupper +assert(pystring:isupper("HELLO") == true) +assert(pystring:isupper("Hello") == false) +assert(pystring:isupper("Hello World") == false) +assert(pystring:isupper("HELLO WORLD!") == true) + +-- isdigit +assert(pystring:isdigit("1234") == true) +assert(pystring:isdigit("123a") == false) +assert(pystring:isdigit("123.45") == false) + +-- ishex +assert(pystring:ishex("1234") == true) +assert(pystring:ishex("123a") == true) +assert(pystring:ishex("abcdef") == true) +assert(pystring:ishex("00ABCDEF") == true) +assert(pystring:ishex("123FG") == false) +assert(pystring:ishex("123.45") == false) + +-- isalnum +assert(pystring:isalnum("1234") == true) +assert(pystring:isalnum("00ABCDEF") == true) +assert(pystring:isalnum("123FG") == true) +assert(pystring:isalnum("123.45") == false) +assert(pystring:isalnum("123 45") == false) + +-- istilte +assert(pystring:istilte("Aaa") == true) +assert(pystring:istilte("aaa") == false) +assert(pystring:istilte("Aaa0") == false) +assert(pystring:istilte("A") == true) + +-- isfloat +assert(pystring:isfloat("1234") == true) +assert(pystring:isfloat("00ABCDEF") == false) +assert(pystring:isfloat("123FG") == false) +assert(pystring:isfloat("123.45") == true) +assert(pystring:isfloat("123 45") == false) + +-- ljust +assert(pystring:ljust("1234", 5) == " 1234") +assert(pystring:ljust("1234", 3) == "1234") +assert(pystring:ljust("1234", 6, "*") == "**1234") + +-- rjust +assert(pystring:rjust("1234", 5) == "1234 ") +assert(pystring:rjust("1234", 3) == "1234") +assert(pystring:rjust("1234", 6, "*") == "1234**") + +-- center +assert(pystring:center("1234", 5) == "1234 ") +assert(pystring:center("1234", 7) == " 1234 ") +assert(pystring:center("1234", 8) == " 1234 ") +assert(pystring:center("1234", 8, "*") == "**1234**") + +-- zfill +assert(pystring:zfill("3.14", 6) == "003.14") + +-- replace +assert(pystring:replace("hello world.", "world", "lua") == "hello lua.") +assert(pystring:replace("hello world world.", "world", "lua") == "hello lua lua.") +assert(pystring:replace("hello world world.", "world", "lua", 1) == "hello lua world.") +assert(pystring:replace("hello %. %*.", "%.", "%*") == "hello %* %*.") +assert(pystring:replace("hello %. %*.", "%.", " ") == "hello %*.") + +-- expandtabs +assert(pystring:expandtabs("hello\tworld.") == "hello world.") -- Gitee From 97891b910ed2f6b5f7a78834eafc9c3ffd021cd2 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 13 Mar 2023 10:20:58 +0800 Subject: [PATCH 74/77] lmd change for string.len --- source/tools/monitor/unity/common/lmd.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/common/lmd.lua b/source/tools/monitor/unity/common/lmd.lua index 685c66d1..bb22f611 100644 --- a/source/tools/monitor/unity/common/lmd.lua +++ b/source/tools/monitor/unity/common/lmd.lua @@ -141,7 +141,7 @@ local function Quotes(quotes, res) local level = 1 for i = start, len do local levels, body = unpack(pystring:split(quotes[i], " ", 1)) - local v = string.len(levels) + local v = #levels if v > level then while v > level do table.insert(res, "
") -- Gitee From e7fc596676b008bcccbafddb13d4b2b68f065c74 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 13 Mar 2023 10:37:59 +0800 Subject: [PATCH 75/77] fix description for softnet_stat. --- source/tools/monitor/unity/beaver/guide/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index 879c25a8..f1fb6247 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -81,7 +81,7 @@ This parser parses the stats from network devices. These stats includes events p | :--- | ---: | :---- | :---- | :--- | | packet\_process | 个 | cpu，对应CPU号 | 所在核收包个数 | collector/proc\_softnet\_stat.lua | | packet\_drop | 个 | cpu，对应CPU号 | 所在核丢包个数 | collector/proc\_softnet\_stat.lua | -| cpu\_collision | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | +| cpu\_collision | 个 | cpu，对应CPU号 | collision occur while obtaining device lock while transmitting. | collector/proc\_softnet\_stat.lua | | received\_rps | 个 | cpu，对应CPU号 | number of times cpu woken up received_rps. | collector/proc\_softnet\_stat.lua | | time\_squeeze | 个 | cpu，对应CPU号 | net\_rx\_action. | collector/proc\_softnet\_stat.lua | | flow\_limit\_count | 个 | cpu，对应CPU号 | number of times reached flow limit count. | collector/proc\_softnet\_stat.lua | -- Gitee From b38499afa5d06d3d151700be4dac958918f3e5a5 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Mon, 13 Mar 2023 11:32:38 +0800 Subject: [PATCH 76/77] add socket status description. --- .../monitor/unity/beaver/guide/metrics.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source/tools/monitor/unity/beaver/guide/metrics.md b/source/tools/monitor/unity/beaver/guide/metrics.md index f1fb6247..5b276235 100644 --- a/source/tools/monitor/unity/beaver/guide/metrics.md +++ b/source/tools/monitor/unity/beaver/guide/metrics.md @@ -73,6 +73,26 @@ | retrans | 次 | | 协议栈溢出次数 | collector/proc\_snmp\_stat.lua | | drop | 次 | | 协议栈丢包次数 | collector/proc\_snmp\_stat.lua | +### sock_stat + +统计所有包状态。[参考连接](https://developer.aliyun.com/article/484451) + + 指标名 | 单位 | 标签说明 | 备注 | 源码路径 | +| :--- | ---: | :---- | :---- | :--- | +| frag\_inuse | 个 | | 使用的IP段数量 | collector/proc\_sockstat.lua | +| frag\_memory | 页 | | IP段使用内存数量 | collector/proc\_sockstat.lua | +| udplite\_inuse | 个 | | udplite 使用量 | collector/proc\_sockstat.lua | +| udp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | +| udp\_inuse | 个 | | udp 使用量 | collector/proc\_sockstat.lua | +| tcp\_mem | 页 | | udp socket 内存使用量，含收发缓冲区队列 | collector/proc\_sockstat.lua | +| tcp\_alloc | 个 | | TCP socket 申请总数 | collector/proc\_sockstat.lua | +| tcp\_tw | 个 | | TCP time wait socket 总数 | collector/proc\_sockstat.lua | +| tcp\_orphan | 个 | | TCP ophan socket 总数 | collector/proc\_sockstat.lua | +| tcp\_inuse | 个 | | TCP 常规 socket 总数 | collector/proc\_sockstat.lua | +| raw\_inuse | 个 | | raw socket 使用量 | collector/proc\_sockstat.lua | +| sockets\_used | 个 | | 总socket 使用量 | collector/proc\_sockstat.lua | + + ### softnets This parser parses the stats from network devices. These stats includes events per cpu\(in row\), number of packets processed i.e packet_process \(first column\), number of packet drops packet\_drops \(second column\), time squeeze eg net\_rx\_action performed time_squeeze\(third column\), cpu collision eg collision occur while obtaining device lock while transmitting cpu\_collision packets \(eighth column\), received_rps number of times cpu woken up received\_rps \(ninth column\), number of times reached flow limit count flow\_limit\_count \(tenth column\), backlog status \(eleventh column\), core id \(twelfth column\). -- Gitee From c8a290bcd34b8f09bf810f3578437fb4d98b166e Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Tue, 14 Mar 2023 15:23:53 +0800 Subject: [PATCH 77/77] plugin: remove compile failed numainfo Why we should bear the compile fail *every* time, looks like: numainfo.c:7:18: fatal error: numa.h: No such file or directory #include We need to take responsibility for our code, but not leave it alone after pushing them to remote. Signed-off-by: Hailong Liu --- source/tools/monitor/unity/collector/plugin/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 9c538406..794d8900 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -5,7 +5,7 @@ OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg unity_nosched unity_irqoff cpudist net_health net_retrans netlink numainfo cpufreq gpuinfo +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg unity_nosched unity_irqoff cpudist net_health net_retrans netlink cpufreq gpuinfo all: $(LIB) $(DEPMOD) -- Gitee