From dc7cb72f2a685e5680f70295091df9981fccc36f Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Mon, 20 Feb 2023 17:48:58 +0800 Subject: [PATCH 01/21] unity/plugin: chang the table name to adjust cload-moni Signed-off-by: Hailong Liu --- .../unity/collector/plugin/proc_loadavg/proc_loadavg.c | 2 +- .../unity/collector/plugin/proc_schedstat/proc_schedstat.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index 7494e325..ace88eaf 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -67,7 +67,7 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); - unity_set_table(line, "proc_loadavg"); + unity_set_table(line, "sched_moni"); full_line(line); return 0; } diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index 41850778..b51d5d38 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -164,10 +164,10 @@ int call(int t, struct unity_lines* lines) { unity_alloc_lines(lines, nr_cpus+1); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); - unity_set_table(lines1[i], "proc_schedstat"); + unity_set_table(lines1[i], "sched_moni"); } line2 = unity_get_line(lines, nr_cpus); - unity_set_table(line2, "proc_schedstat"); + unity_set_table(line2, "sched_moni"); full_line(lines1, line2); return 0; } -- Gitee From d5c91506a977692830a33159f0787582b88a3731 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 21 Feb 2023 01:05:32 +0800 Subject: [PATCH 02/21] http can brower by markdown, html, text, report failure info for every bad call. --- source/tools/monitor/unity/beaver/beaver.c | 37 +++---- source/tools/monitor/unity/beaver/frame.lua | 32 ++++-- .../monitor/unity/beaver/guide/dev_proc.md | 2 +- .../tools/monitor/unity/beaver/guide/guide.md | 12 +-- .../monitor/unity/beaver/guide/hotplugin.md | 2 +- .../tools/monitor/unity/beaver/guide/oop.md | 2 +- .../monitor/unity/beaver/guide/proc_probe.md | 2 +- .../monitor/unity/beaver/guide/pystring.md | 3 +- .../monitor/unity/beaver/guide/webdevel.md | 4 +- source/tools/monitor/unity/beaver/index.lua | 2 +- .../monitor/unity/beaver/localBeaver.lua | 86 ++++++++++------ .../tools/monitor/unity/beaver/url_guide.lua | 48 ++------- source/tools/monitor/unity/beeQ/apps.c | 98 +++++++++++++------ .../monitor/unity/collector/outline/outline.c | 36 +++---- .../tools/monitor/unity/collector/plugin.yaml | 5 + .../unity/collector/plugin/proto_sender.c | 39 +++----- source/tools/monitor/unity/common/lmd.lua | 26 ++++- source/tools/monitor/unity/common/system.lua | 5 + .../tools/monitor/unity/httplib/httpBase.lua | 22 ++++- .../tools/monitor/unity/httplib/httpHtml.lua | 71 +++++++++++++- .../monitor/unity/test/beaver/walkDir.lua | 43 ++++++++ .../tools/monitor/unity/test/unix/pyunix.py | 28 ++++++ source/tools/monitor/unity/tsdb/foxTSDB.lua | 31 +++--- 23 files changed, 420 insertions(+), 216 deletions(-) create mode 100644 source/tools/monitor/unity/test/beaver/walkDir.lua create mode 100644 source/tools/monitor/unity/test/unix/pyunix.py diff --git a/source/tools/monitor/unity/beaver/beaver.c b/source/tools/monitor/unity/beaver/beaver.c index 293cbb4b..9885f056 100644 --- a/source/tools/monitor/unity/beaver/beaver.c +++ b/source/tools/monitor/unity/beaver/beaver.c @@ -12,22 +12,19 @@ #include #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, char *fYaml) { +static int call_init(lua_State *L, int err_func, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -67,6 +64,7 @@ void LuaAddPath(lua_State *L, char *name, char *value) { static lua_State * echos_init(char *fYaml) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -77,22 +75,15 @@ static lua_State * echos_init(char *fYaml) { /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); - LuaAddPath(L, "path", "../beaver/?.lua"); + err_func = lua_reg_errFunc(L); - ret = luaL_loadfile(L, "../beaver/beaver.lua"); - ret = lua_pcall(L, 0, LUA_MULTRET, 0); + ret = lua_load_do_file(L, "../beaver/beaver.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L, fYaml); + ret = call_init(L, err_func, fYaml); if (ret < 0) { goto endCall; } @@ -107,13 +98,14 @@ static lua_State * echos_init(char *fYaml) { static int echos(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "echo"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -147,7 +139,6 @@ int beaver_init(char *fYaml) { } ret = echos(L); lua_close(L); - sleep(5); // to release port } exit(1); } diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 0c243edd..09e95484 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -6,17 +6,16 @@ -- refer to https://blog.csdn.net/zx_emily/article/details/83024065 -local unistd = require("posix.unistd") -local poll = require("posix.poll") - require("common.class") local ChttpComm = require("httplib.httpComm") local pystring = require("common.pystring") +local system = require("common.system") local Cframe = class("frame", ChttpComm) function Cframe:_init_() ChttpComm._init_(self) self._objs = {} + self._obj_res = {} end local function waitDataRest(fread, rest, tReq) @@ -123,6 +122,14 @@ function Cframe:echo404() return pystring:join("\r\n", tHttp) end +function Cframe:findObjRes(path) + for k, v in pairs(self._obj_res) do + if string.find(path, k) then + return v + end + end +end + function Cframe:proc(fread) local stream = waitHttpHead(fread) if stream == nil then -- read return stream or error code or nil @@ -135,13 +142,15 @@ function Cframe:proc(fread) local obj = self._objs[tReq.path] local res, keep = obj:call(tReq) return res, keep - else - print("show all path.") - for k, _ in pairs(self._objs) do - print("path:", k) - end - return self:echo404(), false end + + local obj = self:findObjRes(tReq.path) + if obj then + local res, keep = obj:calls(tReq) + return res, keep + end + + return self:echo404(), false end end @@ -150,4 +159,9 @@ function Cframe:register(path, obj) self._objs[path] = obj end +function Cframe:registerRe(path, obj) + assert(self._obj_res[path] == nil, "the " .. path .. " is already registered.") + self._obj_res[path] = obj +end + return Cframe diff --git a/source/tools/monitor/unity/beaver/guide/dev_proc.md b/source/tools/monitor/unity/beaver/guide/dev_proc.md index 3c006d0d..b21b8863 100644 --- a/source/tools/monitor/unity/beaver/guide/dev_proc.md +++ b/source/tools/monitor/unity/beaver/guide/dev_proc.md @@ -304,4 +304,4 @@ return CkvProc -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index 0ac809e7..d6b61fa3 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,8 @@ # 目录 -1. [插件化与热更新](/guide/hotplugin) -2. [面向对象设计](/guide/oop) -3. [字符串处理](/guide/pystring) -4. [页面开发](/guide/webdevel) -5. [proc和probe记录表](/guide/proc_probe) -6. [采集proc 接口指标](/guide/dev_proc) \ No newline at end of file +1. [插件化与热更新](/guide/hotplugin.md) +2. [面向对象设计](/guide/oop.md) +3. [字符串处理](/guide/pystring.md) +4. [页面开发](/guide/webdevel.md) +5. [proc和probe记录表](/guide/proc_probe.md) +6. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/hotplugin.md b/source/tools/monitor/unity/beaver/guide/hotplugin.md index 5eb67541..5754909c 100644 --- a/source/tools/monitor/unity/beaver/guide/hotplugin.md +++ b/source/tools/monitor/unity/beaver/guide/hotplugin.md @@ -111,4 +111,4 @@ unity监控采用[yaml](http://yaml.org/)对插件进行管理，当前插件分 此时数据只是已经更新入库了，但是要在nodexport上面显示，需要配置beaver/export.yaml 文件，才能将查询从数据表中更新。 -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/oop.md b/source/tools/monitor/unity/beaver/guide/oop.md index 90c265bc..87c4a675 100644 --- a/source/tools/monitor/unity/beaver/guide/oop.md +++ b/source/tools/monitor/unity/beaver/guide/oop.md @@ -113,4 +113,4 @@ Ctwo 继承于Cone，这里重新实现并复用了父类的say方法。 function Cone:say() function Cone.say(self) -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/proc_probe.md b/source/tools/monitor/unity/beaver/guide/proc_probe.md index ac412ac6..3283c9bc 100644 --- a/source/tools/monitor/unity/beaver/guide/proc_probe.md +++ b/source/tools/monitor/unity/beaver/guide/proc_probe.md @@ -22,4 +22,4 @@ libbpf kprobe/kretprobe/trace\_event/perf event 等事件记录在这里 | ----- | --------- | | xxx | xxx | -[返回目录](/guide) +[返回目录](/guide/guide.md) diff --git a/source/tools/monitor/unity/beaver/guide/pystring.md b/source/tools/monitor/unity/beaver/guide/pystring.md index 5e795f4b..c783d0b2 100644 --- a/source/tools/monitor/unity/beaver/guide/pystring.md +++ b/source/tools/monitor/unity/beaver/guide/pystring.md @@ -1,4 +1,5 @@ # 字符串处理 +![pystring](image/python.png) 同为脚本语言，lua 默认的字符串处理并不像python 那么完善。但只要通过拓展，也可以像python 一样对字符串进行处理。当前已经实现了split/strip 等高频使用函数。参考[Python字符串处理](https://www.jianshu.com/p/b758332c44bb) @@ -98,4 +99,4 @@ find 用于子串查找，成功返回首次开始的位置，如果不包含， assert(pystring:find("hello world.", "hello") == 1) ``` -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/webdevel.md b/source/tools/monitor/unity/beaver/guide/webdevel.md index 1fa5b829..b23efd6d 100644 --- a/source/tools/monitor/unity/beaver/guide/webdevel.md +++ b/source/tools/monitor/unity/beaver/guide/webdevel.md @@ -58,11 +58,11 @@ end return CurlGuide ``` -这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop) +这里采用了面向对象方法实现，关于面向对象，可以[参考这里](/guide/oop.md) ## 热更新 * 如果仅修改了markdown文件，直接更新文件刷新页面即可； * 如果修改了lua文件，给主进程发送1号信号，进程会重新装载，新页面也会立即生效； -[返回目录](/guide) \ No newline at end of file +[返回目录](/guide/guide.md) \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/index.lua b/source/tools/monitor/unity/beaver/index.lua index c3ce944c..b6884e0b 100644 --- a/source/tools/monitor/unity/beaver/index.lua +++ b/source/tools/monitor/unity/beaver/index.lua @@ -52,7 +52,7 @@ function CurlIndex:show(tReq) ### Tips - This page is rendered directly via markdown, for [guide](/guide) + This page is rendered directly via markdown, for [guide](/guide/guide.md) ]] local content2 = string.format("\n thread id is:%d\n", unistd.getpid()) local title = "welcome to visit SysAk Agent server." diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 5671ce87..1f253c64 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -34,6 +34,13 @@ function CLocalBeaver:_init_(frame, fYaml) end function CLocalBeaver:_del_() + for fd in pairs(self._cos) do + socket.shutdown(fd, socket.SHUT_RDWR) + local res = self._cffi.del_fd(self._efd, fd) + print("close fd: " .. fd) + assert(res >= 0) + end + if self._efd then self._cffi.deinit(self._efd) end @@ -42,11 +49,6 @@ function CLocalBeaver:_del_() end end -local function posixError(msg, err, errno) - local s = msg .. string.format(": %s, errno: %d", err, errno) - error(s) -end - function CLocalBeaver:_installTmo(fd) self._tmos[fd] = os.time() end @@ -57,7 +59,7 @@ function CLocalBeaver:_checkTmo() -- ! coroutine will del self._tmos cell in loop, so create a mirror table for safety local tmos = system:dictCopy(self._tmos) for fd, t in pairs(tmos) do - if now - t >= 60 then + if now - t >= 10 * 60 then local e = self._ffi.new("native_event_t") e.ev_close = 1 e.fd = fd @@ -81,25 +83,51 @@ function CLocalBeaver:_installFFI() return efd end +local function localBind(fd, tPort) + local try = 0 + local res, err, errno + + -- can reuse for time wait socket. + res, err, errno = socket.setsockopt(fd, socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); + if not res then + system:posixError("set sock opt failed."); + end + + while try < 120 do + res, err, errno = socket.bind(fd, tPort) + if res then + return 0 + elseif errno == 98 then -- port already in use? try 30s; + unistd.sleep(1) + try = try + 1 + else + break + end + end + system:posixError(string.format("bind port %d failed.", tPort.port), err, errno) +end + function CLocalBeaver:_install_fd(port, ip, backlog) local fd, res, err, errno fd, err, errno = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if fd then -- for socket - res, err, errno = socket.bind(fd, {family=socket.AF_INET, addr=ip, port=port}) - if res then -- for bind + local tPort = {family=socket.AF_INET, addr=ip, port=port} + local r, msg = pcall(localBind, fd, tPort) + if r then res, err, errno = socket.listen(fd, backlog) if res then -- for listen return fd else - posixError("socket listen failed", err, errno) + unistd.close(fd) + system:posixError("socket listen failed", err, errno) end - else -- for bind failed + else + print(msg) unistd.close(fd) - posixError("socket bind failed", err, errno) os.exit(1) end else -- socket failed - posixError("create socket failed", err, errno) + system:posixError("create socket failed", err, errno) end end @@ -120,7 +148,7 @@ function CLocalBeaver:read(fd, maxLen) return nil end else - posixError("socket recv error", err, errno) + system:posixError("socket recv error", err, errno) end else print(system:dump(e)) @@ -137,7 +165,6 @@ function CLocalBeaver:write(fd, stream) sent, err, errno = socket.send(fd, stream) if sent then if sent < #stream then -- send buffer may full - print("need to send buffer for " .. (#stream - sent)) res = self._cffi.mod_fd(self._efd, fd, 1) -- epoll write ev assert(res == 0) @@ -149,7 +176,7 @@ function CLocalBeaver:write(fd, stream) stream = string.sub(stream, sent + 1) sent, err, errno = socket.send(fd, stream) if sent == nil then - posixError("socket send error.", err, errno) + system:posixError("socket send error.", err, errno) return nil end else -- need to read ? may something error or closed. @@ -161,7 +188,7 @@ function CLocalBeaver:write(fd, stream) end return 1 else - posixError("socket send error.", err, errno) + system:posixError("socket send error.", err, errno) return nil end end @@ -211,12 +238,12 @@ function CLocalBeaver:accept(fd, e) self:co_add(nfd) self:_installTmo(nfd) else - posixError("accept new socket failed", err, errno) + system:posixError("accept new socket failed", err, errno) end end end -function CLocalBeaver:_poll(bfd, nes) +function CLocalBeaver:_pollFd(bfd, nes) for i = 0, nes.num - 1 do local e = nes.evs[i]; local fd = e.fd @@ -233,10 +260,7 @@ function CLocalBeaver:_poll(bfd, nes) self:_checkTmo() end -function CLocalBeaver:poll() - assert(self._once, "poll loop only run once time.") - self._once = false - +function CLocalBeaver:_poll() local bfd = self._bfd local efd = self._efd while true do @@ -244,17 +268,21 @@ function CLocalBeaver:poll() local res = self._cffi.poll_fds(efd, 10, nes) if res < 0 then - break + return "end poll." end - self:_poll(bfd, nes) + self:_pollFd(bfd, nes) end +end + +function CLocalBeaver:poll() + assert(self._once, "poll loop only run once time.") + self._once = false + + local _, msg = pcall(self._poll, self) + print(msg) - for fd in pairs(self._cos) do - local res = self._cffi.del_fd(self._efd, fd) - assert(res >= 0) - end return 0 end -return CLocalBeaver \ No newline at end of file +return CLocalBeaver diff --git a/source/tools/monitor/unity/beaver/url_guide.lua b/source/tools/monitor/unity/beaver/url_guide.lua index 1362b01e..fcb6cd5c 100644 --- a/source/tools/monitor/unity/beaver/url_guide.lua +++ b/source/tools/monitor/unity/beaver/url_guide.lua @@ -5,56 +5,20 @@ --- require("common.class") +local pystring = require("common.pystring") local ChttpHtml = require("httplib.httpHtml") local CurlGuide = class("CurlIndex", ChttpHtml) function CurlGuide:_init_(frame) ChttpHtml._init_(self) - self._urlCb["/guide"] = function(tReq) return self:guide(tReq) end - self._urlCb["/guide/hotplugin"] = function(tReq) return self:hotplugin(tReq) end - self._urlCb["/guide/oop"] = function(tReq) return self:oop(tReq) end - self._urlCb["/guide/pystring"] = function(tReq) return self:pystring(tReq) end - self._urlCb["/guide/webdevel"] = function(tReq) return self:webdevel(tReq) end - self._urlCb["/guide/proc_probe"] = function(tReq) return self:proc_probe(tReq) end - self._urlCb["/guide/dev_proc"] = function(tReq) return self:dev_proc(tReq) end - self:_install(frame) + self:_installRe("^/guide*", frame) + self._head = "/" -- need to strip + self._filePath = "../beaver/" end -local function loadFile(fPpath) - local path = "../beaver/guide/" .. fPpath - local f = io.open(path,"r") - local s = f:read("*all") - f:close() - return s -end - -function CurlGuide:guide(tReq) - return {title="guide", content=self:markdown(loadFile("guide.md"))} -end - -function CurlGuide:hotplugin(tReq) - return {title="hotplugin", content=self:markdown(loadFile("hotplugin.md"))} -end - -function CurlGuide:oop(tReq) - return {title="oop", content=self:markdown(loadFile("oop.md"))} -end - -function CurlGuide:pystring(tReq) - return {title="pystring", content=self:markdown(loadFile("pystring.md"))} -end - -function CurlGuide:webdevel(tReq) - return {title="webdevel", content=self:markdown(loadFile("webdevel.md"))} -end - -function CurlGuide:proc_probe(tReq) - return {title="proc and probes", content=self:markdown(loadFile("proc_probe.md"))} -end - -function CurlGuide:dev_proc(tReq) - return {title="develop proc interface.", content=self:markdown(loadFile("dev_proc.md"))} +function CurlGuide:callRe(tReq, keep) + return self:reSource(tReq, keep, self._head, self._filePath) end return CurlGuide diff --git a/source/tools/monitor/unity/beeQ/apps.c b/source/tools/monitor/unity/beeQ/apps.c index a0cdc50a..2e522bc9 100644 --- a/source/tools/monitor/unity/beeQ/apps.c +++ b/source/tools/monitor/unity/beeQ/apps.c @@ -15,22 +15,63 @@ static int sample_period = 0; extern char *g_yaml_file; -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +static int lua_traceback(lua_State *L) +{ + const char *errmsg = lua_tostring(L, -1); + lua_getglobal(L, "debug"); + lua_getfield(L, -1, "traceback"); + lua_call(L, 0, 1); + printf("%s \n%s\n", errmsg, lua_tostring(L, -1)); + return 1; +} + +int lua_reg_errFunc(lua_State *L) { + lua_pushcfunction(L, lua_traceback); + return lua_gettop(L); +} -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); +int lua_check_ret(int ret) { + switch (ret) { + case 0: + break; + case LUA_ERRRUN: + printf("lua runtime error.\n"); + break; + case LUA_ERRMEM: + printf("lua memory error.\n"); + case LUA_ERRERR: + printf("lua exec error.\n"); + case LUA_ERRSYNTAX: + printf("file syntax error.\n"); + case LUA_ERRFILE: + printf("load lua file error.\n"); + default: + printf("bad res for %d\n", ret); + exit(1); + } + return ret; } -static int call_init(lua_State *L) { +int lua_load_do_file(lua_State *L, const char* path) { + int err_func = lua_gettop(L); + int ret; + + ret = luaL_loadfile(L, path); + if (ret) { + return lua_check_ret(ret); + } + ret = lua_pcall(L, 0, LUA_MULTRET, err_func); + return lua_check_ret(ret); +} + +static int call_init(lua_State *L, int err_func) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -56,7 +97,7 @@ static int call_init(lua_State *L) { static lua_State * app_recv_init(void) { int ret; - + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); if (L == NULL) { @@ -65,19 +106,14 @@ static lua_State * app_recv_init(void) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "bees.lua"); + ret = lua_load_do_file(L, "bees.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } - ret = call_init(L); + ret = call_init(L, err_func); if (ret < 0) { goto endCall; } @@ -112,6 +148,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { int lret; lua_State *L = (lua_State *)(q->qarg); char *body; + int err_func; if (counter != sighup_counter) { // check counter for signal. lua_close(L); @@ -134,13 +171,13 @@ int app_recv_proc(void* msg, struct beeQ* q) { goto endMem; } memcpy(body, &pMsg->body[0], len); + err_func = lua_gettop(L); lua_getglobal(L, "proc"); lua_pushlstring(L, body, len); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); free(body); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -165,6 +202,7 @@ int app_recv_proc(void* msg, struct beeQ* q) { endReturn: endCall: free(msg); + exit(1); return ret; } @@ -190,6 +228,7 @@ int collector_qout(lua_State *L) { static lua_State * app_collector_init(void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -199,15 +238,10 @@ static lua_State * app_collector_init(void* q, void* proto_q) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "collectors.lua"); + ret = lua_load_do_file(L, "collectors.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } @@ -218,10 +252,9 @@ static lua_State * app_collector_init(void* q, void* proto_q) { lua_pushlightuserdata(L, q); lua_pushlightuserdata(L, proto_q); lua_pushstring(L, g_yaml_file); - ret = lua_pcall(L, 3, 1, 0); + ret = lua_pcall(L, 3, 1, err_func); if (ret < 0) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } @@ -252,6 +285,7 @@ static lua_State * app_collector_init(void* q, void* proto_q) { static int app_collector_work(lua_State **pL, void* q, void* proto_q) { int ret; + int err_func; lua_Number lret; static int counter = 0; @@ -268,12 +302,12 @@ static int app_collector_work(lua_State **pL, void* q, void* proto_q) { counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "work"); lua_pushinteger(L, sample_period); - ret = lua_pcall(L, 1, 1, 0); + ret = lua_pcall(L, 1, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/outline/outline.c b/source/tools/monitor/unity/collector/outline/outline.c index 753049af..eae253a4 100644 --- a/source/tools/monitor/unity/collector/outline/outline.c +++ b/source/tools/monitor/unity/collector/outline/outline.c @@ -5,23 +5,19 @@ #include "outline.h" #include -LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, void* q, char *fYaml) { +static int call_init(lua_State *L, int err_func, void* q, char *fYaml) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, q); lua_pushstring(L, fYaml); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("luaL_call init func error"); - report_lua_failed(L); goto endCall; } @@ -48,6 +44,7 @@ static int call_init(lua_State *L, void* q, char *fYaml) { extern int collector_qout(lua_State *L); static lua_State * pipe_init(void* q, char *fYaml) { int ret; + int err_func; lua_Number lret; /* create a state and load standard library. */ @@ -57,21 +54,17 @@ static lua_State * pipe_init(void* q, char *fYaml) { goto endNew; } luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "outline.lua"); + ret = lua_load_do_file(L, "outline.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, q, fYaml); - if (ret < 0) { + ret = call_init(L, err_func, q, fYaml); + if (ret) { + lua_check_ret(ret); goto endCall; } return L; @@ -85,13 +78,14 @@ static lua_State * pipe_init(void* q, char *fYaml) { static int work(lua_State *L) { int ret; + int err_func; lua_Number lret; + err_func = lua_gettop(L); lua_getglobal(L, "work"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 43eb74f9..a1f78315 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -120,3 +120,8 @@ metrics: head: value help: "loadavg of system from /proc/loadavg" type: "gauge" + - title: sysak_io_burst + from: io_burst + head: value + help: "io burst value." + type: "gauge" \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/proto_sender.c b/source/tools/monitor/unity/collector/plugin/proto_sender.c index bdf1397e..a05211c1 100644 --- a/source/tools/monitor/unity/collector/plugin/proto_sender.c +++ b/source/tools/monitor/unity/collector/plugin/proto_sender.c @@ -10,23 +10,19 @@ #define PROTO_QUEUE_SIZE 64 #define gettidv1() syscall(__NR_gettid) -LUALIB_API void luaL_traceback(lua_State *L, lua_State *L1, const char *msg, int level); +extern int lua_reg_errFunc(lua_State *L); +extern int lua_check_ret(int ret); +int lua_load_do_file(lua_State *L, const char* path); -static void report_lua_failed(lua_State *L) { - fprintf(stderr, "\nFATAL ERROR:%s\n\n", lua_tostring(L, -1)); -} - -static int call_init(lua_State *L, struct beeQ* pushQ) { +static int call_init(lua_State *L, int err_func, struct beeQ* pushQ) { int ret; lua_Number lret; lua_getglobal(L, "init"); lua_pushlightuserdata(L, pushQ); lua_pushinteger(L, (int)gettidv1()); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("proto_sender lua init func error"); - report_lua_failed(L); goto endCall; } @@ -52,6 +48,7 @@ static int call_init(lua_State *L, struct beeQ* pushQ) { extern int collector_qout(lua_State *L); lua_State * proto_sender_lua(struct beeQ* pushQ) { int ret; + int err_func; /* create a state and load standard library. */ lua_State *L = luaL_newstate(); @@ -61,20 +58,15 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { } /* opens all standard Lua libraries into the given state. */ luaL_openlibs(L); + err_func = lua_reg_errFunc(L); - ret = luaL_dofile(L, "proto_send.lua"); + ret = lua_load_do_file(L, "proto_send.lua"); if (ret) { - const char *msg = lua_tostring(L, -1); - perror("luaL_dofile error"); - if (msg) { - luaL_traceback(L, L, msg, 0); - fprintf(stderr, "FATAL ERROR:%s\n\n", msg); - } goto endLoad; } lua_register(L, "collector_qout", collector_qout); - ret = call_init(L, pushQ); + ret = call_init(L, err_func, pushQ); if (ret < 0) { goto endCall; } @@ -88,13 +80,13 @@ lua_State * proto_sender_lua(struct beeQ* pushQ) { struct beeQ* proto_que(lua_State *L) { int ret; + int err_func = lua_gettop(L); struct beeQ* que; lua_getglobal(L, "que"); - ret = lua_pcall(L, 0, 1, 0); + ret = lua_pcall(L, 0, 1, err_func); if (ret) { - perror("proto_que lua que func error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } if (!lua_isuserdata(L, -1)) { // check @@ -120,6 +112,7 @@ struct beeQ* proto_que(lua_State *L) { extern volatile int sighup_counter; int proto_send_proc(void* msg, struct beeQ* q) { int ret = 0; + int err_func; struct unity_lines *lines = (struct unity_lines *)msg; int num = lines->num; struct unity_line * pLine = lines->line; @@ -139,14 +132,14 @@ int proto_send_proc(void* msg, struct beeQ* q) { q->qarg = L; counter = sighup_counter; } + err_func = lua_gettop(L); lua_getglobal(L, "send"); lua_pushnumber(L, num); lua_pushlightuserdata(L, pLine); - ret = lua_pcall(L, 2, 1, 0); + ret = lua_pcall(L, 2, 1, err_func); if (ret) { - perror("lua call error"); - report_lua_failed(L); + lua_check_ret(ret); goto endCall; } diff --git a/source/tools/monitor/unity/common/lmd.lua b/source/tools/monitor/unity/common/lmd.lua index 67a0e1bf..685c66d1 100644 --- a/source/tools/monitor/unity/common/lmd.lua +++ b/source/tools/monitor/unity/common/lmd.lua @@ -10,6 +10,7 @@ require("common.class") local pystring = require("common.pystring") local Clmd = class("lmd") +local srcPath = "" function Clmd:_init_() self._escs = '\\`*_{}[]()>#+-.!' @@ -98,9 +99,24 @@ local function pCode(s) end end +local function images(s) + local name, link = unpack(pystring:split(s, "](", 1)) + name = string.sub(name, 3) -- ![]() + link = string.sub(link, 1, -2) + + if string.sub(name, -1, -1) == "\\" then + return s + end + if string.sub(link, -1, -1) == "\\" then + return s + end + local path = srcPath .. link + return string.format('%s', path, name) +end + local function links(s) local name, link = unpack(pystring:split(s, "](", 1)) - name = string.sub(name, 2) + name = string.sub(name, 2) -- []() link = string.sub(link, 1, -2) if string.sub(name, -1, -1) == "\\" then return s @@ -111,6 +127,10 @@ local function links(s) return string.format('%s', link, name) end +local function pImages(s) + return string.gsub(s, "!%[.-%]%(.-%)", function(s) return images(s) end) +end + local function pLink(s) return string.gsub(s, "%[.-%]%(.-%)", function(s) return links(s) end) end @@ -353,6 +373,7 @@ function Clmd:seg(s) s = pItalic(s) s = pDelete(s) s = pCode(s) + s = pImages(s) s = pLink(s) return pEscape(s) end @@ -362,11 +383,12 @@ function Clmd:pSeg(s) return pystring:join("", {"

", pEnter(s), "

"}) end -function Clmd:toHtml(md) +function Clmd:toHtml(md, path) local mds = pystring:split(md, '\n') local res = {} local len = #mds local stop = 0 + srcPath = path or "" for i = 1, len do local line = mds[i] diff --git a/source/tools/monitor/unity/common/system.lua b/source/tools/monitor/unity/common/system.lua index 1c083430..68dfa7f5 100644 --- a/source/tools/monitor/unity/common/system.lua +++ b/source/tools/monitor/unity/common/system.lua @@ -125,4 +125,9 @@ function system:parseYaml(fYaml) return lyaml.load(s) end +function system:posixError(msg, err, errno) + local s = msg .. string.format(": %s, errno: %d", err, errno) + error(s) +end + return system \ No newline at end of file diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 7ddbc24e..432b622e 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -5,6 +5,7 @@ --- require("common.class") +local system = require("common.system") local ChttpComm = require("httplib.httpComm") local ChttpBase = class("ChttpBase", ChttpComm) @@ -20,22 +21,35 @@ function ChttpBase:_install(frame) end end +function ChttpBase:_installRe(path, frame) + frame:registerRe(path, self) +end + function ChttpBase:echo(tRet, keep) error("ChttpBase:echo is a virtual function.") end local function checkKeep(tReq) local conn = tReq.header["connection"] - if conn and string.lower(conn) == "keep-alive" then - return true + if conn and string.lower(conn) == "close" then + return false end - return false + return true end function ChttpBase:call(tReq) + local keep = checkKeep(tReq) local tRet = self._urlCb[tReq.path](tReq) + local res = self:echo(tRet, keep) + + return res, keep +end + +function ChttpBase:calls(tReq) local keep = checkKeep(tReq) - return self:echo(tRet, keep), keep + local res = self:callRe(tReq, keep) + + return res, keep end return ChttpBase diff --git a/source/tools/monitor/unity/httplib/httpHtml.lua b/source/tools/monitor/unity/httplib/httpHtml.lua index 5205af65..34a9a9ea 100644 --- a/source/tools/monitor/unity/httplib/httpHtml.lua +++ b/source/tools/monitor/unity/httplib/httpHtml.lua @@ -5,13 +5,35 @@ --- require("common.class") +local unistd = require("posix.unistd") local pystring = require("common.pystring") +local system = require("common.system") local ChttpBase = require("httplib.httpBase") local ChttpHtml = class("ChttpHtml", ChttpBase) function ChttpHtml:_init_(frame) ChttpBase._init_(self) + + self._reCb = { + md = function(s, keep, suffix) return self:renderMd(s, keep, suffix) end, + html = function(s, keep, suffix) return self:renderHtml(s, keep, suffix) end, + jpg = function(s, keep, suffix) return self:renderImage(s, keep, "jpeg") end, + jpeg = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + gif = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + png = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + bmp = function(s, keep, suffix) return self:renderImage(s, keep, suffix) end, + txt = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + text = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + log = function(s, keep, suffix) return self:renderText(s, keep, suffix) end, + } +end + +local function loadFile(fPpath) + local f = io.open(fPpath,"r") + local s = f:read("*all") + f:close() + return s end function ChttpHtml:markdown(text) @@ -19,6 +41,43 @@ function ChttpHtml:markdown(text) return md:toHtml(text) end +function ChttpHtml:renderMd(s, keep, suffix) + local tmd = self:markdown(s) + local tRet = { + title = "markdown document render from beaver.", + content = tmd, + cType = "text/html", + } + return self:echo(tRet, keep) +end + +function ChttpHtml:renderHtml(s, keep, suffix) + local cType = "text/html" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderText(s, keep, suffix) + local cType = "text/plain" + return self:pack(cType, keep, s) +end + +function ChttpHtml:renderImage(s, keep, suffix) + local cType = "image/" .. suffix + return self:pack(cType, keep, s) +end + +function ChttpHtml:reSource(tReq, keep, head, srcPath) + local path = tReq.path + path = srcPath .. pystring:lstrip(path, head) + if unistd.access(path) then + local s = loadFile(path) + local _, suffix = unpack(pystring:rsplit(path, ".", 1)) + if system:keyIsIn(self._reCb, suffix) then + return self._reCb[suffix](s, keep, suffix) + end + end +end + local function htmlPack(title, content) local h1 = [[ @@ -41,16 +100,22 @@ local function htmlPack(title, content) return pystring:join("", bodies) end -function ChttpHtml:echo(tRet, keep) +function ChttpHtml:pack(cType, keep, body) local stat = self:packStat(200) local tHead = { - ["Content-Type"] = "text/html", + ["Content-Type"] = cType, ["Connection"] = (keep and "keep-alive") or "close" } - local body = htmlPack(tRet.title, tRet.content) local headers = self:packHeaders(tHead, #body) local tHttp = {stat, headers, body} return pystring:join("\r\n", tHttp) end +function ChttpHtml:echo(tRet, keep) + local cType = tRet.type or "text/html" + local body = htmlPack(tRet.title, tRet.content) + + return self:pack(cType, keep, body) +end + return ChttpHtml diff --git a/source/tools/monitor/unity/test/beaver/walkDir.lua b/source/tools/monitor/unity/test/beaver/walkDir.lua new file mode 100644 index 00000000..27a0fd0d --- /dev/null +++ b/source/tools/monitor/unity/test/beaver/walkDir.lua @@ -0,0 +1,43 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/2/18 12:13 PM +--- + +package.path = package.path .. ";../../?.lua;" +local dirent = require("posix.dirent") +local sysStat = require("posix.sys.stat") +local system = require("common.system") + +local function join(dir, fName) + local paths = {dir, fName} + return table.concat(paths, "/") +end + +local function walk(orig, dir, tbl) + local ls = dirent.dir(dir) + local len = string.len(orig) + for _, l in ipairs(ls) do + if l == ".." or l == '.' then + goto continue + end + local path = join(dir, l) + local stat, err, errno = sysStat.stat(path) + if stat then + local mode = stat.st_mode + if sysStat.S_ISDIR(mode) > 0 then + walk(orig, path, tbl) + elseif sysStat.S_ISREG(mode) > 0 then + table.insert(tbl, string.sub(path, len + 2)) + end + else + system:posixError(string.format("bad access to file %s", path), err, errno) + end + ::continue:: + end +end + +local tbl = {} +local dir = "../../beaver/guide" +walk(dir, dir, tbl) +print(system:dump(tbl)) diff --git a/source/tools/monitor/unity/test/unix/pyunix.py b/source/tools/monitor/unity/test/unix/pyunix.py new file mode 100644 index 00000000..c215808a --- /dev/null +++ b/source/tools/monitor/unity/test/unix/pyunix.py @@ -0,0 +1,28 @@ +import os +import time +import socket + +PIPE_PATH = "/tmp/sysom" +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = PIPE_PATH + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check Netinfo is running.") + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut() + i = 10 + while True: + nf.puts('io_burst,disk=/dev/vda1 limit=10.0,max=%d,log="io log burst"' % i) + i += 1 + time.sleep(5) \ No newline at end of file diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index b808489b..d5efb6fd 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -268,28 +268,31 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday end function CfoxTSDB:qlast(last, ms) + assert(last < 24 * 60 * 60) + local now = self:get_us() local date = self:getDateFrom_us(now) local beg = now - last * 1e6; - if self._man then -- has setup - if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day - return self:query(beg, now, ms) - else - self:_del_() -- destroy old manager - if self:_setupRead(now) ~= 0 then -- try to create new - return ms - else - return self:query(beg, now, ms) - end - end - else + if not self._man then -- check _man is already installed. if self:_setupRead(now) ~= 0 then -- try to create new return ms - else - return self:query(beg, now, ms) end end + + if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day + return self:query(beg, now, ms) + else + local dStop = self:getDateFrom_us(now) + + local beg1 = beg + local beg2 = self.cffi.make_stamp(dStop) + local now1 = beg2 - 1 + local now2 = now + + ms = self:query(beg1, now1, ms) + return self:query(beg2, now2, ms) + end end function CfoxTSDB:qDay(start, stop, ms, tbls, budget) -- Gitee From e0a88f23d5246673ff64db8d7086181b50bd4ab5 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Tue, 21 Feb 2023 01:08:07 +0800 Subject: [PATCH 03/21] http can brower by markdown, html, text, report failure info for every bad call. --- source/tools/monitor/unity/beaver/url_export_raw.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/source/tools/monitor/unity/beaver/url_export_raw.lua b/source/tools/monitor/unity/beaver/url_export_raw.lua index b6c904bf..2629fba8 100644 --- a/source/tools/monitor/unity/beaver/url_export_raw.lua +++ b/source/tools/monitor/unity/beaver/url_export_raw.lua @@ -14,6 +14,7 @@ function CurlExportRaw:_init_(frame, export) self._export = export self._urlCb["/export/metrics"] = function(tReq) return self:show(tReq) end + self._urlCb["/metrics"] = function(tReq) return self:show(tReq) end self:_install(frame) end -- Gitee From 938b52d774064bf8eebf3c3da80229d5e9ddac05 Mon Sep 17 00:00:00 2001 From: zhilan Date: Tue, 21 Feb 2023 10:29:37 +0800 Subject: [PATCH 04/21] unity: use local var in proc_buddyinfo --- source/tools/monitor/unity/collector/proc_buddyinfo.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/proc_buddyinfo.lua b/source/tools/monitor/unity/collector/proc_buddyinfo.lua index 8d85696e..f2fc8c9b 100644 --- a/source/tools/monitor/unity/collector/proc_buddyinfo.lua +++ b/source/tools/monitor/unity/collector/proc_buddyinfo.lua @@ -17,10 +17,10 @@ end function CprocBuddyinfo:proc(elapsed, lines) CvProc.proc(self) - buddyinfo = {} + local buddyinfo = {} for line in io.lines(self.pFile) do if string.find(line,"Normal") then - subline = pystring:split(line,"Normal",1)[2] + local subline = pystring:split(line,"Normal",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end @@ -31,7 +31,7 @@ function CprocBuddyinfo:proc(elapsed, lines) if not buddyinfo then for line in io.lines(self.pFile) do if string.find(line,"DMA32") then - subline = pystring:split(line,"DMA32",1)[2] + local subline = pystring:split(line,"DMA32",1)[2] for num in string.gmatch(subline, "%d+") do table.insert(buddyinfo,tonumber(num)) end -- Gitee From cf6aef8156f5e20908d78071e78640374e86f05b Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 22 Feb 2023 10:54:57 +0800 Subject: [PATCH 05/21] ioMonitor: Add ioMonitor to unity ioMonitor dependon unity, You must start ioMonitor after the unity is started and just as follow: sysak ioMonitor -y [plugin.yaml of unity] Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/Makefile | 4 + source/tools/monitor/ioMonitor/README.md | 2 + .../tools/monitor/ioMonitor/ioMon/__init__.py | 4 + .../monitor/ioMonitor/ioMon/displayClass.py | 510 ++++++++++++++++++ .../ioMonitor/ioMon/exceptCheckClass.py | 187 +++++++ .../ioMonitor/ioMon/exceptDiagnoseClass.py | 267 +++++++++ .../monitor/ioMonitor/ioMon/ioMonCfgClass.py | 137 +++++ .../monitor/ioMonitor/ioMon/ioMonitorClass.py | 369 +++++++++++++ .../monitor/ioMonitor/ioMon/ioMonitorMain.py | 80 +++ source/tools/monitor/ioMonitor/ioMon/nfPut.py | 37 ++ .../monitor/ioMonitor/ioMon/tools/__init__.py | 4 + .../ioMon/tools/iofstool/__init__.py | 4 + .../ioMonitor/ioMon/tools/iofstool/common.py | 129 +++++ .../ioMon/tools/iofstool/diskstatClass.py | 219 ++++++++ .../ioMon/tools/iofstool/fsstatClass.py | 418 ++++++++++++++ .../ioMon/tools/iofstool/iofsstat.py | 107 ++++ .../ioMon/tools/iofstool/iostatClass.py | 244 +++++++++ .../ioMon/tools/iofstool/promiscClass.py | 215 ++++++++ .../ioMon/tools/iowaitstat/__init__.py | 4 + .../ioMon/tools/iowaitstat/iowaitstat.py | 354 ++++++++++++ source/tools/monitor/ioMonitor/ioMonitor.sh | 11 + .../tools/monitor/unity/collector/plugin.yaml | 15 + 22 files changed, 3321 insertions(+) create mode 100755 source/tools/monitor/ioMonitor/Makefile create mode 100644 source/tools/monitor/ioMonitor/README.md create mode 100644 source/tools/monitor/ioMonitor/ioMon/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/displayClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/nfPut.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/__init__.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py create mode 100644 source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py create mode 100755 source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py create mode 100755 source/tools/monitor/ioMonitor/ioMonitor.sh diff --git a/source/tools/monitor/ioMonitor/Makefile b/source/tools/monitor/ioMonitor/Makefile new file mode 100755 index 00000000..1345670e --- /dev/null +++ b/source/tools/monitor/ioMonitor/Makefile @@ -0,0 +1,4 @@ +mods = ioMon +target := ioMonitor + +include $(SRC)/mk/sh.mk diff --git a/source/tools/monitor/ioMonitor/README.md b/source/tools/monitor/ioMonitor/README.md new file mode 100644 index 00000000..4856ff90 --- /dev/null +++ b/source/tools/monitor/ioMonitor/README.md @@ -0,0 +1,2 @@ +# 功能说明 +监控服务主程序，收集系统监控指标，支持查看历史监控数据 diff --git a/source/tools/monitor/ioMonitor/ioMon/__init__.py b/source/tools/monitor/ioMonitor/ioMon/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/displayClass.py b/source/tools/monitor/ioMonitor/ioMon/displayClass.py new file mode 100755 index 00000000..cc51352e --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/displayClass.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import string +import time +import re +import json +import threading +from collections import OrderedDict +from nfPut import CnfPut + + +def bwToValue(bw): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + if str(bw) == '0': + return 0 + for i in range(5, -1, -1): + if units[i] in bw: + return float(bw.split(units[i])[0]) * pow(1024, i) + + +def humConvert(value): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + for i in range(len(units)): + if (value / size) < 1: + return "%.1f%s/s" % (value, units[i]) + value = value / size + + +def iolatencyResultReport(*argvs): + result = [] + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + ioburst = False + nfPrefix = [] + iolatStartT = statusReportDicts['iolatency']['startT'] + iolatEndT = statusReportDicts['iolatency']['endT'] + ioutilStartT = statusReportDicts['ioutil']['startT'] + ioutilEndT = statusReportDicts['ioutil']['endT'] + lastIOburstT = statusReportDicts['iolatency']['lastIOburstT'] + + # If IO burst occurs in the short term(first 180 secs or next 60secs) or + # during IO delay diagnosis, it should be considered as one of + # the delay factors + if iolatStartT - lastIOburstT < 300 or \ + (ioutilStartT >= (iolatStartT - 300) and ioutilEndT <= (iolatEndT + 60)): + statusReportDicts['iolatency']['lastIOburstT'] = iolatStartT + ioburst = True + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 | '\ + 'xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log.stat'): + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + else: + return + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + delays = sorted(ds['delays'], + key=lambda e: (float(e['percent'].strip('%'))), + reverse=True) + maxDelayComp = delays[0]['component'] + maxDelayPercent = float(delays[0]['percent'].strip('%')) + avgLat = format(sum([d['avg'] for d in delays])/1000.0, '.3f') + diagret = 'diagret=\"IO delay(AVG %sms) detected in %s\"' % ( + avgLat, str(ds['diskname'])) + nfPrefix.append(',diag_type=IO-Delay,devname='+str(ds['diskname'])) + + if ioburst and {maxDelayComp, delays[1]['component']}.issubset( + ['disk', 'os(block)']): + if (delays[0]['avg'] / delays[1]['avg']) < 10: + suggest = 'solution=\"reduce IO pressure. Refer to the '\ + 'diagnosis of IO-Burst and optimize some tasks\"' + diskIdx = 0 + if maxDelayComp == 'os(block)': + diskIdx = 1 + reason = ( + 'reason=\"IO burst occurs, too mang IO backlogs'\ + '(disk avg/max lat:%s/%s ms, lat percent:%s,'\ + ' OS dispatch avg/max lat:%s/%s ms, lat percent:%s\"' % + (str(delays[diskIdx]['avg'] / 1000.000), + str(delays[diskIdx]['max'] / 1000.000), + str(delays[diskIdx]['percent']), + str(delays[1 - diskIdx]['avg'] / 1000.000), + str(delays[1 - diskIdx]['max'] / 1000.000), + str(delays[1 - diskIdx]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + else: + statusReportDicts['iolatency']['lastIOburstT'] = lastIOburstT + + suggest = 'solution=\"Please ask the OS kernel expert\"' + maxDelayLog = 'avg/max lat:%s/%s ms, lat percent:%s' %( + str(delays[0]['avg']/1000.000), + str(delays[0]['max']/1000.000), + str(delays[0]['percent'])) + if maxDelayComp == 'disk': + reason = ( + 'reason=\"Disk delay(processing IO slowly, %s)\"' %(maxDelayLog)) + suggest = 'solution=\"Please confirm whether the disk is normal\"' + elif maxDelayComp == 'os(block)': + if delays[1]['component'] == 'disk' and \ + float(delays[1]['percent'].strip('%')) > 20: + with open(argvs[0]+'/resultCons.log') as logF: + data = filter(lambda l : 'F' in l, logF.readlines()) + flushIO = False + if len(data) > 0: + for d in data: + if 'F' in d.split()[-6]: + flushIO = True + break + if flushIO: + suggest = ( + 'Disable flush IO dispatch(echo \"write through\" > '\ + '/sys/class/block/%s/queue/write_cache;'\ + 'echo 0 > /sys/class/block/%s/queue/fua)}' % ( + str(ds['diskname']), str(ds['diskname']))) + suggest += '; Notes: Flush IO is a special instruction to '\ + 'ensure that data is stored persistently on the disk '\ + 'in time, and not saved in the internal cache of the disk.'\ + ' Before disabling, please confirm with the disk FAE '\ + '\"Whether it is necessary to rely on the software to issue'\ + ' flush instructions to ensure data persistent storage\",'\ + ' And avoid data loss due to crash or disk power down' + suggest = 'solution=\"'+suggest+'\"' + else: + suggest = 'solution=\"Please confirm whether the disk is normal\"' + reason = ( + 'reason=\"Disk delay(processing %s slowly, avg/max lat:'\ + '%s/%s ms, lat percent:%s)\"' %( + 'Flush IO' if flushIO else 'IO', + str(delays[1]['avg']/1000.000), + str(delays[1]['max']/1000.000), + str(delays[1]['percent']))) + result.append(diagret+','+reason+','+suggest) + continue + reason = ( + 'reason=\"OS delay(Issuing IO slowly at os(block), %s)\"' %( + maxDelayLog)) + else: + reason = ( + 'reason=\"OS delay(processing IO slowly at %s, %s)\"' %( + str(maxDelayComp), maxDelayLog)) + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + # print(e+'\n') + #nf.put(nfPutPrefix, p+' '+e) + nf.puts(nfPutPrefix+p+' '+e) + statusReportDicts['iolatency']['valid'] = True + + +def iohangResultReport(*argvs): + abnormalDicts={} + firstioDicts={} + result=[] + nf=argvs[1] + nfPutPrefix=str(argvs[2]) + statusReportDicts = argvs[3] + nfPrefix=[] + + os.system('ls -rtd '+argvs[0]+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]+'/result.log'): + with open(argvs[0]+'/result.log') as logF: + data=logF.readline() + else: + return + try: + stat=json.loads(data, object_pairs_hook = OrderedDict) + except Exception: + return + + for ds in stat['summary']: + maxDelay = 0 + hungIO = None + if ds['diskname'] not in abnormalDicts.keys(): + abnormalDicts.setdefault(ds['diskname'], {}) + firstioDicts.setdefault( + ds['diskname'], + {'time':0, 'iotype':0, 'sector':0}) + for hi in ds['hung ios']: + key=hi['abnormal'].split('hang')[0] + delay = float(hi['abnormal'].split('hang')[1].split()[0]) + if delay > maxDelay: + maxDelay = delay + hungIO = hi + if key not in abnormalDicts[ds['diskname']].keys(): + abnormalDicts[ds['diskname']].setdefault(key, 0) + abnormalDicts[ds['diskname']][key] += 1 + t = hungIO['time'].split('.')[0] + tStamp = float(time.mktime(time.strptime(t,'%Y-%m-%d %H:%M:%S'))) + tStamp -= maxDelay + firstioDicts[ds['diskname']]['time'] = \ + time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(tStamp+8*3600)) + firstioDicts[ds['diskname']]['iotype'] = hungIO['iotype'] + firstioDicts[ds['diskname']]['sector'] = hungIO['sector'] + for diskname, val in abnormalDicts.items(): + abnormalDicts[diskname] = OrderedDict( + sorted(val.items(), key=lambda e: e[1], reverse=True)) + + with open(argvs[0]+'/result.log.stat') as logF: + data = logF.readline() + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + + for ds in stat['summary']: + hungIOS = sorted(ds['hung ios'], + key = lambda e: (float(e['percent'].strip('%'))), + reverse = True) + maxDelayComp=hungIOS[0]['component'] + maxDelayPercent=float(hungIOS[0]['percent'].strip('%')) + maxDelay=format(hungIOS[0]['max']/1000.0, '.3f') + diagret='diagret=\"IO hang %sms detected in %s' % ( + maxDelay, ds['diskname'])+'\"' + nfPrefix.append(',diag_type=IO-Hang,devname='+str(ds['diskname'])) + for key in abnormalDicts[ds['diskname']].keys(): + if maxDelayComp in key: + detail = str( + ''.join(re.findall(re.compile(r'[(](.*?)[)]', re.S), key))) + break + reason = ('reason=\"%s hang(%s, avg/max delay:%s/%s ms), first hang['\ + 'time:%s, iotype:%s, sector:%d]\"' %( + maxDelayComp, detail, + str(hungIOS[0]['avg']/1000.000), + str(hungIOS[0]['max']/1000.000), + firstioDicts[ds['diskname']]['time'], + firstioDicts[ds['diskname']]['iotype'], + firstioDicts[ds['diskname']]['sector'])) + if maxDelayComp == 'Disk' or maxDelayComp == 'OS': + suggest = 'solution=\"Please confirm whether the disk is normal\"' + if maxDelayComp == 'OS': + suggest = 'solution=\"Please ask the OS kernel expert\"' + result.append(diagret+','+reason+','+suggest) + + for e, p in zip(result, nfPrefix): + nf.puts(nfPutPrefix+p+' '+e) + #nf.put(nfPutPrefix, p+' '+e) + statusReportDicts['iohang']['valid'] = True + + +def ioutilDataParse(data, resultInfo): + tUnit = None + totalBw = totalIops = 0 + for ds in data['mstats']: + iops = ds['iops_rd'] + ds['iops_wr'] + bps = bwToValue(ds['bps_wr']) + bwToValue(ds['bps_rd']) + totalBw += bps + totalIops += iops + key = ds['comm']+':'+ds['pid']+':'+ds['cid'][0:20]+':'+ds['device'] + if not tUnit: + if ds['bps_wr'] != '0': + tUnit = ds['bps_wr'].split('/')[1] + else: + tUnit = ds['bps_rd'].split('/')[1] + if key not in resultInfo.keys(): + resultInfo.setdefault(key, + {'disk':ds['device'], 'maxIops':0, 'maxBps':0, 'file':ds['file']}) + resultInfo[key]['maxBps'] = max(bps, resultInfo[key]['maxBps']) + resultInfo[key]['maxIops'] = max(iops, resultInfo[key]['maxIops']) + if resultInfo[key]['maxBps'] != bps or resultInfo[key]['maxIops'] != iops: + resultInfo[key]['file'] = ds['file'] + if 'bufferio' in resultInfo.keys(): + del resultInfo[key]['bufferio'] + if 'bufferio' in ds.keys() and 'bufferio' not in resultInfo[key].keys(): + resultInfo[key].setdefault('bufferio', ds['bufferio']) + return totalIops,totalBw,tUnit + + +def ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret): + top = 1 + suggestPS = reason = '' + resultInfo = \ + sorted(resultInfo.items(), key=lambda e: e[1]['maxBps'], reverse=True) + for key, val in resultInfo: + if val['maxIops'] < 50 or val['maxBps'] < 1024 * 1024 * 5: + continue + file = ', target file:'+str(val['file']) if val['file'] != '-' else '' + if 'kworker' in str(key): + kTasklist = [] + if 'bufferio' in val.keys(): + for i in val["bufferio"]: + if 'KB' in i["Wrbw"]: + continue + kTasklist.append(i['task']) + file += ('%s Wrbw %s disk %s file %s;' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + if len(kTasklist): + file = '(Write bio from: '+file+')' + if top == 1: + suggestPS = '(Found \'kworker\' flush dirty pages, Try to reduce'\ + ' the buffer-IO write?%s or check the config /proc/sys/vm/'\ + '{dirty_ratio,dirty_background_ratio} too small?)' %( + '('+';'.join(kTasklist)+')' if len(kTasklist) else '') + maxBps = humConvert(val['maxBps']).replace('s', tUnit) + reason += ('%d. task[%s], access disk %s with iops:%s, bps:%s%s; ' %( + top, str(key.rsplit(':',1)[0]), str(val['disk']), + str(val['maxIops']), maxBps, file)) + if top == 1 and suggestPS == '': + suggestPS = '(Found task \'%s\')' %(str(key.rsplit(':',1)[0])) + top += 1 + suggest = \ + 'Optimize the tasks that contributes the most IO flow%s' % suggestPS + putIdx = ',diag_type=IO-Burst ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + if reason != '': + nf.puts(nfPutPrefix+putIdx+putField) + # print(prefix+reason+suggest+'\n') + + +def ioutilResultReport(*argvs): + resultInfo= {} + nf= argvs[1] + nfPutPrefix= str(argvs[2]) + statusReportDicts = argvs[3] + totalBw = 0 + maxIops = maxBw = 0 + minIops = minBw = sys.maxsize + tUnit = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook =OrderedDict) + except Exception: + return + iops,bw,tUnit = ioutilDataParse(stat, resultInfo) + maxIops = max(maxIops, iops) + minIops = min(minIops, iops) + maxBw = max(maxBw, bw) + minBw = min(minBw, bw) + totalBw += bw + if totalBw < 1024 * 1024 * 10: + return + + if resultInfo: + content = 'Iops:'+str(minIops)+'~'+str(maxIops)+\ + ', Bps:'+humConvert(minBw).replace('s', tUnit)+\ + '~'+humConvert(maxBw).replace('s', tUnit) + diagret = 'IO-Burst('+content+') detected' + ioutilReport(nf, nfPutPrefix, resultInfo, tUnit, diagret) + statusReportDicts['ioutil']['valid'] = True + + +def iowaitDataParse(data, resultInfo): + unkownDisable = False + for io in data['iowait']: + if 'many dirty' in io['reason'] or 'queue full' in io['reason']: + unkownDisable = True + if 'Unkown' in io['reason'] and unkownDisable == True: + continue + key = io['comm']+':'+io['tgid']+':'+io['pid'] + if key not in resultInfo.keys(): + resultInfo.setdefault( + key, {'timeout': 0, 'maxIowait': 0, 'reason': ''}) + if float(io['iowait']) > float(resultInfo[key]['maxIowait']): + resultInfo[key]['maxIowait'] = io['iowait'] + resultInfo[key]['timeout'] = io['timeout'] + resultInfo[key]['reason'] = io['reason'] + return data['global iowait'],unkownDisable + + +def iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret): + top = 0 + reason = '' + resDicts = { + 'Too many dirty pages':False, + 'Device queue full':False, + 'Ioscheduler queue full':False} + + for key, val in resultInfo.items(): + if unkownDisable == True and 'Unkown' in val['reason']: + del resultInfo[key] + + resultInfo = OrderedDict( + sorted(resultInfo.items(), key=lambda e: float(e[1]['maxIowait']), + reverse=True)[:3]) + for key, val in resultInfo.items(): + if unkownDisable == True: + resDicts[val['reason']] = True + top += 1 + reason += ( + '%d. task[%s], wait %sms, contribute iowait %s due to \'%s\'; ' %( + top, str(key), str(val['timeout']), str(val['maxIowait'])+'%', + str(val['reason']))) + + if unkownDisable == True: + if resDicts['Too many dirty pages'] == True: + suggest = 'Reduce io-write pressure or Adjust /proc/sys/vm/'\ + '{dirty_ratio,dirty_bytes} larger carefully' + else: + if resDicts['Device queue full'] and resDicts['Ioscheduler queue full']: + suggest = \ + 'Device queue full -> Disk busy due to disk queue full, '\ + 'Please reduce io pressure;'\ + 'Ioscheduler queue full -> Io scheduler busy due to '\ + 'scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + elif resDicts['Device queue full']: + suggest = 'Disk busy due to disk queue full, '\ + 'Please reduce io pressure' + elif resDicts['Ioscheduler queue full']: + suggest = 'Io scheduler busy due to scheduler queue full, '\ + 'Please reduce io pressure or Adjust '\ + '/sys/block//queue/nr_requests larger carefully' + else: + suggest = 'Report stacktrace to OS kernel specialist' + + putIdx = ',diag_type=IOwait-high ' + putField = 'diagret=\"%s\",reason=\"%s\",solution=\"%s\"' %( + diagret, reason, suggest) + #nf.put(nfPutPrefix, + nf.puts(nfPutPrefix+putIdx+putField) + + +def iowaitResultReport(*argvs): + resultInfo = {} + nf = argvs[1] + nfPutPrefix = str(argvs[2]) + statusReportDicts = argvs[3] + maxGiowait = 0 + minGiowait = sys.maxsize + unkownDisable = None + + os.system('ls -rtd '+os.path.dirname(argvs[0])+'/../* | head -n -5 |'\ + ' xargs --no-run-if-empty rm {} -rf') + if os.path.exists(argvs[0]): + with open(argvs[0]) as logF: + dataList = logF.readlines() + else: + return + + for data in dataList: + try: + stat = json.loads(data, object_pairs_hook=OrderedDict) + except Exception: + return + gIowait,disable = iowaitDataParse(stat, resultInfo) + if not unkownDisable: + unkownDisable = disable + maxGiowait = max(maxGiowait, gIowait) + minGiowait = min(minGiowait, gIowait) + + if resultInfo: + content = str(minGiowait)+'%~'+str(maxGiowait)+'%' + diagret = 'IOwait high('+content+') detected' + iowaitReport(nf, nfPutPrefix, unkownDisable, resultInfo, diagret) + statusReportDicts['iowait']['valid'] = True + # print(diagret+reason+solution+'\n') + + +class displayClass(object): + def __init__(self, sender): + self.funcResultReportDicts = { + 'iohang': iohangResultReport, + 'ioutil': ioutilResultReport, + 'iolatency': iolatencyResultReport, + 'iowait': iowaitResultReport} + self.statusReportDicts = { + 'iohang': {'startT': 0, 'endT': 0, 'valid': False}, + 'ioutil': {'startT': 0, 'endT': 0, 'valid': False, + 'iopsThresh': 0, 'bpsThresh': 0}, + 'iolatency': {'startT': 0, 'endT': 0, 'valid': False, + 'lastIOburstT': 0}, + 'iowait': {'startT': 0, 'endT': 0, 'valid': False}, + } + self._sender = sender + self._nfPutPrefix = 'IOMonDiagLog' + + def markIoburst(self, now): + self.statusReportDicts['iolatency']['lastIOburstT'] = now + + def setIoburstThresh(self, iopsThresh, bpsThresh): + self.statusReportDicts['ioutil']['iopsThresh'] = iopsThresh + self.statusReportDicts['ioutil']['bpsThresh'] = bpsThresh + + def diagnoseValid(self, diagType): + return self.statusReportDicts[diagType]['valid'] + + def start(self, timeout, diagType, filepath, startTime, endTime): + self.statusReportDicts[diagType]['startT'] = startTime + self.statusReportDicts[diagType]['endT'] = endTime + self.statusReportDicts[diagType]['valid'] = False + argvs = [ + filepath, self._sender, self._nfPutPrefix, self.statusReportDicts] + timer = threading.Timer(timeout, + self.funcResultReportDicts[diagType], + argvs) + timer.start() diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py new file mode 100755 index 00000000..23b2a9ec --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptCheckClass.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- + +import sys +import string + + +class exceptCheckClass(): + def __init__(self, window): + self.window = int(window) if window is not None else 100 + self._exceptChkDicts = {} + + def addItem(self, key): + exceptChkItem = { + 'baseThresh': { + 'nrSample': 0, + 'moveWinData': [], + 'curWinMinVal': sys.maxsize, + 'curWinMaxVal': 0, + 'moveAvg': 0, + 'thresh': 0}, + 'compensation': { + 'thresh': 0, + 'shouldUpdThreshComp': True, + 'decRangeThreshAvg': 0, + 'decRangeCnt': 0, + 'minStableThresh': sys.maxsize, + 'maxStableThresh': 0, + 'stableThreshAvg': 0, + 'nrStableThreshSample': 0}, + 'dynTresh': sys.maxsize, + 'usedWin': 0} + self._exceptChkDicts.setdefault(key, exceptChkItem) + + # The sliding window calculates the basic threshold, through which the spikes + # and burrs in the IO indicators can be screened. The calculation idea is as + # follows: + # 1. take 100 data as a group for calculation (calculate 1 ~ 100 data for the + # first time, 2 ~ 101 for the second time, 3 ~ 102 for the third time, and + # so on), and calculate the average value mavg of 100 data in the current + # window + # 2. obtain the maximum value Max and minimum value min of 100 data, then record + # the thresh (MAX((max-mavg),(mavg-min))) each time, and calculate the average + # value(threshavg) of all thresh at this time each time, taking threshavg as + # the basic threshold for this time + # 3. The next basic threshold follows steps 1, 2, and so on + def _calcBaseThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + thresh = None + + bt['nrSample'] += 1 + if bt['nrSample'] >= self.window: + if len(bt['moveWinData']) < self.window: + bt['moveWinData'].append(e) + else: + bt['moveWinData'][exceptChkDict['usedWin'] % self.window] = e + moveAvg = float( + format(sum(bt['moveWinData']) / float(self.window), '.1f')) + + # Find the min and max values of this window so far + maxVal = max(bt['curWinMaxVal'], e) + minVal = min(bt['curWinMinVal'], e) + nrThreshSample = bt['nrSample'] + 1 - self.window + thresh = float( + format(max(maxVal - moveAvg, moveAvg - minVal), '.1f')) + # Calculate base threshold + threshAvg = float(format( + (bt['thresh'] * (nrThreshSample - 1) + thresh) / nrThreshSample, + '.3f')) + bt['thresh'] = threshAvg + bt['moveAvg'] = moveAvg + bt['curWinMaxVal'] = maxVal + bt['curWinMinVal'] = minVal + + exceptChkDict['usedWin'] += 1 + if exceptChkDict['usedWin'] >= self.window: + # the next window, set min and Max to 0 + bt['curWinMaxVal'] = 0 + bt['curWinMinVal'] = sys.maxsize + exceptChkDict['usedWin'] = 0 + else: + # Here, only the first window will enter to ensure that + # the data in one window is accumulated + bt['moveWinData'].append(e) + bt['curWinMaxVal'] = max(bt['curWinMaxVal'], e) + bt['curWinMinVal'] = min(bt['curWinMinVal'], e) + exceptChkDict['usedWin'] += 1 + return thresh + + # Called by _calcCompThresh to calculate the compensation value + # under normal steady state + def _calcStableThresh(self, ct, curBaseThresh, curThresh): + # Discard points exceeding (base-threshold / 10) + avg = ct['decRangeThreshAvg'] + if (curThresh - avg) < ((curBaseThresh - avg) / 10.0): + tSum = ct['stableThreshAvg'] * \ + ct['nrStableThreshSample'] + curThresh + ct['nrStableThreshSample'] += 1 + ct['stableThreshAvg'] = tSum / ct['nrStableThreshSample'] + ct['minStableThresh'] = min(ct['minStableThresh'], curThresh) + ct['maxStableThresh'] = max(ct['maxStableThresh'], curThresh) + # 1.5 windows of stable data have been counted, + # which can be used as normal threshold compensation value + if ct['nrStableThreshSample'] >= (self.window * 1.5): + ct['thresh'] = \ + max(ct['stableThreshAvg'] - ct['minStableThresh'], + ct['maxStableThresh'] - ct['stableThreshAvg']) + ct['shouldUpdThreshComp'] = False + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Calculate the threshold compensation value and superimpose this value + # on the basic threshold to eliminate false alarms + def _calcCompThresh(self, key, lastBaseThresh, curThresh): + exceptChkDict = self._exceptChkDicts[key] + curBaseThresh = exceptChkDict['baseThresh']['thresh'] + ct = exceptChkDict['compensation'] + + # It is not confirmed whether the current state is constant + # (constant state is defined as IO index fluctuation, which is stable) + # 1. the max basic threshold of this window is the compensation value + # 2. enter a new window to reset to the current basic threshold + if ct['shouldUpdThreshComp'] == True and \ + (ct['thresh'] < curBaseThresh or exceptChkDict['usedWin'] == 0): + ct['thresh'] = curBaseThresh + + # Continuous monotonic decreasing, constant steady state, + # constant compensation threshold inferred + if curBaseThresh < lastBaseThresh: + tSum = ct['decRangeThreshAvg'] * ct['decRangeCnt'] + curThresh + ct['decRangeCnt'] += 1 + ct['decRangeThreshAvg'] = tSum / ct['decRangeCnt'] + # The monotonic decline has continued for 1.5 windows, + # indicating that IO pressure may return to normality + if ct['decRangeCnt'] >= (self.window * 1.5): + self._calcStableThresh(ct, curBaseThresh, curThresh) + else: + # As long as the basic threshold curve is not + # continuously monotonically decreasing, + # reset to 0 and make statistics again + ct['minStableThresh'] = sys.maxsize + ct['maxStableThresh'] = 0 + ct['stableThreshAvg'] = ct['decRangeThreshAvg'] = 0 + ct['nrStableThreshSample'] = ct['decRangeCnt'] = 0 + + # Update the dynamic threshold of the corresponding indicator type + # and call it after collecting the IO indicators. The key is await, + # util, IOPs, BPS, etc + def updateDynThresh(self, key, e): + exceptChkDict = self._exceptChkDicts[key] + bt = exceptChkDict['baseThresh'] + ct = exceptChkDict['compensation'] + lastBaseThresh = bt['thresh'] + + curThresh = self._calcBaseThresh(key, e) + if curThresh is not None: + self._calcCompThresh(key, lastBaseThresh, curThresh) + exceptChkDict['dynTresh'] = \ + bt['thresh'] + bt['moveAvg'] + ct['thresh'] + + # Turn off the threshold compensation of the corresponding indicators. + # Generally, when it is detected that the IO util exceeds 20%, + # it will be disabled according to the situation of each indicator + def disableThreshComp(self, key): + exceptChkDict = self._exceptChkDicts[key] + ct = exceptChkDict['compensation'] + bt = exceptChkDict['baseThresh'] + + #if exceptChkDict['dynTresh'] == sys.maxsize: + # return + + if ct['shouldUpdThreshComp'] == True: + ct['shouldUpdThreshComp'] = False + exceptChkDict['dynTresh'] = bt['thresh'] + bt['moveAvg'] + ct['thresh'] = 0.000001 + + + def getNrDataSample(self, key): + return self._exceptChkDicts[key]['baseThresh']['nrSample'] + + # Get the dynamic threshold of the corresponding indicator type, + # call it after collecting the IO indicators, and judge whether + # the indicators are abnormal. The key is await, util, IOPs, BPS, etc + def getDynThresh(self, key): + return self._exceptChkDicts[key]['dynTresh'] diff --git a/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py new file mode 100755 index 00000000..98f912ce --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/exceptDiagnoseClass.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +import os +import string +import time +from collections import OrderedDict +import threading +from tools.iofstool.iofsstat import iofsstatStart +from tools.iowaitstat.iowaitstat import iowaitstatStart +from displayClass import displayClass + + +class runDiag(object): + def __init__(self, logRootPath, sender): + self.funcDicts = { + 'iohang': self.startIohangDiagnose, + 'ioutil': self.startIoutilDiagnose, + 'iolatency': self.startIolatencyDiagnose, + 'iowait': self.startIowaitDiagnose} + self.lastDiagTimeDicts = \ + {'iohang': 0, 'ioutil': 0, 'iolatency': 0, 'iowait': 0} + self.display = displayClass(sender) + self.sysakPath = 'sysak' + self.logRootPath = logRootPath + + + def _recentDiagnoseValid(self, diagType): + return self.display.diagnoseValid(diagType) + + + def startIohangDiagnose(self, *argv): + devname = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iohang'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/hangdetect/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iohang'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag hangdetect -o -t 3000 -T 10 -f '+ + file+' > '+outlog+' &') + self.display.start(20, 'iohang', logdir, now, now+60) + + + def startIolatencyDiagnose(self, *argv): + devname = argv[0] + thresh = argv[1] + ioburst = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['iolatency'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/latency/'+startTime + file = logdir+'/result.log.seq' + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iolatency'] = now + if devname is not None: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' '+devname+' > '+outlog+' &') + else: + os.system(self.sysakPath+' -g iosdiag latency -t '+str(thresh) + + ' -T 45 -f '+file+' > '+outlog+' &') + if ioburst: + self.display.markIoburst(now) + self.display.start(60, 'iolatency', logdir, now, now+60) + + + def startIoutilDiagnose(self, *argv): + devname = argv[0] + bwThresh = argv[1] + iopsThresh = argv[2] + now = time.time() + if now - self.lastDiagTimeDicts['ioutil'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iofsstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['ioutil'] = now + #self.display.setIoburstThresh(iopsThresh, bwThresh) + argvs = ['-j',outlog,'-n','-m','-c','1','-t','5','-T','40', + '-i',str(iopsThresh),'-b',str(bwThresh)] + threading.Thread(target=iofsstatStart, args=(argvs,)).start() + self.display.start(55, 'ioutil', outlog, now, now+60) + + + def startIowaitDiagnose(self, *argv): + iowaitThresh = argv[0] + now = time.time() + if now - self.lastDiagTimeDicts['iowait'] <= 60: + return + startTime = time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime(now)) + logdir = self.logRootPath+'/iosdiag/iowaitstat/'+startTime + outlog = logdir+'/resultCons.log' + if not os.path.exists(logdir): + try: + os.makedirs(logdir) + except Exception: + return + self.lastDiagTimeDicts['iowait'] = now + argvs = ['-j', outlog, '-t', '5', '-w', str(iowaitThresh), '-T', '45'] + threading.Thread(target=iowaitstatStart, args=(argvs,)).start() + self.display.start(55, 'iowait', outlog, now, now+60) + + + def runDiagnose(self, diagType, argv): + self.funcDicts[diagType](*list(argv)) + + +class diagnoseClass(runDiag): + def __init__(self, window, logRootPath, sender): + super(diagnoseClass, self).__init__(logRootPath, sender) + self.window = window + self.diagnoseDicts = OrderedDict() + self._diagStat = OrderedDict( + {'iohang': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'ioutil': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iowait': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}, + 'iolatency': {'run': False, 'argv': [0, 0, 0, 0, 0, 0, 0, 0]}}) + + + def addItem(self, devname, key, reportInterval, triggerInterval): + diagRecord = { + 'statWindow': self.window, + 'trigger': False, + 'lastReport': 0, + 'reportInterval': reportInterval, + 'reportCnt': 0, + 'lastDiag': 0, + 'triggerInterval': triggerInterval, + 'diagArgs': [0, 0, 0, 0, 0, 0, 0, 0]} + if devname not in self.diagnoseDicts.keys(): + self.diagnoseDicts.setdefault(devname, {key: diagRecord}) + else: + self.diagnoseDicts[devname].setdefault(key, diagRecord) + + + def setUpDiagnose(self, devname, key, nrSample, *argv): + diagnoseDicts = self.diagnoseDicts[devname][key] + lastDiag = diagnoseDicts['lastDiag'] + lastReport = diagnoseDicts['lastReport'] + statWindow = diagnoseDicts['statWindow'] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if lastReport == 0 or (nrSample-lastReport) > statWindow: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 1 + else: + diagnoseDicts['reportCnt'] += 1 + if diagnoseDicts['reportCnt'] > reportInterval: + if lastDiag == 0 or (nrSample-lastDiag) > triggerInterval: + diagnoseDicts['trigger'] = True + diagnoseDicts['reportCnt'] = 0 + diagnoseDicts['lastDiag'] = nrSample + else: + diagnoseDicts['lastReport'] = nrSample + diagnoseDicts['reportCnt'] = 0 + elif triggerInterval != 0: + if lastDiag == 0 or (nrSample-lastDiag) >= triggerInterval: + diagnoseDicts['lastDiag'] = nrSample + diagnoseDicts['trigger'] = True + else: + diagnoseDicts['trigger'] = True + + for idx, val in enumerate(argv): + diagnoseDicts['diagArgs'][idx] = val + + + def isException(self, devname, key): + diagnoseDicts = self.diagnoseDicts[devname][key] + reportInterval = diagnoseDicts['reportInterval'] + triggerInterval = diagnoseDicts['triggerInterval'] + + if reportInterval != 0: + if (diagnoseDicts['reportCnt'] + 1) >= reportInterval: + return True + elif triggerInterval != 0: + return True + else: + return True + return False + + + def clearDiagStat(self): + for diagType, stat in self._diagStat.items(): + stat['run'] = False + stat['argv'][0:] = [0, 0, 0, 0, 0, 0, 0, 0] + + + def checkDiagnose(self): + diagnoseDicts = self.diagnoseDicts + diagInfo = {'iohang': [], 'iolatency': [], 'ioutil': []} + diagStat = self._diagStat + ioburst = False + + for devname, diagDict in diagnoseDicts.items(): + if devname == 'system': + if diagDict['iowait']['trigger'] == True: + diagStat['iowait']['run'] = True + diagStat['iowait']['argv'][0] = \ + diagDict['iowait']['diagArgs'][0] + diagDict['iowait']['trigger'] = False + continue + + for diagType in ['iohang', 'iolatency', 'ioutil']: + if diagDict[diagType]['trigger'] == True: + if diagType == 'iolatency': + ioburst = diagDict['iolatency']['diagArgs'][1] + diagInfo[diagType].append(devname) + diagDict[diagType]['trigger'] = False + + for diagType, value in diagInfo.items(): + diagStat[diagType]['run'] = True + if len(value) > 1: + diagStat[diagType]['argv'][0] = None + elif len(value) == 1: + diagStat[diagType]['argv'][0] = value[0] + else: + diagStat[diagType]['run'] = False + + if diagStat['ioutil']['run'] == True: + for idx in [1,2]: + val = sorted( + [diagnoseDicts[dev]['ioutil']['diagArgs'][idx-1] + for dev in diagInfo['ioutil']], + reverse=True) + diagStat['ioutil']['argv'][idx] = val[-1] + + if diagStat['iolatency']['run'] == True: + diagStat['iolatency']['argv'][1] = sorted( + [diagnoseDicts[dev]['iolatency']['diagArgs'][0] + for dev in diagInfo['iolatency']], + reverse=True)[-1] + diagStat['iolatency']['argv'][2] = ioburst + + for diagType, stat in diagStat.items(): + if stat['run'] == True: + self.runDiagnose(diagType, stat['argv']) + stat['run'] = False + + + # In displayClass, after the diagnostic log is reported to the remote end, + # it will be marked as a valid diagnosis, and In exceptDiagnoseClass, + # clear the valid mark before each diagnosis + def recentDiagnoseValid(self, diagType): + return self._recentDiagnoseValid(diagType) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py new file mode 100755 index 00000000..0610135c --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonCfgClass.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import json +from collections import OrderedDict + +globalCfgDicts = {} +globalCfgPath = '' +def loadCfg(cfgPath): + with open(cfgPath) as f: + data = f.read() + return json.loads(data) + + +def loadCfgHander(signum, frame): + global globalCfgDicts + globalCfgDicts = loadCfg(globalCfgPath) + + +class ioMonCfgClass(object): + def __init__(self, cfgArg, resetCfg, logRootPath): + global globalCfgPath + self.cfgPath = logRootPath+'/ioMon/ioMonCfg.json' + globalCfgPath = self.cfgPath + cfg = self._paserCfg(cfgArg) + hasArgs = any(list(cfg.values())) + if not os.path.exists(self.cfgPath) or resetCfg: + cfg['iowait'] = int(cfg['iowait']) if cfg['iowait'] else 5 + cfg['await'] = int(cfg['await']) if cfg['await'] else 10 + cfg['util'] = int(cfg['util']) if cfg['util'] else 20 + cfg['iops'] = int(cfg['iops']) if cfg['iops'] else 150 + cfg['bps'] = int(cfg['bps']) if cfg['bps'] else 31457280 + cfg['cycle'] = int(cfg['cycle']) if cfg['cycle'] else 1000 + cfg['diagIowait'] = cfg['diagIowait'] if cfg['diagIowait'] else 'on' + cfg['diagIoburst'] = cfg['diagIoburst'] if cfg['diagIoburst'] else 'on' + cfg['diagIolat'] = cfg['diagIolat'] if cfg['diagIolat'] else 'on' + cfg['diagIohang'] = cfg['diagIohang'] if cfg['diagIohang'] else 'off' + self._updateCfg(cfg) + return + else: + self._loadCfg() + if hasArgs: + self._updateCfg(cfg) + + + def _paserCfg(self, cfgArg): + cfgDicts = { + 'iowait':None, 'await':None, 'util':None, 'iops':None, 'bps':None, + 'cycle':None, 'diagIowait':None, 'diagIoburst':None, + 'diagIolat':None, 'diagIohang':None} + try: + cfgList = \ + cfgArg.split(',') if cfgArg is not None and len(cfgArg) > 0 else [] + for cfg in cfgList: + errstr = None + c = cfg.split('=') + if c[0] not in cfgDicts.keys() or len(c[1]) == 0: + errstr = "bad cfg item: %s, must be in %s" %( + cfg, str(cfgDicts.keys())) + elif 'diag' not in c[0] and not c[1].isdigit(): + errstr = "monitor cfg argv must be digit: %s." %cfg + elif 'diag' in c[0] and c[1] not in ['on', 'off']: + errstr = \ + "diagnose cfg argv must be [\'on\', \'off\']: %s." %cfg + if errstr: + print(errstr) + sys.exit(0) + cfgDicts[c[0]] = c[1] + except Exception: + print "bad cfg: %s." %cfg + sys.exit(0) + return cfgDicts + + + def _setGlobalCfgDicts(self, CfgDicts): + global globalCfgDicts + globalCfgDicts = CfgDicts + + + def _getGlobalCfgDicts(self): + global globalCfgDicts + return globalCfgDicts + + + def _updateCfg(self, cfgDicts): + oldCfg = {} + if not os.path.exists(self.cfgPath): + if not os.path.exists(os.path.dirname(self.cfgPath)): + os.mkdir(os.path.dirname(self.cfgPath)) + else: + oldCfg = loadCfg(self.cfgPath) + f = open(self.cfgPath, 'w+') + newCfg = json.loads(json.dumps(cfgDicts)) + if oldCfg: + for key,val in newCfg.items(): + if val is not None: + oldCfg[key] = val + newCfg = oldCfg + s = json.dumps(newCfg, indent=4) + f.write(s) + f.close() + self._setGlobalCfgDicts(newCfg) + + + def _loadCfg(self): + self._setGlobalCfgDicts(loadCfg(self.cfgPath)) + + + def createCfgFlagFile(self): + f = open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag', 'w+') + f.write(str(os.getpid())) + f.close() + signal.signal(signal.SIGUSR2, loadCfgHander) + + + def notifyIoMon(self): + try: + with open(os.path.dirname(self.cfgPath)+'/.ioMonCfgFlag') as f: + pid = f.read() + with open('/proc/'+str(pid)+'/cmdline') as f: + cmdline = f.read().strip() + except Exception: + sys.exit(0) + if 'ioMonEntry' in cmdline: + os.system('kill -USR2 '+str(pid)) + + + def getCfgItem(self, key): + val = str(self._getGlobalCfgDicts()[key]) + if val.isdigit(): + val = int(val) + return val diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py new file mode 100755 index 00000000..0521a7ff --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +from exceptDiagnoseClass import diagnoseClass +from exceptCheckClass import exceptCheckClass +from ioMonCfgClass import ioMonCfgClass +from collections import OrderedDict +from nfPut import CnfPut + +class ioMonitorClass(object): + def __init__(self, logRootPath, cfg, pipeFile): + self.window = 60 + self.cfg = cfg + self.cfg.createCfgFlagFile() + self.diagSwitch = { + 'diagIowait': {'sw': self.cfg.getCfgItem('diagIowait'), + 'esi':'IOwait-High'}, + 'diagIoburst': {'sw': self.cfg.getCfgItem('diagIoburst'), + 'esi':'IO-Delay'}, + 'diagIolat': {'sw': self.cfg.getCfgItem('diagIolat'), + 'esi':'IO-Burst'}, + 'diagIohang': {'sw': self.cfg.getCfgItem('diagIohang'), + 'esi':'IO-Hang'} + } + self._sender = CnfPut(pipeFile) + self._nfPutTlb = 'IOMonIndForDisksIO' + self._nfPutTlb4System = 'IOMonIndForSystemIO' + self.fieldDicts = OrderedDict() + self.exceptChkDicts = {'system': exceptCheckClass(self.window)} + self.exceptChkDicts['system'].addItem('iowait') + self.diagnose = diagnoseClass(self.window, logRootPath, self._sender) + self.diagnose.addItem('system', 'iowait', 0, 60) + self.fDiskStats = open("/proc/diskstats") + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + self.uploadInter = 0 + self.exceptionStat = {'system': {'IOwait-High': {'cur':0,'max':0}}} + self.dataStat = {'system': {'iowait': 0}} + + + def _addMonitorAttrForDisk(self, disk): + dataStat = self.dataStat + exceptChkDicts = self.exceptChkDicts + diagnose = self.diagnose + exceptionStat = self.exceptionStat + + # used for reporting per-index to database + dataStat.setdefault( + disk, {'await': 0, 'util': 0, 'iops': 0, 'bps': 0, 'qusize': 0}) + + # add exception-check attr for per-index on per-disk + exceptChkDicts.setdefault(disk, exceptCheckClass(self.window)) + for key in ['util', 'await', 'iops', 'bps']: + exceptChkDicts[disk].addItem(key) + + # add diagnose attr for per-index on per-disk + diagnoseDict = { + 'iohang': {'triggerInterval': self.window * 5, + 'reportInterval': 10}, + 'ioutil': {'triggerInterval': 60, 'reportInterval': 0}, + 'iolatency': {'triggerInterval': 60, 'reportInterval': 0} + } + for key, item in diagnoseDict.items(): + diagnose.addItem( + disk, key, item['reportInterval'], item['triggerInterval']) + # used for reporting exception to database + exceptionStat.setdefault( + disk, + {'IO-Delay':{'cur':0,'max':0}, 'IO-Burst':{'cur':0,'max':0}, + 'IO-Hang':{'cur':0,'max':0}}) + + + def _removeDiskMonitor(self, disk): + del self.fieldDicts[disk] + del self.dataStat[disk] + del self.exceptChkDicts[disk] + del self.diagnose[disk] + del self.exceptionStat[disk] + + + def _disableThreshComp(self, devname, qusize): + exceptChkDicts = self.exceptChkDicts + exceptChkDicts[devname].disableThreshComp('util') + exceptChkDicts[devname].disableThreshComp('iops') + exceptChkDicts[devname].disableThreshComp('bps') + if qusize > 1: + exceptChkDicts[devname].disableThreshComp('await') + + + def _calcIowait(self): + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + iowait = float(format( + (statList[4] - self.cpuStatIowait['iowait']) * 100.0 / + (sum(statList) - self.cpuStatIowait['sum']), '.2f')) + return iowait + + + def _calcIoIndex(self, devname, field, secs): + ds = self.dataStat[devname] + uploadInter = self.uploadInter + + rws = field['1'][1] + field['5'][1] - field['1'][0] - field['5'][0] + iops = round(rws / secs, 1) + ds['iops'] = (ds['iops'] * (uploadInter - 1) + iops) / uploadInter + + rwSecs = field['3'][1] + field['7'][1] - field['3'][0] - field['7'][0] + bps = rwSecs / secs * 512 + ds['bps'] = (ds['bps'] * (uploadInter - 1) + bps) / uploadInter + + qusize = round((field['11'][1] - field['11'][0]) / secs / 1000, 2) + ds['qusize'] = (ds['qusize'] * (uploadInter - 1) + qusize) / uploadInter + + rwTiks = field['4'][1] + field['8'][1] - field['4'][0] - field['8'][0] + wait = round(rwTiks / (iops * secs), 2) if iops else 0 + ds['await'] = (ds['await'] * (uploadInter - 1) + wait) / uploadInter + + times = field['10'][1] - field['10'][0] + util = round(times * 100.0 / (secs * 1000), 2) + util = util if util <= 100 else 100.0 + ds['util'] = (ds['util'] * (uploadInter - 1) + util) / uploadInter + + return {'iops': iops*secs, 'bps': bps*secs, 'qusize': qusize*secs, + 'wait': wait, 'util': util} + + + def _checkDiagSwitchChange(self, t): + s = self.diagSwitch[t] + newSW = self.cfg.getCfgItem(t) + if newSW != s['sw'] and newSW == 'on': + for k,v in self.exceptionStat.items(): + if s['esi'] in v.keys(): + v[s['esi']]['max'] = 0 + s['sw'] = newSW + return newSW + + + def _checkIOwaitException(self, iowait): + exceptChk = self.exceptChkDicts['system'] + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + diagnose = self.diagnose + uploadInter = self.uploadInter + + if iowait >= self.cfg.getCfgItem('iowait'): + exceptChk.disableThreshComp('iowait') + + dataStat['iowait'] = \ + (dataStat['iowait'] * (uploadInter - 1) + iowait) / uploadInter + + # Detect iowait exception + minThresh = self.cfg.getCfgItem('iowait') + iowaitThresh = max(exceptChk.getDynThresh('iowait'), minThresh) + if iowait >= iowaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIowait') + rDiagValid = diagnose.recentDiagnoseValid('iowait') + # Configure iowait diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('iowait') + iowaitArg = max(int(iowait * 0.25), minThresh) + diagnose.setUpDiagnose('system', 'iowait', nrSample, iowaitArg) + + exceptChk.updateDynThresh('iowait', iowait) + + def _checkIoburstException(self, devname, es, bps, iops, exceptChk): + bpsLowW = self.cfg.getCfgItem('bps') + bpsHighW = max(exceptChk.getDynThresh('bps'), bpsLowW) + bpsMiddW = max(bpsLowW, bpsHighW / 2) + iopsLowW = self.cfg.getCfgItem('iops') + iopsHighW = max(exceptChk.getDynThresh('iops'), iopsLowW) + iopsMiddW = max(iopsLowW, iopsHighW / 2) + ioburst = exception = False + + if iops >= iopsMiddW or bps >= bpsMiddW: + ioburst = True + bpsOver = True if bps >= bpsHighW else False + iopsOver = True if iops >= iopsHighW else False + + if iopsOver or bpsOver: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIoburst') + rDiagValid = self.diagnose.recentDiagnoseValid('ioutil') + # Configure IO load diagnosis + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + bpsArg = iopsArg = 0 + if bpsOver == True: + bpsArg = max(int(bps * 0.25), bpsLowW) + if iopsOver == True: + iopsArg = max(int(iops * 0.7), iopsLowW) + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose( + devname, 'ioutil', nrSample, bpsArg, iopsArg) + return ioburst + + def _checkIohangException( + self, devname, es, util, qusize, iops, exceptChk): + # Detect IO hang + if util >= 100 and qusize >= 1 and iops < 50: + # Configure IO hang diagnosis + if self.diagnose.isException(devname, 'iohang') == True: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIohang') + rDiagValid = self.diagnose.recentDiagnoseValid('iohang') + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('util') + self.diagnose.setUpDiagnose(devname, 'iohang', nrSample) + + def _checkUtilException(self, devname, util, iops, bps, qusize): + exceptChk = self.exceptChkDicts[devname] + exceptionStat = self.exceptionStat[devname] + diagnose = self.diagnose + ioburst = False + + utilMinThresh = self.cfg.getCfgItem('util') + utilThresh = max(exceptChk.getDynThresh('util'), utilMinThresh) + if util >= utilThresh: + es = exceptionStat['IO-Burst'] + ioburst = \ + self._checkIoburstException(devname, es, bps, iops, exceptChk) + if not ioburst: + es = exceptionStat['IO-Hang'] + self._checkIohangException( + devname, es, util, qusize, iops, exceptChk) + exceptChk.updateDynThresh('util', util) + exceptChk.updateDynThresh('iops', iops) + exceptChk.updateDynThresh('bps', bps) + return ioburst + + + def _checkAwaitException(self, devname, wait, ioburst): + exceptChk = self.exceptChkDicts[devname] + es = self.exceptionStat[devname]['IO-Delay'] + diagnose = self.diagnose + + awaitMinThresh = self.cfg.getCfgItem('await') + awaitThresh = max(exceptChk.getDynThresh('await'), awaitMinThresh) + if wait >= awaitThresh: + es['cur'] += 1 + diagSW = self._checkDiagSwitchChange('diagIolat') + rDiagValid = diagnose.recentDiagnoseValid('iolatency') + # Configuring IO delay diagnostics + if diagSW == 'on' and (es['cur'] > es['max'] or not rDiagValid): + nrSample = exceptChk.getNrDataSample('await') + waitArg = max(int(wait * 0.4), awaitMinThresh) + diagnose.setUpDiagnose( + devname, 'iolatency', nrSample, waitArg, ioburst) + exceptChk.updateDynThresh('await', wait) + + + def _reportDataToRemote(self, devList): + # report datastat&&exception to database + nCycle = 1000.0 / float(self.cfg.getCfgItem('cycle')) + dataStat = self.dataStat['system'] + es = self.exceptionStat['system']['IOwait-High'] + + putIdx = ',idx_type=system_Indicator,devname=system ' + putField = 'iowait=%f,iowaithighCnt=%f' %( + dataStat['iowait'], es['cur'] / nCycle) + self._sender.puts(self._nfPutTlb4System + putIdx + putField) + + es['max'] = max(es['max'] if es['cur'] else 0, es['cur']) + es['cur'] = 0 + cur = {'IO-Delay':0, 'IO-Burst':0, 'IO-Hang':0} + for devname in devList: + dataStat = self.dataStat[devname] + es = self.exceptionStat[devname] + for type in cur.keys(): + cur[type] = int(es[type]['cur']) / nCycle + es[type]['max'] = \ + max(es[type]['max'] if cur[type] else 0, cur[type]) + es[type]['cur'] = 0 + + putIdx = ',idx_type=iostat_Indicator,devname=%s ' % devname + putField = 'await=%f,util=%f,iops=%f,bps=%f,qusize=%f' %( + dataStat['await'], dataStat['util'], dataStat['iops'], + dataStat['bps'] / 1024.0, dataStat['qusize'] + ) + putField += ',iodelayCnt=%f,ioburstCnt=%f,iohangCnt=%f' %( + cur['IO-Delay'], cur['IO-Burst'], cur['IO-Hang']) + self._sender.puts(self._nfPutTlb + putIdx + putField) + + + def _collectBegin(self): + fieldDicts = self.fieldDicts + + # collect iowait begin + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + # collect iostat begin + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + + if stat[2] in fieldDicts.keys(): + field = fieldDicts[stat[2]] + else: + field = { + '1': [0, 0], '3': [0, 0], '4': [0, 0], + '5': [0, 0], '7': [0, 0], '8': [0, 0], + '10': [0, 0], '11': [0, 0]} + # add data staticsis for per-disk + fieldDicts.setdefault(stat[2], field) + self._addMonitorAttrForDisk(stat[2]) + + for idx, value in field.items(): + value[0] = long(stat[int(idx) + 2]) + + + def _collectEnd(self, secs): + fieldDicts = self.fieldDicts + exceptChkDicts = self.exceptChkDicts + uploadInter = self.uploadInter + + self.uploadInter = \ + 1 if ((uploadInter * secs) % 60) == 0 else (uploadInter + 1) + + # Calculate iowait + iowait = self._calcIowait() + # Detect iowait exception + self._checkIOwaitException(iowait) + + # collect iostat end + self.fDiskStats.seek(0) + for stat in self.fDiskStats.readlines(): + stat = stat.split() + if os.path.exists('/sys/block/'+stat[2]) == False: + if stat[2] in fieldDicts.keys(): + self._removeDiskMonitor(stat[2]) + continue + for idx, value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx) + 2]) + + for devname, field in fieldDicts.items(): + io = self._calcIoIndex(devname, field, secs) + if io['util'] >= self.cfg.getCfgItem('util'): + # There is IO Burst at present, turn off threshold compensation + self._disableThreshComp(devname, io['qusize']) + # Detect util exception + ioburst = self._checkUtilException( + devname, io['util'], io['iops'], io['bps'], io['qusize']) + # Detect await exception + self._checkAwaitException(devname, io['wait'], ioburst) + + if ((self.uploadInter * secs) % 60) == 0: + self._reportDataToRemote(fieldDicts.keys()) + + + def monitor(self): + while True: + secs = self.cfg.getCfgItem('cycle') / 1000.0 + self._collectBegin() + time.sleep(secs) + self._collectEnd(secs) + # Check if it is necessary to start the diagnosis + self.diagnose.checkDiagnose() + self.fDiskStats.close() + diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py new file mode 100755 index 00000000..921a3435 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorMain.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +import argparse +import signal +from ioMonCfgClass import ioMonCfgClass +from ioMonitorClass import ioMonitorClass + +setcfg_descripton = """set monitor cfg, like -s \'xxxx=xxx,xxx=xxx\' +iowait, The min cpu-iowait not report exceptions(1~100, default 5). +await, The min partition-await not report exceptions(N ms, default 10). +util, The min partition-util not report exceptions(1~100, default 20). +bps, The min partition-bps not report exceptions(bytes, default 30MB). +iops, The min partition-iops not report exceptions(default 150). +cycle, The cycle of Monitoring data collection(default 500ms). +diagIowait, Disable or enable diagnose while reporting iowait exceptions(default on). +diagIolat, Disable or enable diagnose while reporting latency exceptions(default on). +diagIoburst, Disable or enable diagnose while reporting ioburst exceptions(default on). +diagIohang, Disable or enable diagnose while reporting iohang exceptions(default on). +""" + +def getRunArgsFromYaml(yamlF): + logRootPath = '' + pipeFile = '' + with open(yamlF) as f: + lines = f.readlines() + for l in lines: + if not l.startswith('#'): + if 'proc_path:' in l: + logRootPath = l.split()[1].strip('\n') + elif 'outline:' in l: + pipeFile = lines[lines.index(l) + 1].split()[1].strip('\n') + if logRootPath and pipeFile: + break + if not logRootPath or not pipeFile: + raise ValueError( + 'Unable to get labels \"proc_path\" and \"outline\" in %s' % yamlF) + return logRootPath+'/run',pipeFile + + +def main(): + examples = """e.g. + ./ioMonitorMain.py -y [yaml_file] + Start ioMonitor + ./ioMonitorMain.py -y [yaml_file] --reset_cfg --only_set_cfg + Only reset cfg to default + ./ioMonitorMain.py -y [yaml_file] -s 'iowait=10,iops=200,diagIowait=on' --only_set_cfg + Only set min-iowait&&min-iops and disable iowait diagnose to config. + """ + parser = argparse.ArgumentParser( + description="start ioMonitor.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-y','--yaml_file', + help='Specify the socket pipe for data upload'\ + ' and exception log path') + parser.add_argument('-s','--set_cfg', help=setcfg_descripton) + parser.add_argument('-r','--reset_cfg', action='store_true', + help='Reset cfg to default') + parser.add_argument('-o','--only_set_cfg', action='store_true', + help='Only set cfg') + args = parser.parse_args() + + signal.signal(signal.SIGCHLD, signal.SIG_IGN) + logRootPath,pipeFile = getRunArgsFromYaml(args.yaml_file) + if args.only_set_cfg: + if not os.path.exists(logRootPath+'/ioMon/ioMonCfg.json'): + print("%s" % ("config fail, not found ioMonCfg.json")) + return + if args.set_cfg is None and not args.reset_cfg: + print("%s" % ("--set_cfg or --reset_cfg not found.")) + return + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMonCfg.notifyIoMon() + return + + ioMonCfg = ioMonCfgClass(args.set_cfg, args.reset_cfg, logRootPath) + ioMon = ioMonitorClass(logRootPath, ioMonCfg, pipeFile) + ioMon.monitor() + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/nfPut.py b/source/tools/monitor/ioMonitor/ioMon/nfPut.py new file mode 100755 index 00000000..5fcaea96 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/nfPut.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： nfPut + Description : + Author : liaozhaoyan + date： 2022/4/28 +------------------------------------------------- + Change Activity: + 2022/4/28: +------------------------------------------------- +""" +__author__ = 'liaozhaoyan' + +import os +import socket +import requests +MAX_BUFF = 128 * 1024 + + +class CnfPut(object): + def __init__(self, pipeFile): + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self._path = pipeFile + if not os.path.exists(self._path): + raise ValueError("pipe path is not exist. please check unity is running.") + + + def puts(self, s): + if len(s) > MAX_BUFF: + raise ValueError("message len %d, is too long ,should less than%d" % (len(s), MAX_BUFF)) + return self._sock.sendto(s, self._path) + + +if __name__ == "__main__": + nf = CnfPut("/tmp/sysom") + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py new file mode 100755 index 00000000..f01dbf27 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/common.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import string +import re +from subprocess import PIPE, Popen + + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def echoFile(filename, txt): + execCmd("echo \'"+txt+"\' > "+filename) + + +def echoFileAppend(filename, txt): + execCmd("echo \'"+txt+"\' >> "+filename) + + +def humConvert(value, withUnit): + units = ["B", "KB", "MB", "GB", "TB", "PB"] + size = 1024.0 + + if value == 0: + return value + + for i in range(len(units)): + if (value / size) < 1: + if withUnit: + return "%.1f%s/s" % (value, units[i]) + else: + return "%.1f" % (value) + value = value / size + + +def getDevt(devname): + try: + with open('/sys/class/block/' + devname + '/dev') as f: + dev = f.read().split(':') + return ((int(dev[0]) << 20) + int(dev[1])) + except Exception: + return -1 + + +def getDevtRegion(devname): + if os.path.exists('/sys/block/'+devname): + isPart = False + elif os.path.exists('/sys/class/block/'+devname): + isPart = True + else: + return [-1, -1] + + master = devname if not isPart else \ + os.readlink('/sys/class/block/'+devname).split('/')[-2] + partList = list( + filter(lambda x: master in x, + os.listdir('/sys/class/block/'+master))) + if not partList: + partList = [] + partList.append(master) + return [getDevt(p) for p in partList] + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def getContainerId(pid): + try: + piddir = "/proc/"+str(pid) + with open(piddir+"/cgroup") as f: + # ... + # cpuset,cpu,cpuacct:/docker/e2afa607d8f13e5b1f89d38ee86d86.... + # memory:/docker/e2afa607d8f13e5b1f89d38ee86..... + # ... + cid = f.read().split('memory:')[1].split('\n')[0].rsplit('/',1)[1] + if not cid: + cid = '-' + except Exception: + cid = '-' + return cid + + +def getFullNameFromProcPid(pid, ino): + try: + piddir = "/proc/"+str(pid) + # list the open files of the task + fdList = os.listdir(piddir+"/fd") + for f in fdList: + try: + path = os.readlink(piddir+"/fd/"+f) + if '/dev/' in path or '/proc/' in path or '/sys/' in path: + continue + + if os.path.isfile(path) and os.stat(path).st_ino == int(ino): + return path + except (IOError, EOFError) as e: + continue + except Exception: + pass + return "-" + + +def supportKprobe(name): + file = '/sys/kernel/debug/tracing/available_filter_functions' + with open(file) as f: + ss = f.read() + if ss.find(name) > 0: + return True + return False diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py new file mode 100755 index 00000000..c9755bfe --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/diskstatClass.py @@ -0,0 +1,219 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import time +import json +import string +from collections import OrderedDict +from common import humConvert + + +class diskstatClass(object): + def __init__(self, devname, utilThresh, json, nodiskStat, Pattern): + self.devname = devname + self.json = json + self.cycle = 1 + self.started = False + self.Pattern = Pattern + self.nodiskStat = nodiskStat + self.utilThresh = int(utilThresh) if utilThresh is not None else 0 + self.fieldDicts = OrderedDict() + self.diskInfoDicts = {} + self.deviceStatDicts = {} + self.f = open("/proc/diskstats") + if json: + self.fJson = open(json, 'w+') + + + def getDevNameByDevt(self, devt): + try: + return self.diskInfoDicts[str(devt)]['partName'] + except Exception: + return '-' + + def getMasterDev(self, devt): + try: + return self.diskInfoDicts[str(devt)]['master'] + except Exception: + return '-' + + def getDiskStatInd(self, disk, key): + return self.deviceStatDicts[disk][key] + + def start(self): + fieldDicts = self.fieldDicts + diskInfoDicts = self.diskInfoDicts + deviceStatDicts = self.deviceStatDicts + + if self.started: + return + self.started = True + self.cycle = time.time() + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + + field = {\ + '1':[0,0], '2':[0,0], '3':[0,0], '4':[0,0],\ + '5':[0,0], '6':[0,0], '7':[0,0], '8':[0,0],\ + '10':[0,0], '11':[0,0]} + for idx,value in field.items(): + value[0] = long(stat[int(idx)+2]) + if stat[2] not in fieldDicts.keys(): + fieldDicts.setdefault(stat[2], field) + path = os.readlink('/sys/class/block/'+stat[2]).split('/') + master = path[-2] + if master not in path[-1]: + master = path[-1] + diskInfoDicts.setdefault( + str((int(stat[0])<<20)+int(stat[1])), + {'partName': stat[2], 'master': master}) + deviceStat = { + 'r_rqm':0, 'w_rqm':0, 'r_iops':0, 'w_iops':0, 'r_bps':0, + 'w_bps':0, 'wait':0, 'r_wait':0, 'w_wait':0, 'util%':-1} + deviceStatDicts.setdefault(stat[2], deviceStat) + else: + deviceStatDicts[stat[2]]['util%'] = -1 + fieldDicts[stat[2]].update(field) + + def stop(self): + fieldDicts = self.fieldDicts + deviceStatDicts = self.deviceStatDicts + self.cycle = max(int(time.time()-self.cycle), 1) + + if not self.started: + return + self.started = False + + self.f.seek(0) + for stat in self.f.readlines(): + stat = stat.split() + if self.devname is not None and self.devname not in stat[2] and \ + stat[2] not in self.devname: + continue + for idx,value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx)+2]) + + for devname,field in fieldDicts.items(): + if self.devname is not None and devname not in self.devname and \ + self.devname not in devname: + continue + util = round((field['10'][1]-field['10'][0])*100.0/(self.cycle*1000),2) + util = util if util <= 100 else 100.0 + if util < self.utilThresh: + continue + deviceStatDicts[devname]['util%'] = util + r_iops = field['1'][1]-field['1'][0] + deviceStatDicts[devname]['r_iops'] = r_iops + r_rqm = field['2'][1]-field['2'][0] + deviceStatDicts[devname]['r_rqm'] = r_rqm + w_iops = field['5'][1]-field['5'][0] + deviceStatDicts[devname]['w_iops'] = w_iops + w_rqm = field['6'][1]-field['6'][0] + deviceStatDicts[devname]['w_rqm'] = w_rqm + r_bps = (field['3'][1]-field['3'][0]) * 512 + deviceStatDicts[devname]['r_bps'] = r_bps + w_bps = (field['7'][1]-field['7'][0]) * 512 + deviceStatDicts[devname]['w_bps'] = w_bps + r_ticks = field['4'][1]-field['4'][0] + w_ticks = field['8'][1]-field['8'][0] + wait = round((r_ticks+w_ticks)/(r_iops+w_iops), 2) if (r_iops+w_iops) else 0 + deviceStatDicts[devname]['wait'] = wait + r_wait = round(r_ticks / r_iops, 2) if r_iops else 0 + deviceStatDicts[devname]['r_wait'] = r_wait + w_wait = round(w_ticks / w_iops, 2) if w_iops else 0 + deviceStatDicts[devname]['w_wait'] = w_wait + + + def __showJson(self): + deviceStatDicts = self.deviceStatDicts + + statJsonStr = '{\ + "time":"",\ + "diskstats":[]}' + dstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + dstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + for devname,stat in deviceStatDicts.items(): + if stat['util%'] < 0: + continue + dstatJsonStr = '{\ + "diskname":"","r_rqm":0,"w_rqm":0,"r_iops":0,"w_iops":0,\ + "r_bps":0,"w_bps":0,"wait":0,"r_wait":0,"w_wait":0,"util%":0}' + dstatDict = json.loads(dstatJsonStr, object_pairs_hook=OrderedDict) + dstatDict["diskname"] = devname + for key,val in stat.items(): + dstatDict[key] = val + dstatDicts["diskstats"].append(dstatDict) + if len(dstatDicts["diskstats"]) > 0: + data = json.dumps(dstatDicts) + self.writeDataToJson(data) + return + + def show(self): + secs = self.cycle + deviceStatDicts = self.deviceStatDicts + if self.nodiskStat: + return + + if self.enableJsonShow() == True: + self.__showJson() + return + + if self.Pattern: + WrTotalIops = 0 + RdTotalIops = 0 + WrTotalBw = 0 + RdTotalBw = 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (("device-stat:"),"r_rqm","w_rqm","r_iops","w_iops","r_bps",\ + "w_bps","wait","r_wait","w_wait","util%")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for devname,stat in deviceStatDicts.items(): + if (not self.devname and not os.path.exists('/sys/block/'+devname)) or \ + stat['util%'] < 0: + continue + if self.Pattern: + WrTotalIops += stat['w_iops'] + RdTotalIops += stat['r_iops'] + WrTotalBw += stat['w_bps'] + RdTotalBw += stat['r_bps'] + stWbps = humConvert(stat['w_bps'], True).replace('s', stSecs) if stat['w_bps'] else 0 + stRbps = humConvert(stat['r_bps'], True).replace('s', stSecs) if stat['r_bps'] else 0 + print('%-20s%-8s%-8s%-8s%-8s%-12s%-12s%-8s%-8s%-8s%-8s' %\ + (devname, str(stat['r_rqm']), str(stat['w_rqm']), str(stat['r_iops']), + str(stat['w_iops']), stRbps, stWbps, str(stat['wait']), str(stat['r_wait']), + str(stat['w_wait']), str(stat['util%']))) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + ((WrTotalIops+RdTotalIops), RdTotalIops, WrTotalIops, + (humConvert((WrTotalBw+RdTotalBw), True).replace('s', stSecs) if (WrTotalBw+RdTotalBw > 0) else 0), + (humConvert(RdTotalBw, True).replace('s', stSecs) if RdTotalBw else 0), + (humConvert(WrTotalBw, True).replace('s', stSecs) if WrTotalBw else 0))) + print("") + + def clear(self): + self.f.close() + if self.enableJsonShow(): + self.fJson.close() + + def notCareDevice(self, devname): + if not self.nodiskStat and self.deviceStatDicts[devname]['util%'] < 0: + return True + return False + + def disableShow(self): + deviceStatDicts = self.deviceStatDicts + for devname,stat in deviceStatDicts.items(): + if self.deviceStatDicts[devname]['util%'] >= 0: + return False + return True + + def enableJsonShow(self): + return True if self.json else False + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py new file mode 100755 index 00000000..763de302 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/fsstatClass.py @@ -0,0 +1,418 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevt,getDevtRegion +from common import humConvert,supportKprobe +from common import execCmd,echoFile,echoFileAppend +from common import getTgid,fixComm,getContainerId,getFullNameFromProcPid +from mmap import PAGESIZE + + +def getMntPath(fileInfoDict): + mntfname = fileInfoDict['mntfname'] + fsmountInfo = fileInfoDict['fsmountinfo'] + + if len(fsmountInfo) <= 0: + return '-' + + if mntfname.isspace() or len(mntfname) == 0: + return fsmountInfo[0].split()[1] + try: + for l in fsmountInfo: + if l.find(mntfname) > -1: + return l.split()[1] + return '-' + except IndexError: + return fsmountInfo[0].split()[1] + + +def getFullName(fileInfoDict): + fileSuffix = '' + + mntdir = getMntPath(fileInfoDict) + if mntdir == '/': + mntdir = '' + + for f in [ + fileInfoDict['d3fname'], fileInfoDict['d2fname'], + fileInfoDict['d1fname'], fileInfoDict['bfname']]: + if f != '/' and f != '(fault)': + fileSuffix += ('/' + f) + if fileInfoDict['d3fname'] != '/' and fileInfoDict['d3fname'] != '(fault)': + fileSuffix = '/...' + fileSuffix + filename = mntdir + fileSuffix + + if '...' in filename: + f = getFullNameFromProcPid( + fileInfoDict['pid'], fileInfoDict['ino']) + if f != '-': + filename = f + return filename + + +class fsstatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, bwThresh, top, + json, nodiskStat, miscStat, Pattern): + super(fsstatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.expression = [] + self.pid = pid + self.miscStat = miscStat + self.devname = devname + self.top = int(top) if top is not None else 99999999 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = getDevtRegion(devname) if devname is not None else [-1, -1] + tracingBaseDir = "/sys/kernel/debug/tracing" + self.kprobeEvent = tracingBaseDir+"/kprobe_events" + self.tracingDir = tracingBaseDir+'/instances/iofsstat4fs' + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.kprobe = [] + self.kprobeArgsFormat = 'dev=+0x10(+0x28(+0x20(%s))):u32 '\ + 'inode_num=+0x40(+0x20(%s)):u64 len=%s:u64 '\ + 'mntfname=+0x0(+0x28(+0x0(+0x10(%s)))):string '\ + 'bfname=+0x0(+0x28(+0x18(%s))):string '\ + 'd1fname=+0x0(+0x28(+0x18(+0x18(%s)))):string '\ + 'd2fname=+0x0(+0x28(+0x18(+0x18(+0x18(%s))))):string '\ + 'd3fname=+0x0(+0x28(+0x18(+0x18(+0x18(+0x18(%s)))))):string %s' + + kprobeArgs = self._getKprobeArgs('None') + self.ftracePaserCommArgs = ' comm=(.*)' if 'comm=' in kprobeArgs else '' + mmapKprobeArgs = self._getKprobeArgs('mmap') + self.fsmountInfo = self._getFsMountInfo() + for entry in self.fsmountInfo: + fstype = entry.split()[2] + self._kprobeReadWrite(fstype, kprobeArgs) + self._kprobeMmap(fstype, mmapKprobeArgs) + if len(self.kprobe) <= 0: + print("%s" % ("error: not available kprobe")) + sys.exit(-1) + self.outlogFormatBase = 10 + + def _kprobeReadWrite(self, fstype, kprobeArgs): + for op in ['write', 'read']: + kPoints = [ + fstype+"_file_"+op+"_iter", fstype+"_file_"+op, + fstype+"_file_aio_"+op, "generic_file_aio_"+op] + if list(set(self.kprobe) & set(kPoints)): + continue + kprobe = None + for k in kPoints: + if supportKprobe(k): + kprobe = k + break + if not kprobe: + if self.enableJsonShow() == False: + print("warnning: not available %s kprobe" % op) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _kprobeMmap(self, fstype, kprobeArgs): + for kprobe in [fstype+"_page_mkwrite", 'filemap_fault']: + if kprobe in self.kprobe: + continue + if not supportKprobe(kprobe): + if self.enableJsonShow() == False: + print("not support kprobe %s" % kprobe) + continue + pointKprobe = 'p '+kprobe+' '+kprobeArgs + self.kprobe.append(kprobe) + self.expression.append(pointKprobe) + + def _getKprobeArgs(self, type): + commArgs = '' + vinfo = execCmd('uname -r') + version = vinfo.split('.') + + if type == 'mmap': + offFile = '0xa0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0xa0%s' + offLen = '0x0(+0x0%s)' if int(version[0]) > 4 or ( + int(version[0]) == 4 and int(version[1]) > 10) else '0x0%s' + else: + offLen = '0x10' + offFile = '0x0' if int(version[0]) > 3 or ( + int(version[0]) == 3 and int(version[1]) > 10) else '0x8' + if int(version[0]) <= 3: + offLen = '0x8' if int(version[1]) < 13 else '0x18' + + if int(version[0]) > 3: + commArgs = 'comm=$comm' + + re= execCmd('lscpu | grep -E \"Architecture|架构\" | sed \"s/：/:/g\"') + arch = re.split(':')[1].strip() + regs = { + "arm":['(%r0)','(%r1)'], + "x86":['(%di)', '(%si)'], + "aarch64":['(%x0)','(%x1)']} + argv0 = argv1 = '' + for key,val in regs.items(): + if arch.startswith(key): + if type == 'mmap': + argv0 = '+' + (offFile % val[0]) + argv1 = '+' + (offLen % val[0]) + argv2 = argv1 + else: + argv2 = argv0 = '+' + offFile + val[0] + argv1 = '+' + offLen + val[1] + break + if argv0 == '': + raise ValueError('arch %s not support' % arch) + + kprobeArgs = self.kprobeArgsFormat % ( + argv0, argv0, argv1, argv0, argv0, argv0, argv0, argv2, commArgs) + return kprobeArgs + + def _getFsMountInfo(self): + devList = [] + if self.devname is not None: + devList.append('/dev/'+self.devname) + else: + sysfsBlockDirList = os.listdir("/sys/block") + for dev in sysfsBlockDirList: + devList.append('/dev/'+dev) + with open("/proc/mounts") as f: + fsmountInfo = list(filter(lambda x: any( + e in x for e in devList), f.readlines())) + return fsmountInfo + + def config(self): + devt = self.devt + + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + if devt[0] > 0: + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + if devt[0] > 0: + dev = getDevt(self.devname) + if dev == min(devt): + echoFile(filterKprobe, + "dev>="+str(min(devt))+"&&dev<="+str(max(devt))) + else: + echoFile(filterKprobe, "dev=="+str(dev)) + echoFile(enableKprobe, "1") + fmt = execCmd("grep print "+self.kprobeDir+"/"+probe+"/format") + matchObj = re.match(r'(.*) dev=(.*) inode_num=(.*)', fmt) + if 'x' in matchObj.group(2): + self.outlogFormatBase = 16 + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(fsstatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(fsstatClass, self).stop() + + def clear(self): + for exp in self.expression: + probe = 'p_'+exp.split()[1]+'_0' + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if not os.path.exists(enableKprobe): + continue + echoFile(enableKprobe, "0") + if self.devt[0] > 0: + filterKprobe = self.kprobeDir+"/"+probe+"/filter" + echoFile(filterKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + super(fsstatClass, self).clear() + + def _paserTraceToStat(self, traceText): + bwTotal = 0 + stat = {} + mStat = {} + fileInfoDict = { + 'device': 0, 'mntfname': '', 'bfname': '', 'd1fname': '', + 'd2fname': '', 'd3fname': '', 'fsmountinfo': '', 'ino': 0, + 'pid': 0} + commArgs = self.ftracePaserCommArgs + hasCommArgs = True if len(commArgs) else False + + # pool-1-thread-2-5029 [002] .... 5293018.252338: p_ext4_file_write_iter_0:\ + # (ext4_file_write_iter+0x0/0x6d0 [ext4]) dev=265289729 inode_num=530392 len=38 + # ... + for entry in traceText: + if ('dev=' not in entry) or ('.so' in entry and 'lib' in entry) or ( + '=\"etc\"' in entry) or ('=\"usr\"' in entry and ( + '=\"bin\"' in entry or '=\"sbin\"' in entry)): + continue + + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) dev=(.*) inode_num=(.*) len=(.*)'+ + ' mntfname=(.*) bfname=(.*) d1fname=(.*) d2fname=(.*)'+ + ' d3fname=(.*)'+commArgs, entry) + if matchObj is None: + continue + + pid = (matchObj.group(1).rsplit('-', 1))[1].strip() + dev = int(matchObj.group(4), self.outlogFormatBase) + if (self.pid is not None and int(pid) != self.pid) or \ + str(dev) == '0': + continue + + if hasCommArgs: + comm = matchObj.group(12).strip("\"") + else: + comm = (matchObj.group(1).rsplit('-', 1))[0].strip() + comm = fixComm(comm, pid) + if '..' in comm: + continue + + device = self.getDevNameByDevt(dev) + if device == '-': + continue + if self.miscStat is not None: + disk = self.getMasterDev(dev) + if not mStat.has_key(disk): + mStat.setdefault(disk, {}) + stat = mStat[disk] + + ino = int(matchObj.group(5), self.outlogFormatBase) + inoTask = str(ino)+':'+str(comm)+':'+device + if not stat.has_key(inoTask): + fsmountinfo = [f for f in self.fsmountInfo if ('/dev/'+device) in f] + fileInfoDict['device'] = device + fileInfoDict['mntfname'] = matchObj.group(7).strip("\"") + fileInfoDict['bfname'] = matchObj.group(8).strip("\"") + fileInfoDict['d1fname'] = matchObj.group(9).strip("\"") + fileInfoDict['d2fname'] = matchObj.group(10).strip("\"") + fileInfoDict['d3fname'] = matchObj.group(11).strip("\"") + fileInfoDict['fsmountinfo'] = fsmountinfo + fileInfoDict['ino'] = ino + fileInfoDict['pid'] = pid + stat.setdefault(inoTask, + {"inode":str(ino), "comm": comm, "tgid": getTgid(pid), "pid": pid, + "cnt_wr": 0, "bw_wr": 0, "cnt_rd": 0, "bw_rd": 0, "device": device, + "cid":getContainerId(pid), "file": getFullName(fileInfoDict)}) + + size = int(matchObj.group(6), self.outlogFormatBase) + if 'filemap_fault' in entry or 'page_mkwrite' in entry: + size = PAGESIZE + if 'write' in entry or 'page_mkwrite' in entry: + stat[inoTask]["cnt_wr"] += 1 + stat[inoTask]["bw_wr"] += int(size) + if 'read' in entry or 'filemap_fault' in entry: + stat[inoTask]["cnt_rd"] += 1 + stat[inoTask]["bw_rd"] += int(size) + if pid != stat[inoTask]["pid"]: + stat[inoTask]["pid"] = pid + stat[inoTask]["tgid"] = getTgid(pid) + if stat[inoTask]["cid"] == '-': + stat[inoTask]["cid"] = getContainerId(pid) + bwTotal += int(size) + return bwTotal,stat,mStat + + def _joinMiscStat(self, mStat): + for d,val in self.miscStat: + if d not in mStat.keys(): + mStat.setdefault(d, {}) + mStat[d].update(dict(val)) + tmpStat = [] + for d,val in mStat.items(): + idxSort = 'bw_wr' + if self.getDiskStatInd(d, 'w_iops') < self.getDiskStatInd(d, 'r_iops'): + idxSort = 'bw_rd' + s = sorted( + val.items(), key=lambda e: (e[1][idxSort]), reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return 0 + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","fsstats":[]}' + fstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + fstatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + fsstatJsonStr = '{\ + "inode":0,"comm":"","tgid":0,"pid":0,"cnt_rd":0,\ + "bw_rd":0,"cnt_wr":0,"bw_wr":0,"device":0,"cid":0,"file":0}' + fsstatDict = json.loads( + fsstatJsonStr, object_pairs_hook=OrderedDict) + for key, val in item.items(): + fsstatDict[key] = val + fstatDicts["fsstats"].append(fsstatDict) + if len(fstatDicts["fsstats"]) > 0: + self.writeDataToJson(json.dumps(fstatDicts)) + + def printStat(self, stat): + secs = self.cycle + print("%-20s%-8s%-8s%-24s%-8s%-12s%-8s%-12s%-12s%-12s%-32s%s" + % ("comm", "tgid", "pid", "cid", "cnt_rd", "bw_rd", "cnt_wr", + "bw_wr", "inode", "device", "filepath")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["cnt_wr"] + item["cnt_rd"]) == 0: + continue + item["bw_wr"] = \ + humConvert(item["bw_wr"], True).replace('s', stSecs) if item["bw_wr"] else 0 + item["bw_rd"] = \ + humConvert(item["bw_rd"], True).replace('s', stSecs) if item["bw_rd"] else 0 + print("%-20s%-8s%-8s%-24s%-8d%-12s%-8d%-12s%-12s%-12s%s" + % (item["comm"], item["tgid"], item["pid"], item["cid"][0:20], + item["cnt_rd"], item["bw_rd"], item["cnt_wr"], item["bw_wr"], + item["inode"], item["device"], item["file"])) + print("") + + def show(self): + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = f.read().split('\n') + #traceText = f.readlines() + #traceText = \ + # list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + bwTotal,stat,mStat = self._paserTraceToStat(traceText) + + if self.miscStat is not None: + return self._joinMiscStat(mStat) + elif (self.bwThresh and (bwTotal/secs) < self.bwThresh): + return + + stat = sorted(stat.items(), key=lambda e: ( + e[1]["bw_wr"]+e[1]["bw_rd"]), reverse=True)[:self.top] + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + if self.disableShow() == False: + super(fsstatClass, self).show() + + if self.enableJsonShow() == True: + self.showJson(stat) + else: + self.printStat(stat) + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py new file mode 100755 index 00000000..18290315 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iofsstat.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import threading +from collections import OrderedDict +from iostatClass import iostatClass +from fsstatClass import fsstatClass +from promiscClass import promiscClass +import time + +global_iofsstat_stop = False +def signal_exit_handler(signum, frame): + global global_iofsstat_stop + global_iofsstat_stop = True + +def exit_handler(): + global global_iofsstat_stop + global_iofsstat_stop = True + +def iofsstatStart(argv): + global global_iofsstat_stop + global_iofsstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iofsstat.py -d vda -c 1 + Report iops and bps of process for vda per 1secs + ./iofsstat.py -d vda1 --fs -c 1 + Report fs IO-BW and file of process for vda1(must be parttion mounted by filesystem) per 1secs + ./iofsstat.py -m -c 5 -t 5 + Report top5 iops&&bps&&file of process with misc mode per 5secs + ./iofsstat.py -d vda -c 1 -b 1048576 -i 350 + Report process that iops over 350 or bps over 1048576 for vda per 1secs + ./iofsstat.py -u 90 + Report disk that io-util over %90 + """ + parser = argparse.ArgumentParser( + description="Report IO statistic for partitions.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-T','--Timeout', help='Specify the timeout for program exit(secs).') + parser.add_argument('-t','--top', help='Report the TopN with the largest IO resources.') + parser.add_argument('-u','--util_thresh', help='Specify the util-thresh to report.') + parser.add_argument('-b','--bw_thresh', help='Specify the BW-thresh to report.') + parser.add_argument('-i','--iops_thresh', help='Specify the IOPS-thresh to report.') + parser.add_argument('-c','--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-d','--device', help='Specify the disk name.') + parser.add_argument('-p','--pid', help='Specify the process id.') + parser.add_argument('-j','--json', help='Specify the json-format output.') + parser.add_argument('-f','--fs', action='store_true', + help='Report filesystem statistic for partitions.') + parser.add_argument('-P','--Pattern', action='store_true', + help='Report IO pattern(--fs not support).') + parser.add_argument('-n','--nodiskStat', action='store_true', + help='Not report disk stat.') + parser.add_argument('-m','--misc', action='store_true', + help='Promiscuous mode.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + secs = float(args.cycle) if args.cycle is not None else 0 + devname = args.device + pid = int(args.pid) if args.pid else None + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + interval = secs if loop == True else 1 + if args.misc: + c = promiscClass(devname, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, + args.Pattern) + else: + if args.fs: + c = fsstatClass(devname, pid, args.util_thresh, args.bw_thresh, + args.top, args.json, args.nodiskStat, None, args.Pattern) + else: + c = iostatClass(devname, pid, args.util_thresh, args.iops_thresh, + args.bw_thresh, args.top, args.json, args.nodiskStat, None, + args.Pattern) + c.config() + while global_iofsstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iofsstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py new file mode 100755 index 00000000..e7d03e85 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/iostatClass.py @@ -0,0 +1,244 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from diskstatClass import diskstatClass +from common import getDevtRegion +from common import humConvert,echoFile +from common import getContainerId + + +class iostatClass(diskstatClass): + def __init__( + self, devname, pid, utilThresh, iopsThresh, bwThresh, + top, json, nodiskStat, miscStat, Pattern): + super(iostatClass, self).__init__( + devname, utilThresh, json, nodiskStat, Pattern) + self.pid = pid + self.miscStat = miscStat + self.top = int(top) if top is not None else 99999999 + self.iopsThresh = int(iopsThresh) if iopsThresh is not None else 0 + self.bwThresh = int(bwThresh) if bwThresh is not None else 0 + self.devt = min(getDevtRegion(devname)) if devname is not None else -1 + self.tracingDir = "/sys/kernel/debug/tracing/instances/iofsstat4io" + self.blkTraceDir = self.tracingDir+"/events/block" + + def config(self): + devt = self.devt + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + if devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "dev=="+str(devt)) + else: + echoFile(self.blkTraceDir+"/block_getrq/filter", "") + echoFile(self.blkTraceDir+"/block_getrq/enable", "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + super(iostatClass, self).start() + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + super(iostatClass, self).stop() + + def clear(self): + echoFile(self.blkTraceDir+"/block_getrq/enable", "0") + if self.devt > 0: + echoFile(self.blkTraceDir+"/block_getrq/filter", "0") + super(iostatClass, self).clear() + + def showJson(self, stat): + secs = self.cycle + statJsonStr = '{"time":"","iostats":[]}' + iostatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iostatDicts['time'] = time.strftime( + '%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat.items(): + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + iostatJsonStr = '{\ + "comm":"","pid":0,"bps_rd":0,"iops_rd":0,"iops_wr":0,"bps_wr":0,"device":0}' + iostatDict = json.loads(iostatJsonStr, object_pairs_hook=OrderedDict) + for key in ['comm', 'pid', 'bps_rd', 'iops_rd', 'iops_wr', 'bps_wr', 'device']: + iostatDict[key] = item[key] + iostatDicts["iostats"].append(iostatDict) + if len(iostatDicts["iostats"]) > 0: + self.writeDataToJson(json.dumps(iostatDicts)) + + def patternIdx(self, size): + dp = [ + ("pat_W4K", (4*1024)), ("pat_W16K", (16*1024)), + ("pat_W32K", (32*1024)), ("pat_W64K",(64*1024)), + ("pat_W128K", (128*1024)), ("pat_W256K", (256*1024)), + ("pat_W512K", (512*1024))] + for d in dp: + if size <= d[1]: + return d[0] + return 'pat_Wlarge' + + def patternPercent(self, pat, total): + if total == 0 or pat == 0: + return '0' + return format(pat / (total * 1.0) * 100, '.2f') + '%' + + def show(self): + iopsTotal = 0 + WrIopsTotal = 0 + RdIopsTotal = 0 + bwTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + stat = {} + mStat = {} + secs = self.cycle + with open(self.tracingDir+"/trace") as f: + traceText = list( + filter(lambda x: 'block_getrq' in x, f.readlines())) + # jbd2/vda1-8-358 ... : block_getrq: 253,0 WS 59098136 + 120 [jbd2/vda1-8] + for entry in traceText: + oneIO = entry.split() + matchObj = re.match( + r'(.*) \[([^\[\]]*)\] (.*) \[([^\[\]]*)\]\n', entry) + comm = matchObj.group(4) + pid = matchObj.group(1).rsplit('-', 1)[1].strip() + if self.pid is not None and int(pid) != self.pid: + continue + devinfo = oneIO[-6-comm.count(' ')].split(',') + dev = ((int(devinfo[0]) << 20) + int(devinfo[1])) + if str(dev) == '0': + continue + device = self.getDevNameByDevt(dev) + if device == '-' or self.notCareDevice(device) == True: + continue + if self.miscStat is not None: + if not mStat.has_key(device): + mStat.setdefault(device, {}) + stat = mStat[device] + iotype = oneIO[-5-comm.count(' ')] + sectors = oneIO[-2-comm.count(' ')] + task = str(pid)+':'+device + if bool(stat.has_key(task)) != True: + stat.setdefault(task, + {"comm":"", "pid": pid, "iops_rd": 0, + "iops_wr": 0, "bps_rd": 0, "bps_wr": 0, + "flushIO": 0, "device": device, + "cid":getContainerId(pid), + "pat_W4K":0, "pat_W16K":0, "pat_W32K":0, + "pat_W64K":0, "pat_W128K":0, "pat_W256K":0, + "pat_W512K":0, "pat_Wlarge":0}) + size = int(sectors) * 512 + if len(comm) > 0: + stat[task]["comm"] = comm + if 'R' in iotype: + stat[task]["iops_rd"] += 1 + stat[task]["bps_rd"] += size + bwTotal += size + iopsTotal += 1 + if 'W' in iotype: + stat[task]["iops_wr"] += 1 + stat[task]["bps_wr"] += size + bwTotal += size + iopsTotal += 1 + if self.Pattern and size > 0 and size < 1024 * 1024 * 100: + stat[task][self.patternIdx(size)] += 1 + if 'F' in iotype: + stat[task]["flushIO"] += 1 + + if self.iopsThresh or self.bwThresh: + if (self.bwThresh and bwTotal >= self.bwThresh) or \ + (self.iopsThresh and iopsTotal >= self.iopsThresh): + pass + else: + return + + if self.enableJsonShow() == False: + print(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime())) + super(iostatClass, self).show() + + if self.miscStat is not None: + tmpStat = [] + for d,val in mStat.items(): + s = sorted(val.items(), + key=lambda e: (e[1]["bps_wr"]+e[1]["bps_rd"]), + reverse=True)[:self.top] + tmpStat.append((d, s)) + del self.miscStat[:] + self.miscStat.extend(tmpStat) + return + + stat = sorted(stat.items(), + key=lambda e: (e[1]["iops_rd"] + e[1]["iops_wr"]), + reverse=True)[:self.top] + + if self.enableJsonShow() == True: + self.showJson(stat) + return + + tPattern = '' + if self.Pattern: + WrIopsTotal = 0 + RdIopsTotal = 0 + WrBwTotal = 0 + RdBwTotal = 0 + tPattern = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + "pat_W4K", "pat_W16K", "pat_W32K", "pat_W64K", "pat_W128K", + "pat_W256K", "pat_W512K", "pat_Wlarge" + )) + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", tPattern)) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in stat: + if (item["iops_rd"] + item["iops_wr"]) == 0: + continue + patPercent = '' + if self.Pattern: + WrIopsTotal += item["iops_wr"] + RdIopsTotal += item["iops_rd"] + WrBwTotal += item["bps_wr"] + RdBwTotal += item["bps_rd"] + patPercent = ('%-12s%-12s%-12s%-12s%-12s%-12s%-12s%-12s' % ( + self.patternPercent(item["pat_W4K"], item["iops_wr"]), + self.patternPercent(item["pat_W16K"], item["iops_wr"]), + self.patternPercent(item["pat_W32K"], item["iops_wr"]), + self.patternPercent(item["pat_W64K"], item["iops_wr"]), + self.patternPercent(item["pat_W128K"], item["iops_wr"]), + self.patternPercent(item["pat_W256K"], item["iops_wr"]), + self.patternPercent(item["pat_W512K"], item["iops_wr"]), + self.patternPercent(item["pat_Wlarge"], item["iops_wr"]) + )) + item["bps_rd"] = \ + humConvert(item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else 0 + item["bps_wr"] = \ + humConvert(item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else 0 + patPercent += item["cid"] + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-12s%s' % (item["comm"], + str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], + item["device"], patPercent)) + if self.Pattern: + print('totalIops:%d(r:%d, w:%d), totalBw:%s(r:%s, w:%s)' % + (iopsTotal, RdIopsTotal, WrIopsTotal, + (humConvert(bwTotal, True).replace('s', stSecs) if bwTotal else 0), + (humConvert(RdBwTotal, True).replace('s', stSecs) if RdBwTotal else 0), + (humConvert(WrBwTotal, True).replace('s', stSecs) if WrBwTotal else 0))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py new file mode 100755 index 00000000..b3f90fc5 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iofstool/promiscClass.py @@ -0,0 +1,215 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import time +import re +import json +from collections import OrderedDict +from common import humConvert +from iostatClass import iostatClass +from fsstatClass import fsstatClass + + +class promiscClass(): + def __init__( + self, devname, utilThresh, iopsThresh, bwThresh, top, json, + nodiskStat, Pattern): + self._iostat = [] + self._fsstat = [] + self.fs = fsstatClass(devname, None, utilThresh, bwThresh, + top, json, nodiskStat, self._fsstat, Pattern) + self.io = iostatClass(devname, None, utilThresh, iopsThresh, + bwThresh, top, json, nodiskStat, self._iostat, + Pattern) + + + def _selectKworker(self, iostat, fsItem, kworker): + select = None + largeFound = False + diff = sys.maxsize + for k in kworker: + fsRestBw = fsItem["bw_wr"] + if 'restBW' in fsItem.keys(): + fsRestBw = fsItem["restBW"] + if fsRestBw > (iostat[k]["bps_wr"] * 15) or \ + (fsRestBw * 50) < iostat[k]["bps_wr"]: + continue + d = abs(fsItem["bw_wr"] - iostat[k]["bps_wr"]) + diff = min(d, diff) + if iostat[k]["bps_wr"] > fsItem["bw_wr"]: + if not largeFound or diff == d: + select = k + largeFound = True + continue + if not largeFound and diff == d: + select = k + return select + + + def _addBioToKworker(self, iostat, kworker, fsItem): + repeated = False + k = self._selectKworker(iostat, fsItem, kworker) + if not k: + return False, 0 + if 'bufferio' not in iostat[k].keys(): + iostat[k].setdefault('bufferio', []) + task = fsItem["comm"]+':'+str(fsItem["tgid"])+':'+str(fsItem["pid"])+\ + ':'+fsItem["cid"][0:20] + bio = {'task': task, 'Wrbw': fsItem["bw_wr"], 'file': fsItem["file"], + 'device': fsItem["device"]} + for d in iostat[k]["bufferio"]: + if task == d['task'] and d['file'] == bio['file'] and \ + d['device'] == bio['device']: + d['Wrbw'] = max(d['Wrbw'], bio["Wrbw"]) + repeated = True + break + if not repeated: + iostat[k]["bufferio"].append(bio) + return True, iostat[k]["bps_wr"] + + + def _checkDeleteItem(self, addOK, costBW, item): + now = time.time() + # After 10 secs without adding to any kworker, we will delete the fsItem + agingTime = 10 + if 'restBW' not in item.keys(): + item.setdefault('restBW', item["bw_wr"]) + item.setdefault('startAging', now) + if addOK: + item["startAging"] = time.time() + item["restBW"] = item["restBW"] - costBW if addOK else item["restBW"] + if item["restBW"] <= 0 or (item["restBW"] < item["bw_wr"] and \ + (now - item["startAging"]) >= agingTime): + return True + return False + + + def _miscIostatFromFsstat(self): + fsstats = self._fsstat + iostats = dict(self._iostat) + for disk, fsItems in fsstats: + if disk not in iostats.keys(): + continue + rmList = [] + iostat = dict(iostats[disk]) + kworker = [key for key,val in iostat.items() if 'kworker' in val['comm']] + for key, item in fsItems: + taskI = item["pid"]+':'+disk + if taskI in iostat.keys(): + if 'file' not in iostat[taskI].keys(): + iostat[taskI].setdefault('file', []) + iostat[taskI]['cid'] = item['cid'] + iostat[taskI]["file"].append(item["file"]) + if item["bw_wr"] <= (iostat[taskI]["bps_wr"] * 15): + rmList.append((key, item)) + continue + if kworker: + if item["bw_wr"] < item["bw_rd"]: + rmList.append((key, item)) + continue + addOK,cost = self._addBioToKworker(iostat, kworker, item) + deleted = self._checkDeleteItem(addOK, cost, item) + if deleted: + rmList.append((key, item)) + for key, item in rmList: + fsItems.remove((key, item)) + iostats[disk] = iostat + return iostats + + + def _miscShowJson(self, iostats): + secs = self.io.cycle + statJsonStr = '{"time":"","mstats":[]}' + mstatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + mstatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + stSecs = str(secs)+'s' if secs > 1 else 's' + + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + if 'file' not in item.keys(): + item.setdefault('file', '-') + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + mstatDicts["mstats"].append(item) + if len(mstatDicts["mstats"]) > 0: + self.io.writeDataToJson(json.dumps(mstatDicts)) + + + def miscShow(self): + secs = self.io.cycle + if not self._fsstat and not self._iostat: + return + + iostats = self._miscIostatFromFsstat() + if not iostats: + return + tmp = {} + for d in iostats.values(): + tmp.update(dict(d)) + iostats = sorted( + tmp.items(), + key=lambda e: (int(e[1]["bps_rd"])+int(e[1]["bps_wr"])), + reverse=True) + if self.io.enableJsonShow() == True: + self._miscShowJson(iostats) + return + + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + ("comm", "pid", "cid", "iops_rd", "bps_rd", "iops_wr", "bps_wr", + "device", "file")) + stSecs = str(secs)+'s' if secs > 1 else 's' + for key, item in iostats: + if (item["iops_rd"]+item["iops_wr"]) == 0 or (item["bps_rd"]+item["bps_wr"]) == 0: + continue + item["bps_rd"] = humConvert( + item["bps_rd"], True).replace('s', stSecs) if item["bps_rd"] else '0' + item["bps_wr"] = humConvert( + item["bps_wr"], True).replace('s', stSecs) if item["bps_wr"] else '0' + file = str(item["file"]) if 'file' in item.keys() else '-' + print('%-20s%-8s%-24s%-12s%-16s%-12s%-12s%-8s%s' % + (item["comm"], str(item["pid"]), item["cid"][0:20], str(item["iops_rd"]), + item["bps_rd"], str(item["iops_wr"]), item["bps_wr"], item["device"], file)) + if 'kworker' in item["comm"] and 'bufferio' in item.keys(): + for i in item["bufferio"]: + i["Wrbw"] = humConvert(i["Wrbw"], True).replace('s', stSecs) + print(' |----%-32s WrBw:%-12s Device:%-8s File:%s' % + (i['task'], i["Wrbw"], i["device"], i["file"])) + print("") + + + def config(self): + self.fs.config() + self.io.config() + + def start(self): + self.clear() + self.fs.start() + self.io.start() + + def stop(self): + self.fs.stop() + self.io.stop() + + def clear(self): + del self._iostat[:] + + def show(self): + self.fs.show() + self.io.show() + self.miscShow() + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py new file mode 100644 index 00000000..cb8e4b62 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +if __name__ == "__main__": + pass diff --git a/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py new file mode 100755 index 00000000..69f473c8 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMon/tools/iowaitstat/iowaitstat.py @@ -0,0 +1,354 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import signal +import string +import argparse +import time +import re +import json +import threading +from collections import OrderedDict +from subprocess import PIPE, Popen +import shlex + +global_iowaitstat_stop = False + + +def signal_exit_handler(signum, frame): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def exit_handler(): + global global_iowaitstat_stop + global_iowaitstat_stop = True + +def execCmd(cmd): + p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + return p.stdout.read().decode('utf-8') + + +def getTgid(pid): + try: + with open("/proc/"+str(pid)+"/status") as f: + return ''.join(re.findall(r'Tgid:(.*)', f.read())).lstrip() + except IOError: + return '-' + return '-' + + +def fixComm(comm, pid): + try: + if ".." in comm: + with open("/proc/"+str(pid)+"/comm") as f: + return f.read().rstrip('\n') + except IOError: + return comm + return comm + + +def echoFile(filename, txt): + os.system("echo \""+txt+"\" > "+filename) + + +def echoFileAppend(filename, txt): + os.system("echo \""+txt+"\" >> "+filename) + + +def supportKprobe(name): + cmd = "cat /sys/kernel/debug/tracing/available_filter_functions | grep " + name + ss = execCmd(cmd).strip() + for res in ss.split('\n'): + if ':' in res: + res = res.split(":", 1)[1] + if ' [' in res: # for ko symbol + res = res.split(" [", 1)[0] + if res == name: + return True + return False + +class iowaitClass(): + def __init__(self, pid, cycle, top, json, iowait_thresh): + self.pid = pid + self.top = int(top) if top is not None else 99999999 + self.json = json + self.cycle = cycle + self.iowait_thresh = int(iowait_thresh) if iowait_thresh is not None else 0 + self.kprobeEvent = "/sys/kernel/debug/tracing/kprobe_events" + self.tracingDir = "/sys/kernel/debug/tracing/instances/iowait" + self.kprobeDir = self.tracingDir+"/events/kprobes" + self.expression = [] + self.kprobe = [] + self.cpuStatIowait = {'sum': 0, 'iowait': 0} + if json: + self.fJson = open(json, 'w+') + + for kprobe,retProbe in {'io_schedule_timeout':True, 'io_schedule':True}.items(): + if supportKprobe(kprobe) == False: + print("not available %s kprobe" % kprobe) + continue + self.expression.append('p:p_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('p_%s_0' % kprobe) + if retProbe == True: + self.expression.append('r:r_%s_0 %s' % (kprobe, kprobe)) + self.kprobe.append('r_%s_0' % kprobe) + if len(self.kprobe) == 0: + print "not available kprobe" + sys.exit(0) + + def config(self): + if not os.path.exists(self.tracingDir): + os.mkdir(self.tracingDir) + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + + echoFileAppend(self.kprobeEvent, exp) + echoFile(enableKprobe, "1") + + def start(self): + echoFile(self.tracingDir+"/trace", "") + echoFile(self.tracingDir+"/tracing_on", "1") + with open("/proc/stat") as fStat: + cpuStatList = map(long, fStat.readline().split()[1:]) + self.cpuStatIowait['sum'] = sum(cpuStatList) + self.cpuStatIowait['iowait'] = cpuStatList[4] + + def stop(self): + echoFile(self.tracingDir+"/tracing_on", "0") + + def clear(self): + for exp in self.expression: + probe = exp.split()[0].split(':')[1] + enableKprobe = self.kprobeDir+"/"+probe+"/enable" + if os.path.exists(enableKprobe): + echoFile(enableKprobe, "0") + echoFileAppend(self.kprobeEvent, '-:%s' % probe) + if self.json: + self.fJson.close() + + def writeDataToJson(self, data): + self.fJson.write(data+'\n') + + def showJson(self, stat, totalTimeout, gloabIowait): + top = 0 + statJsonStr = '{"time":"", "global iowait":0,"iowait":[]}' + iowaitStatDicts = json.loads(statJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDicts['time'] = time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()) + iowaitStatDicts['global iowait'] = gloabIowait + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + iowaitStatJsonStr = '{"comm":"","pid":0,"tgid":0,"timeout":0,"iowait":0,"reason":0}' + iowaitStatDict = json.loads( + iowaitStatJsonStr, object_pairs_hook=OrderedDict) + iowaitStatDict["comm"] = item["comm"] + iowaitStatDict["pid"] = pid + iowaitStatDict["tgid"] = item["tgid"] + iowaitStatDict["timeout"] = item["timeout"] + iowaitStatDict["iowait"] = iowait + iowaitStatDict["reason"] = reason + iowaitStatDicts["iowait"].append(iowaitStatDict) + if len(iowaitStatDicts["iowait"]) > 0: + self.writeDataToJson(json.dumps(iowaitStatDicts)) + + def show(self): + top = 0 + totalTimeout = 0 + stat = {} + secs = self.cycle + traceText = [] + + with open("/proc/stat") as fStat: + statList = map(long, fStat.readline().split()[1:]) + gloabIowait = float(format( + (statList[4]-self.cpuStatIowait['iowait'])*100.0 / + (sum(statList)-self.cpuStatIowait['sum']), '.2f')) + if gloabIowait < self.iowait_thresh: + return + + with open(self.tracingDir+"/trace") as f: + traceLoglist = list(filter(lambda x: any(e in x for e in self.kprobe), f.readlines())) + traceText = traceLoglist + + # jbd2/vda2-8-605 [001] .... 38890020.539912: p_io_schedule_0: (io_schedule+0x0/0x40) + # jbd2/vda2-8-605 [002] d... 38890020.540633: r_io_schedule_0: (bit_wait_io+0xd/0x50 <- io_schedule) + # <...>-130620 [002] .... 38891029.116442: p_io_schedule_timeout_0: (io_schedule_timeout+0x0/0x40) + # <...>-130620 [002] d... 38891029.123657: r_io_schedule_timeout_0: (balance_dirty_pages+0x270/0xc60 <- io_schedule_timeout) + for entry in traceText: + matchObj = re.match(r'(.*) \[([^\[\]]*)\] (.*) (.*): (.*): (.*)\n', entry) + if matchObj is None: + continue + commInfo = matchObj.group(1).rsplit('-', 1) + pid = commInfo[1].strip() + if self.pid is not None and pid != self.pid: + continue + if bool(stat.has_key(pid)) != True: + comm = fixComm(commInfo[0].lstrip(), pid) + if '..' in comm: + continue + stat.setdefault(pid, + {"comm": comm, "tgid": getTgid(pid), + "timeout": 0, "reason": {}, "entry": []}) + stat[pid]["entry"].append({ + 'time':matchObj.group(4), + 'point':matchObj.group(5), + 'trace':matchObj.group(6)}) + + if stat: + for key,item in stat.items(): + item["entry"] = sorted(item["entry"], key=lambda e: float(e["time"]), reverse=False) + count = 0 + startT = 0 + for entry in item["entry"]: + count += 1 + if (count % 2 != 0 and 'p_' not in entry['point']) or \ + (count % 2 == 0 and 'r_' not in entry['point']): + count = 0 + startT = 0 + continue + + if count % 2 != 0: + startT = float(entry['time']) + continue + + if startT > 0 and float(entry['time']) > startT: + if re.split('[(,+]', entry['trace'])[1] in re.split('[-,)]', entry['trace'])[1]: + count = 0 + startT = 0 + continue + item['timeout'] += (float(entry['time']) - startT) + totalTimeout += (float(entry['time']) - startT) + startT = 0 + if entry['trace'] not in item['reason'].keys(): + item['reason'].setdefault(entry['trace'], 0) + item['reason'][entry['trace']] += 1 + + if stat: + stat = OrderedDict(sorted(stat.items(), key=lambda e: e[1]["timeout"], reverse=True)) + if self.json: + self.showJson(stat, totalTimeout, gloabIowait) + return + else: + head = str(time.strftime('%Y/%m/%d %H:%M:%S', time.localtime()))+' -> global iowait%: '+str(gloabIowait) + print head + + print("%-32s%-8s%-8s%-16s%-12s%s" % ("comm", "tgid", "pid", "waitio(ms)", "iowait(%)", "reasons")) + for pid, item in stat.items(): + if item["timeout"] == 0: + continue + if top >= self.top: + break + top += 1 + iowait = str(round(item["timeout"] / totalTimeout * gloabIowait, 2)) + item["timeout"] = str(round(item["timeout"]*1000, 3)) + reason = '' + maxCnt = 0 + for key, val in item['reason'].items(): + if 'balance_dirty' in key: + reason = 'Too many dirty pages' + break + elif 'blk_mq_get_tag' in key: + reason = 'Device queue full' + break + elif 'get_request' in key: + reason = 'Ioscheduler queue full' + break + else: + if val > maxCnt: + reason = 'Unkown[stacktrace:'+key.replace('<-', '->')+']' + maxCnt = val + print("%-32s%-8s%-8s%-16s%-12s%s" + % (item["comm"], item["tgid"], pid, item["timeout"], iowait, str(reason))) + print("") + + def entry(self, interval): + self.start() + time.sleep(float(interval)) + self.stop() + self.show() + +def iowaitstatStart(argv): + global global_iowaitstat_stop + global_iowaitstat_stop = False + if os.geteuid() != 0: + print("%s" % ("This program must be run as root. Aborting.")) + sys.exit(0) + examples = """e.g. + ./iowaitstat.py + Report iowait for tasks + ./iowaitstat.py -c 1 + Report iowait for tasks per secs + ./iowaitstat.py -p [PID] -c 1 + Report iowait for task with [PID] per 1secs + """ + parser = argparse.ArgumentParser( + description="Report iowait for tasks.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples) + parser.add_argument('-p', '--pid', help='Specify the process id.') + parser.add_argument('-T', '--Timeout', + help='Specify the timeout for program exit(secs).') + parser.add_argument( + '-t', '--top', help='Report the TopN with the largest iowait.') + parser.add_argument('-c', '--cycle', help='Specify refresh cycle(secs).') + parser.add_argument('-j', '--json', help='Specify the json-format output.') + parser.add_argument('-w','--iowait_thresh', help='Specify the iowait-thresh to report.') + args = parser.parse_args(argv) if argv else parser.parse_args() + + pid = int(args.pid) if args.pid else None + secs = float(args.cycle) if args.cycle is not None else 0 + if argv is None: + signal.signal(signal.SIGINT, signal_exit_handler) + signal.signal(signal.SIGHUP, signal_exit_handler) + signal.signal(signal.SIGTERM, signal_exit_handler) + if args.Timeout is not None: + timeoutSec = args.Timeout if args.Timeout > 0 else 10 + secs = secs if secs > 0 else 1 + if argv is None: + signal.signal(signal.SIGALRM, signal_exit_handler) + signal.alarm(int(timeoutSec)) + else: + timer = threading.Timer(int(timeoutSec), exit_handler) + timer.start() + loop = True if secs > 0 else False + c = iowaitClass(pid, secs, args.top, args.json, args.iowait_thresh) + c.config() + interval = secs if loop == True else 1 + while global_iowaitstat_stop != True: + c.entry(interval) + if loop == False: + break + c.clear() + +def main(): + iowaitstatStart(None) + +if __name__ == "__main__": + main() diff --git a/source/tools/monitor/ioMonitor/ioMonitor.sh b/source/tools/monitor/ioMonitor/ioMonitor.sh new file mode 100755 index 00000000..dfd3d468 --- /dev/null +++ b/source/tools/monitor/ioMonitor/ioMonitor.sh @@ -0,0 +1,11 @@ +#!/bin/sh +#****************************************************************# +# ScriptName: ioMonitor.sh +# Author: $SHTERM_REAL_USER@alibaba-inc.com +# Create Date: 2021-06-06 16:53 +# Modify Author: $SHTERM_REAL_USER@alibaba-inc.com +# Modify Date: 2021-06-06 16:53 +# Function: +#***************************************************************# +TOOLS_ROOT="$SYSAK_WORK_PATH/tools" +python $TOOLS_ROOT/ioMon/ioMonitorMain.py $* diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index cc3f9066..7a64bdcc 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -125,4 +125,19 @@ metrics: head: value help: "buddyinfo of system from /proc/loadavg" type: "gauge" + - title: sysak_IOMonIndForDisksIO + from: IOMonIndForDisksIO + head: value + help: "Disk IO indicators and abnormal events" + type: "gauge" + - title: sysak_IOMonIndForSystemIO + from: IOMonIndForSystemIO + head: value + help: "System indicators and abnormal events about IO" + type: "gauge" + - title: sysak_IOMonDiagLog + from: IOMonDiagLog + head: value + help: "Diagnose log for IO exception" + type: "gauge" -- Gitee From b62cef584bba1fc3558081e757cc758ab8945c82 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Wed, 22 Feb 2023 10:59:55 +0800 Subject: [PATCH 06/21] unity: bpfsample: add bpfsample which is period sampling and add corresponded doc Signed-off-by: Shuyi Cheng --- .../tools/monitor/unity/beaver/guide/bpf.md | 71 ++++++++----- .../monitor/unity/beaver/guide/bpf_perf.md | 100 ++++++++++++++++++ .../monitor/unity/collector/plugin/Makefile | 2 +- .../unity/collector/plugin/bpfsample/Makefile | 8 ++ .../plugin/bpfsample/bpfsample.bpf.c | 23 ++++ .../collector/plugin/bpfsample/bpfsample.c | 41 +++++++ .../collector/plugin/bpfsample/bpfsample.h | 52 +++++++++ 7 files changed, 270 insertions(+), 27 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/guide/bpf_perf.md create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c create mode 100644 source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h diff --git a/source/tools/monitor/unity/beaver/guide/bpf.md b/source/tools/monitor/unity/beaver/guide/bpf.md index 81b67b62..9edada61 100644 --- a/source/tools/monitor/unity/beaver/guide/bpf.md +++ b/source/tools/monitor/unity/beaver/guide/bpf.md @@ -1,13 +1,13 @@ -## 基于 eBPF 的监控开发手册 +## 基于 eBPF 的周期性采样监控开发手册 -我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample` 路径提供了一个基于 eBPF 的周期性采样监控开发样例。其主要包含三个部分： 1. Makefile: 用于编译该工具； -2. bpfsample2.bpf.c: 此处编写 eBPF 程序 -3. bpfsmaple2.c: 此处编写用户态程序 +2. bpfsample.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple.c: 此处编写用户态程序 接下分别介绍这三个部分。 @@ -16,9 +16,9 @@ ```Makefile newdirs := $(shell find ./ -type d) -bpfsrcs := bpfsample2.bpf.c -csrcs := bpfsample2.c -so := libbpfsample2.so +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so include ../bpfso.mk ``` @@ -30,36 +30,32 @@ include ../bpfso.mk 开发者只需要关注上述三个变量的修改即可。 -### bpfsample2.bpf.c: eBPF 程序的编写 +### bpfsample.bpf.c: eBPF 程序的编写 ```c #include #include -#include "bpfsample2.h" +#include "bpfsample.h" -BPF_PERF_OUTPUT(perf, 1024); +BPF_ARRAY(count, u64, 200); SEC("kprobe/netstat_seq_show") int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) { - struct event e = {}; - - e.ns = ns(); - e.cpu = cpu(); - e.pid = pid(); - comm(e.comm); - - bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } return 0; } - ``` -1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 -2. `bpfsample2.h` 是开发者自定义的头文件 +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_ARRAY` 的helper函数，以及内核结构体的定义 +2. `bpfsample.h` 是开发者自定义的头文件 -### bpfsample2.c: 用户态程序的编写 +### bpfsample.c: 用户态程序的编写 unity 监控框架提供了三个函数，分别是： @@ -79,22 +75,45 @@ void deinit(void) } ``` -在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： +在 `init` 函数里，需要去 load, attach eBPF程序。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name);` 。因此，一般 `init` 函数具体形式如下： ```c int init(void *arg) { - return LOAD_SKEL_OBJECT(bpf_sample2, perf);; + return LOAD_SKEL_OBJECT(bpf_sample); +} +``` + +对于 `call` 函数，我们需要周期性去读取 `map` 数据。本样例，在 `call` 函数读取 `count` map里面的数据，去统计事件触发的频次。 + + +```c +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; } ``` -对于 `call` 函数，我们保持不变，即直接 `return 0`。 对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： ```c int deinit(void *arg) { - return DESTORY_SKEL_BOJECT(bpf_sample2); + return DESTORY_SKEL_BOJECT(bpf_sample); } ``` \ No newline at end of file diff --git a/source/tools/monitor/unity/beaver/guide/bpf_perf.md b/source/tools/monitor/unity/beaver/guide/bpf_perf.md new file mode 100644 index 00000000..2614c60c --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/bpf_perf.md @@ -0,0 +1,100 @@ + + +## 基于 eBPF 的事件监控开发手册 + + +我们在 `source/tools/monitor/unity/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 94f8322c..52d02c14 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 bpfsample all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile new file mode 100644 index 00000000..0b8e79a2 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample.bpf.c +csrcs := bpfsample.c +so := libbpfsample.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c new file mode 100644 index 00000000..12556f2a --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.bpf.c @@ -0,0 +1,23 @@ + + +#include +#include +#include "bpfsample.h" + + + +BPF_ARRAY(count, u64, 200); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + int default_key = 0; + u64 *value = bpf_map_lookup_elem(&count, &default_key); + if (value) { + __sync_fetch_and_add(value, 1); + } + return 0; +} + + + diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c new file mode 100644 index 00000000..18eff585 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.c @@ -0,0 +1,41 @@ + + +#include "bpfsample.h" +#include "bpfsample.skel.h" + +#include +#include +#include "../../../../unity/beeQ/beeQ.h" + +DEFINE_SEKL_OBJECT(bpfsample); + +int init(void *arg) +{ + printf("bpfsample plugin install.\n"); + return LOAD_SKEL_OBJECT(bpfsample); +} + +int call(int t, struct unity_lines *lines) +{ + int countfd = bpf_map__fd(bpfsample->maps.count); + int default_key = 0; + uint64_t count = 0; + uint64_t default_count = 0; + struct unity_line* line; + + bpf_map_lookup_elem(countfd, &default_key, &count); + bpf_map_update_elem(countfd, &default_key, &default_count, BPF_ANY); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "bpfsample"); + unity_set_value(line, 0, "value", count); + + return 0; +} + +void deinit(void) +{ + printf("bpfsample plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(bpfsample); +} diff --git a/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h new file mode 100644 index 00000000..9bd981fc --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/bpfsample/bpfsample.h @@ -0,0 +1,52 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif -- Gitee From 92fc70eac38ace9287b14de8f7114e909d3a9285 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 06:17:08 +0000 Subject: [PATCH 07/21] unity: Add ebpf nosched for unity Signed-off-by: Hailong Liu --- .../tools/monitor/unity/collector/plugin.yaml | 9 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../collector/plugin/unity_nosched/Makefile | 8 + .../plugin/unity_nosched/unity_nosched.bpf.c | 206 ++++++++++++++++++ .../plugin/unity_nosched/unity_nosched.c | 153 +++++++++++++ .../plugin/unity_nosched/unity_nosched.h | 92 ++++++++ 6 files changed, 468 insertions(+), 2 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 7a64bdcc..6d687b2f 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -32,7 +32,9 @@ plugins: - so: proc_loadavg description: "collect load avg" - + - + so: unity_nosched + description: "nosched:sys hold cpu and didn't scheduling" metrics: - title: sysak_proc_cpu_total @@ -140,4 +142,9 @@ metrics: head: value help: "Diagnose log for IO exception" type: "gauge" + - title: sched_moni_jitter + from: sched_moni_jitter + head: value + help: "nosched:sys hold cpu and didn't scheduling" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index 94f8322c..b29bc4e8 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile new file mode 100644 index 00000000..3f88651e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_nosched.bpf.c +csrcs := unity_nosched.c +so := libunity_nosched.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c new file mode 100644 index 00000000..85011166 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.bpf.c @@ -0,0 +1,206 @@ +#include +#include +#include "sched_jit.h" +#include "unity_nosched.h" + +BPF_PERF_OUTPUT(perf, 1024); + +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BITS_PER_LONG 64 +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +struct bpf_map_def SEC("maps") args_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(int), + .value_size = sizeof(struct args), + .max_entries = 1, +}; + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 1000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(max_entries, MAX_MONI_NR); + __type(key, u64); + __type(value, struct latinfo); +} info_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +static inline int test_bit(int nr, const volatile unsigned long *addr) +{ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +static inline int test_ti_thread_flag(struct thread_info *ti, int nr) +{ + int result; + unsigned long *addr; + unsigned long tmp = _(ti->flags); + + addr = &tmp; + result = 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); + return result; +} + +static inline int test_tsk_thread_flag_low(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)(BPF_CORE_READ(tsk, stack)); + return test_ti_thread_flag(tfp, flag); +} + +/* + * Note: This is based on + * 1) ->thread_info is always be the first element of task_struct if CONFIG_THREAD_INFO_IN_TASK=y + * 2) ->state now is the most nearly begin of task_struct except ->thread_info if it has. + * return ture if struct thread_info is in task_struct */ +static bool test_THREAD_INFO_IN_TASK(struct task_struct *p) +{ + volatile long *pstate; + size_t len; + + pstate = &(p->state); + + len = (u64)pstate - (u64)p; + return (len == sizeof(struct thread_info)); +} + +static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) +{ + struct thread_info *tfp; + + tfp = (struct thread_info *)tsk; + return test_ti_thread_flag(tfp, flag); +} + +static inline int test_tsk_need_resched(struct task_struct *tsk, int flag) +{ + if (test_THREAD_INFO_IN_TASK(tsk)) + return test_tsk_thread_flag(tsk, flag); + else + return test_tsk_thread_flag_low(tsk, flag); +} + +SEC("kprobe/account_process_tick") +int BPF_KPROBE(account_process_tick, struct task_struct *p, int user_tick) +{ + int args_key; + u64 cpuid; + u64 resched_latency, now; + struct latinfo lati, *latp; + struct args args, *argsp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argsp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argsp) + return 0; + + if (_(p->pid) == 0) + return 0; + + if(!test_tsk_need_resched(p, _(argsp->flag))) + return 0; + + now = bpf_ktime_get_ns(); + __builtin_memset(&cpuid, 0, sizeof(u64)); + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + if (!latp->last_seen_need_resched_ns) { + latp->last_seen_need_resched_ns = now; + latp->ticks_without_resched = 0; + latp->last_perf_event = now; + } else { + latp->ticks_without_resched++; + resched_latency = now - latp->last_perf_event; + if (resched_latency > _(argsp->thresh)) { + struct event event = {0}; + event.stamp = latp->last_seen_need_resched_ns; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + latp->last_perf_event = now; + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&lati, 0, sizeof(struct latinfo)); + lati.last_seen_need_resched_ns = now; + lati.last_perf_event = now; + bpf_map_update_elem(&info_map, &cpuid, &lati, BPF_ANY); + } + + return 0; +} + +/* +struct trace_event_raw_sched_switch { + struct trace_entry ent; + char prev_comm[16]; + pid_t prev_pid; + int prev_prio; + long int prev_state; + char next_comm[16]; + pid_t next_pid; + int next_prio; + char __data[0]; +}; + */ +SEC("tp/sched/sched_switch") +int handle_switch(struct trace_event_raw_sched_switch *ctx) +{ + int args_key; + u64 cpuid; + struct latinfo lati, *latp; + struct args *argp; + + __builtin_memset(&args_key, 0, sizeof(int)); + argp = bpf_map_lookup_elem(&args_map, &args_key); + if (!argp) + return 0; + + cpuid = bpf_get_smp_processor_id(); + latp = bpf_map_lookup_elem(&info_map, &cpuid); + if (latp) { + u64 now; + struct event event = {0}; + + now = bpf_ktime_get_ns(); + event.enter = latp->last_seen_need_resched_ns; + if (argp->thresh && event.enter && + (now - event.enter > argp->thresh)) { + event.stamp = now; + event.exit = now; + event.cpu = cpuid; + event.delay = now - latp->last_seen_need_resched_ns; + latp->last_perf_event = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + latp->last_seen_need_resched_ns = 0; + } + + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c new file mode 100644 index 00000000..2ab26875 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "unity_nosched.h" +#include "sched_jit.h" +#include "unity_nosched.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +#ifdef __x86_64__ +#define TIF_NEED_RESCHED 3 +#elif defined (__aarch64__) +#define TIF_NEED_RESCHED 1 +#endif + +unsigned int nr_cpus; +struct sched_jit_summary summary; + +static void update_summary(struct sched_jit_summary* summary, const struct event *e) +{ + summary->num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event *e = (struct event *)data; + + e->delay = e->delay/(1000*1000); + if (e->cpu > nr_cpus - 1) + return; + if (e->exit != 0) + update_summary(&summary, e); +} + + +DEFINE_SEKL_OBJECT(unity_nosched); + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err, argfd, args_key; + struct args args; + + bump_memlock_rlimit1(); + unity_nosched = unity_nosched_bpf__open(); + if (!unity_nosched) { + err = errno; + printf("failed to open BPF object\n"); + return -err; + } + + err = unity_nosched_bpf__load(unity_nosched); + if (err) { + fprintf(stderr, "Failed to load BPF skeleton\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -err; + } + + argfd = bpf_map__fd(unity_nosched->maps.args_map); + args_key = 0; + args.flag = TIF_NEED_RESCHED; + args.thresh = 50*1000*1000; /* 50ms */ + + err = bpf_map_update_elem(argfd, &args_key, &args, 0); + if (err) { + fprintf(stderr, "Failed to update flag map\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + nr_cpus = libbpf_num_possible_cpus(); + memset(&summary, 0, sizeof(summary)); + { + struct perf_thread_arguments *perf_args = + malloc(sizeof(struct perf_thread_arguments)); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_nosched); + return -ENOMEM; + } + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); + perf_args->mapfd = bpf_map__fd(unity_nosched->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + } + err = unity_nosched_bpf__attach(unity_nosched); + if (err) { + printf("failed to attach BPF programs: %s\n", strerror(err)); + DESTORY_SKEL_BOJECT(unity_nosched); + return err; + } + + printf("unity_nosched plugin install.\n"); + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "noschd"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + printf("unity_nosched plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_nosched); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h new file mode 100644 index 00000000..f7280f4b --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h @@ -0,0 +1,92 @@ + + +#ifndef BPF_SAMPLE_H +#define BPF_SAMPLE_H + +#define MAX_MONI_NR 1024 + +#define PERF_MAX_STACK_DEPTH 32 +struct args { + int flag; + unsigned long long thresh; +}; + +struct latinfo { + unsigned long long last_seen_need_resched_ns; + unsigned long long last_perf_event; + int ticks_without_resched; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + if (!perf_args) \ + { \ + __ret = -ENOMEM; \ + printf("failed to allocate memory: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + memset(perf_args, 0, sizeof(struct perf_thread_arguments)); \ + perf_args->mapfd = bpf_map__fd(skel_name->maps.perf); \ + perf_args->sample_cb = handle_event; \ + perf_args->lost_cb = handle_lost_events; \ + perf_args->ctx = arg; \ + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif + +#endif -- Gitee From 4e8b29372534a1b040058e013e72dfa03e5ab6e8 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 07:31:52 +0000 Subject: [PATCH 08/21] unity/load_avg: Taking use the dynamic proc path Signed-off-by: Hailong Liu --- .../collector/plugin/proc_loadavg/proc_loadavg.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c index ace88eaf..d06dd792 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c +++ b/source/tools/monitor/unity/collector/plugin/proc_loadavg/proc_loadavg.c @@ -4,6 +4,7 @@ #include "proc_loadavg.h" #define LOADAVG_PATH "/proc/loadavg" +char *real_proc_path; struct stats_load { unsigned long nr_running; @@ -15,6 +16,14 @@ struct stats_load { int init(void * arg) { + int i, lenth; + char *mntpath = get_unity_proc(); + + lenth = strlen(mntpath)+strlen(LOADAVG_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, LOADAVG_PATH); printf("proc_loadavg plugin install.\n"); return 0; } @@ -28,7 +37,7 @@ int full_line(struct unity_line *uline) fp = NULL; errno = 0; - if ((fp = fopen(LOADAVG_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_loadavg install FAIL fopen\n"); return ret; @@ -74,5 +83,7 @@ int call(int t, struct unity_lines* lines) { void deinit(void) { + if (real_proc_path) + free(real_proc_path); printf("proc_loadavg plugin uninstall\n"); } -- Gitee From cb2dcb50b04289f497a4f3cf61d4b62fd49578d3 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 22 Feb 2023 08:27:26 +0000 Subject: [PATCH 09/21] unity/schedstat: Taking use the dynamic proc path Signed-off-by: Hailong Liu --- .../collector/plugin/proc_schedstat/proc_schedstat.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index b51d5d38..f4691264 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -19,13 +19,22 @@ struct sched_stats { }; long nr_cpus; +char *real_proc_path; struct unity_line **lines1; struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; int init(void * arg) { int ret; + int i, lenth; + char *mntpath = get_unity_proc(); + lenth = strlen(mntpath)+strlen(SCHEDSTAT_PATH); + real_proc_path = calloc(lenth+2, 1); + if (!real_proc_path) + return -errno; + snprintf(real_proc_path, lenth+1, "%s%s", mntpath, SCHEDSTAT_PATH); + printf("path=%s\n", real_proc_path); errno = 0; lines1 = NULL; @@ -87,7 +96,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) fp = NULL; errno = 0; idx = 0; - if ((fp = fopen(SCHEDSTAT_PATH, "r")) == NULL) { + if ((fp = fopen(real_proc_path, "r")) == NULL) { ret = errno; printf("WARN: proc_schedstat install FAIL fopen\n"); return ret; -- Gitee From 48eeefa32059fe5be46dc45e3019cfae8758b3ea Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Wed, 22 Feb 2023 16:34:54 +0800 Subject: [PATCH 10/21] ioMonitor: Fix potential keyError in fieldDicts Signed-off-by: guangshui.li --- source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py index 0521a7ff..2a9dec0c 100755 --- a/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py +++ b/source/tools/monitor/ioMonitor/ioMon/ioMonitorClass.py @@ -339,8 +339,11 @@ class ioMonitorClass(object): if stat[2] in fieldDicts.keys(): self._removeDiskMonitor(stat[2]) continue - for idx, value in fieldDicts[stat[2]].items(): - value[1] = long(stat[int(idx) + 2]) + try: + for idx, value in fieldDicts[stat[2]].items(): + value[1] = long(stat[int(idx) + 2]) + except Exception: + continue for devname, field in fieldDicts.items(): io = self._calcIoIndex(devname, field, secs) -- Gitee From 7a805a572079352ed97601153a23e08f0fc33d4d Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 22 Feb 2023 18:03:16 +0800 Subject: [PATCH 11/21] add develop documents for unity. --- .../monitor/unity/beaver/guide/develop.md | 677 ++++++++++++++++++ .../unity/beaver/guide/image/frame.png | Bin 0 -> 38680 bytes .../unity/beaver/guide/image/queue.png | Bin 0 -> 14649 bytes source/tools/monitor/unity/beeQ/pack.sh | 2 + source/tools/monitor/unity/beeQ/run.sh | 6 +- .../tools/monitor/unity/collector/plugin.yaml | 2 +- .../collector/plugin/threads/sample_threads.c | 6 +- 7 files changed, 690 insertions(+), 3 deletions(-) create mode 100644 source/tools/monitor/unity/beaver/guide/develop.md create mode 100644 source/tools/monitor/unity/beaver/guide/image/frame.png create mode 100644 source/tools/monitor/unity/beaver/guide/image/queue.png diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md new file mode 100644 index 00000000..4e78d9e3 --- /dev/null +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -0,0 +1,677 @@ +# 1、unity 监控框架概述 + +unity 监控框架以插件化开发为主，支持coolbpf 应用，以及多种数据发布方式。具有配置灵活，资源占用率低等优点，适合在服务器监控等领域部署。 + +![frame](image/frame.png) + +# 2、开发构建流程 + +## 2.1 clone 代码 + +开发机可以访问gitee和registry.cn-hangzhou.aliyuncs.com，并且已经安装了docker 和 git。 + +``` +git clone -b unity https://gitee.com/anolis/sysak.git +``` + +## 2.2 拉起容器 + +``` +docker run -v /root/1ext/code/:/root/code -v /:/mnt/host:ro -v /var/run/docker.sock:/var/run/docker.sock --net=host --name unity --privileged -itd registry.cn-hangzhou.aliyuncs.com/sysom/sysom:v1.0 /bin/sh +``` + +docker 参数说明： + +* /root/1ext/code/:/root/code -> 将代码目录挂载到容器目录下，方便代码同步 +* /:/mnt/host:ro/:/mnt/host:ro ->将host根目录的以只读方式挂载进来 +* /var/run/docker.sock:/var/run/docker.sock -> 挂载host 侧的docker 接口，可以根据自己开发机的实际情况进行选择 +* --name unity docker 名，可以自定义命名 +* --privileged 特权容器模式，如果要在容器里面进行调试，该选项不能省略 + +启动编译 + +``` +./configure --enable-libbpf --enable-target-unity + make +``` + +编译后在 sysak/out/.sysak_components/tools/dist 目录下会生成目标包文件。 + +## 2.3 准备 plugin.yaml 配置文件 + +unity 监控启动脚本默认会从 /etc/sysak/plugin.yaml 读取配置。典型的配置表说明： + +``` +config: + freq: 20 # 采集间隔 + port: 8400 # 监听端口 + bind_addr: 0.0.0.0 # 监听ip + backlog: 32 # 服务监听对队列长度， + identity: # 实例id配置模式，当前支持以下五种模式 + # hostip: 获取主机IP + # curl: 通过网络请求获取，需要指定url 参数，适合ECS场景 + # file: 从文件读取，需要指定path 参数 + # specify: 指定id，需要指定name参数 + mode: specify + name: test_specify + proc_path: /mnt/host/ # proc 文件路径，在host侧，为 / 在容器侧，如配置 -v /:/mnt/host 则配置为 /mnt/host + +outline: # 外部数据入口，适合接入外部数据场景 + - /tmp/sysom # 外部unix socket 路径，可以指定多个 + +plugins: # 插件列表 对应 /collector/plugin 路径下编译出来的c库文件。 + - so: kmsg # 库名 + description: "collect dmesg info." # 描述符 + …… + +metrics: # export 导出的 metrics 列表 + - + title: sysak_proc_cpu_total # 显示的表名 + from: cpu_total # 数据源头，对应collector生成的数据表 + head: mode # 字段名，在prometheus 中以label 方式呈现 + help: "cpu usage info for total." # help 说明 + type: "gauge" # 数据类型 + …… +``` + +## 2.4 启动监控 + +进入 sysak/out/.sysak_components/tools/dist/app/beeQ 目录下， 执行run.sh 脚本，启动监控 +执行 curl 即可以查询到实时数据 + +``` +curl 127.0.0.1:8400/metrics +``` + +# 3、监控开发 + +## 3.1、监控指标采集 by lua + +本节将描述讲解如何基于lua 开发proc 数据采集。 + +### 3.1.1、纯pystring 处理方法 + +预备知识，lua + +* [pystring](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/pystring.md) 库，处理字符串 +* [面向对象设计](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/beaver/guide/oop.md) + +以提取 /proc/net/sockstat 数据为例，原始的信息如下： + +``` +#cat /proc/net/sockstat +sockets: used 83 +TCP: inuse 6 orphan 0 tw 0 alloc 33 mem 2 +UDP: inuse 6 mem 12 +UDPLITE: inuse 0 +RAW: inuse 0 +FRAG: inuse 0 memory 0 +``` + +#### 3.1.1.1、数据处理策略 +sockstat 接口导出的数据非常有规律，基本上是 + +``` +[大标题]: [小标题] [值] …… +[大标题]: [小标题] [值] …… +``` + +这种方法进行组合，可以针对以上方式进行处理。 + +#### 3.1.1.2、数据格式 + +监控使用 [protobuf](https://www.jianshu.com/p/a24c88c0526a) 来序列化和存取数据，标准数据.proto 文件描述如下： + +``` + message labels { + required string name = 1; + required string index = 2; + } + message values { + required string name = 1; + required double value = 2; + } + message logs { + required string name = 1; + required string log = 2; + } + message dataLine{ + required string line = 1; + repeated labels ls = 2; + repeated values vs = 3; + repeated logs log = 4; + } + message dataLines{ + repeated dataLine lines = 1; + } + } +``` + +想了解监控 对 protobuf的处理，可以参考 [这个通用库](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/common/protoData.lua) + +#### 3.1.1.3、 vproc 虚基础类 +vproc 是所有 proc 接口数据采集的基础类，提供了通用的数据封装函数。根据前面的proto 文件描述，存储数据实质就是一堆数据表行组成的，在[vproc](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/vproc.lua) 声明如下： + +``` +function CvProc:_packProto(head, labels, vs, log) + return {line = head, ls = labels, vs = vs, log = log} +end +``` + +添加数据行： + +``` +function CvProc:appendLine(line) + table.insert(self._lines["lines"], line) +end +``` + +将生成好的数据往外部table 中推送并清空本地数据： + +``` +function CvProc:push(lines) + for _, v in ipairs(self._lines["lines"]) do + table.insert(lines["lines"], v) + end + self._lines = nil + return lines +end +``` + +#### 3.1.1.4、整体代码实现 +了解了vproc 类后，就可以从vproc 实现一个 /proc/net/sockstat 数据采集接口。代码 实现和注释如下： + +``` +require("class") -- 面向对象 class 声明 +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") + +local CprocSockStat = class("procsockstat", CvProc) -- 从vproc 继承 + +function CprocSockStat:_init_(proto, pffi, pFile) -- 调用构造函数 + CvProc._init_(self, proto, pffi, pFile or "/proc/net/sockstat") +end + +function CprocSockStat:proc(elapsed, lines) -- 在主循环中会周期性调用proc 函数进行收集数据 + CvProc.proc(self) -- 新建本地表 + local vs = {} -- 用于暂存有效数据 + for line in io.lines(self.pFile) do -- 读取文件内容 + local cells = pystring:split(line, ":", 1) -- 按: 分割标题和内容 + if #cells > 1 then -- 防止 空行产生无效数据 + local head, body = cells[1], cells[2] + head = string.lower(head) -- 标题统一小写 + body = pystring:lstrip(body, " ") -- 去除开头的空格 + local bodies = pystring:split(body, " ") -- 按空格分割内容 + local len = #bodies / 2 + for i = 1, len do + local title = string.format("%s_%s", head, bodies[2 * i - 1]) -- 组合数值标题 + local v = { + name=title, + value=tonumber(bodies[2 * i]) + } + table.insert(vs, v) -- 添加到暂存表中 + end + end + end + self:appendLine(self:_packProto("sock_stat", nil, vs)) -- 保存到本地表中 + return self:push(lines) --推送到全局表，并发送出去 +end + +return CprocSockStat -- 这一行不能少 +``` + +#### 3.1.1.5、注册到主循环中 + +[loop.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/loop.lua) 是周期性采样所有数据的循环实现。首先将文件引入： + +``` +local CprocSockStat = require("collector.proc_sockstat") +``` + +然后添加到collector 表中 + +``` +CprocSockStat.new(self._proto, procffi), +``` + +此时数据已经保存在本地 + +#### 3.1.1.6、导出到export + +要将采集到的指标采集到export，只需要在 [plugin.yaml](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/plugin.yaml) 中添加以下行做配置即可： + +``` + - title: sysak_sock_stat + from: sock_stat # 代码中声明的表行 + head: value + help: "sock stat counters from /proc/net/sockstat" + type: "gauge" +``` + +#### 3.1.1.7、 数据呈现 +用浏览器打开本地8400端口，到指标链接中，就可以提取到以下新增数据 + +``` +# HELP sysak_sock_stat sock stat counters. +# TYPE sysak_sock_stat gauge +sysak_sock_stat{value="frag_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udplite_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="udp_mem",instance="12345abdc"} 8.0 +sysak_sock_stat{value="tcp_mem",instance="12345abdc"} 1.0 +sysak_sock_stat{value="tcp_alloc",instance="12345abdc"} 32.0 +sysak_sock_stat{value="frag_memory",instance="12345abdc"} 0.0 +sysak_sock_stat{value="sockets_used",instance="12345abdc"} 80.0 +sysak_sock_stat{value="raw_inuse",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_tw",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_orphan",instance="12345abdc"} 0.0 +sysak_sock_stat{value="tcp_inuse",instance="12345abdc"} 5.0 +``` + +### 3.1.2、FFI 处理方式 +关于lua ffi 说明，可以先参考[lua扩展ffi](https://luajit.org/ext_ffi.html)，本质是lua 可以通过ffi 接口直接调用C库参数，无需经过中间栈上传参等操作。 + +ffi的注意点： + +* ffi 数组下标是从0开始，和lua下标从1开始不一样； +* 可以直接引用ffi 中的数据结构，效率要比原生lua 高很多； +* ffi 是luajit 的功能，原生lua 并不支持； + +#### 3.1.2.1、 为什么要使用ffi？ +pystring 虽然可以高效处理字符串数据，但是相比c语言中的scanf 接口来说效率还是要低很多。因此按行读取proc 数据，可以采用 ffi 接口来显著提升数据处理效率 + +#### 3.1.2.2、 ffi 数据结构和api 说明 + +proc 数据以变参为主，下面的结构体主要用于scanf 获取变参， 用于上层数据处理 + +``` +#define VAR_INDEX_MAX 64 + +// 变参整数类型，用于收集纯整数类型的数据 +typedef struct var_long { + int no; // 收集到参数数量 + long long value[VAR_INDEX_MAX]; //参数列表 +}var_long_t; + +// 变参字符串类型 +typedef struct var_string { + int no; // 收集到参数数量 + char s[VAR_INDEX_MAX][32]; //参数列表 +}var_string_t; + +// 变参 k vs 类型 +typedef struct var_kvs { + int no; // 收集到参数数量 + char s[32]; // 标题 + long long value[VAR_INDEX_MAX]; // 参数列表 +}var_kvs_t; +``` + +导出的c api + +``` +int var_input_long(const char * line, struct var_long *p); +int var_input_string(const char * line, struct var_string *p); +int var_input_kvs(const char * line, struct var_kvs *p); +``` + +综合来说： + +* var\_long\_t 适合纯整数数字输出的场景 +* var\_string\_t 适合纯字符串输出的场景 +* var\_kvs\_t 适合单字符串 + 多整形数字 组合的场景，如 /proc/stat的内容输出 + +其它重复组合场景可以先按照 var\_string\_t 来收集，然后对指定位置的数字字符串通过tonumber 进行转换。 + +#### 3.1.2.3 实际应用例子 +以[kvProc.lua](https://gitee.com/chuyansz/sysak/blob/opensource_branch/source/tools/monitor/unity/collector/kvProc.lua) 为例，它实现了一个通用kv组合的proc接口数据的数据高效的处理方法。如经常使用到的 /proc/meminfo ，是典型的kv值例子 + +``` +#cat /proc/meminfo +MemTotal: 2008012 kB +MemFree: 104004 kB +MemAvailable: 1060412 kB +Buffers: 167316 kB +Cached: 877672 kB +SwapCached: 0 kB +Active: 1217032 kB +Inactive: 522236 kB +Active(anon): 694948 kB +Inactive(anon): 236 kB +Active(file): 522084 kB +Inactive(file): 522000 kB +…… +``` +对应处理代码说明，重点需要关注**readKV**函数实现。 + +``` +local system = require("common.system") +require("common.class") +local CvProc = require("collecotor.vproc") + +local CkvProc = class("kvProc", CvProc) + +function CkvProc:_init_(proto, pffi, mnt, pFile, tName) + CvProc._init_(self, proto, pffi, pFile) -- 从基础类继承 + self._protoTable = { + line = tName, -- 表名 如/proc/meminfo 可以取 meminfo 为表名 + ls = nil, + vs = {} + } +end + +function CkvProc:checkTitle(title) -- 去除label中的保留字符，防止数据保存失败 + local res = string.gsub(title, ":", "") --去除 :和) + res = string.gsub(res, "%)", "") + res = string.gsub(res, "%(", "_") --（替换为_ + return res +end + +function CkvProc:readKV(line) -- 处理单行数据 + local data = self._ffi.new("var_kvs_t") -- 新增一个 var_kvs_t 结构体 + assert(self._cffi.var_input_kvs(self._ffi.string(line), data) == 0) --调用c api 进行读取 + assert(data.no >= 1) --确保访问成功 + + local name = self._ffi.string(data.s) -- 标题处理 + name = self:checkTitle(name) + local value = tonumber(data.value[0]) + + local cell = {name=name, value=value} -- 生存一段数据 + table.insert(self._protoTable["vs"], cell) -- 将数据存入表中 +end + +function CkvProc:proc(elapsed, lines) --处理数据 + self._protoTable.vs = {} + CvProc.proc(self) + for line in io.lines(self.pFile) do --遍历行 + self:readKV(line) -- 处理数据 + end + self:appendLine(self._protoTable) -- 添加到大表中 + return self:push(lines) --往外推送 +end + +return CkvProc +``` + +## 3.2、C 插件开发 + +在collector/plugin/sample 目录下有一个示例工程，它的本质其实就是一个so文件的编译项目。首先要看下sample 同级目录下的公共头文件 plugin_head.h，该头文件提供了数据生成的API，降低开发者实现难度。 + +``` +/// \brief 申请数据行数量，在填入数据前统一申请，根据实际情况填入 + /// \param lines 数据结构体 + /// \param num 申请行号数量 + /// \return 成功返回 0 + inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); + /// \brief 获取对应行数据，用于填入数据 + /// \param lines 数据结构体 + /// \param i 对应行下标 + /// \return 返回对应的数据行 + inline struct unity_line * unity_get_line(struct unity_lines * lines, unsigned int i) __attribute__((always_inline)); + /// \brief 设置数据行 表名 + /// \param line 行指针 + /// \param table 表名 + /// \return 成功返回 0 + inline int unity_set_table(struct unity_line * line, const char * table) __attribute__((always_inline)); + /// \brief 设置数据行 索引信息 + /// \param line 行指针 + /// \param i 索引下标 + /// \param name 索引名 + /// \param index 索引内容 + /// \return 成功返回 0 + inline int unity_set_index(struct unity_line * line, unsigned int i, const char * name, const char * index) __attribute__((always_inline)); + /// \brief 设置数据行 指标信息 + /// \param line 行指针 + /// \param i 指标下标 + /// \param name 指标名 + /// \param value 指标内容 + /// \return 成功返回 0 + inline int unity_set_value(struct unity_line * line, unsigned int i, const char * name, double value) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \param line 行指针 + /// \param name 日志名 + /// \param value 日志内容 + /// \return 成功返回 0 + inline int unity_set_log(struct unity_line * line, const char * name, const char * log) __attribute__((always_inline)); + /// \brief 设置数据行 日志信息 + /// \return 返回mount 目录 + char* get_unity_proc(void); +``` + +**数据规格限制** + +1. unity\_set\_table 中 table 参数长度应该小于32（不含） +2. unity\_set\_index 中 name、index和unity\_set\_value 中 name 参数长度应该要小于16（不含） +3. unity\_set\_index 下标从0开始，并小于 4，即最多4个索引。而且下标数值应该连续，否则数据会从留白处截断 +4. unity\_set\_index 下标从0开始，并小于 32，即最多32个数值。而且下标数值应该连续，否则数据会从留白处截断； +5. unity\_set\_log 中的log 指针需要开发者进行释放； +6. get\_unity\_proc参考2.3节中 proc_path 中的内容； + +### 3.2.1、sample 用例代码 + +适合周期性数据采集的场景，通过周期性调用call 函数来收集数据 + +参考 sample.c + +``` + + /// \brief 插件构造函数，在加载so的时候，会调用一次init + /// \param arg 当前未使用，为NULL + /// \return 成功返回 0 + int init(void * arg) { + printf("sample plugin install.\n"); + return 0; + } + + /// \brief 插件调用函数，通过调用在函数来收集要采集的指标 + /// \param t，间隔周期，如15s的采样周期，则该值为15 + /// \param lines 数值指针，用于填充采集到的数据。 + /// \return 成功返回 0 + int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 2); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "sample1"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + line = unity_get_line(lines, 1); + unity_set_table(line, "sample_tbl2"); + unity_set_value(line, 0, "value1", 3.0 + value); + unity_set_value(line, 1, "value2", 4.0 + value); + unity_set_value(line, 2, "value3", 3.1 + value); + unity_set_value(line, 3, "value4", 4.1 + value); + + value += 0.1; + return 0; + } + + /// \brief 插件析构函数，调用完该函数时，必须要确保该插件已申请的资源已经全部释放完毕。 + /// \return 成功返回 0 + void deinit(void) { + printf("sample plugin uninstall\n"); + } +``` + +### 3.2.3、threads 代码 + +sample 适合常规数据采集，周期性遍历插件拉取指标的场景。但在实际实践中，还存在数据主动推送的场景。如下图紫线路径所示： + +![dataflow](image/queue.png) + +这种场景下，可以通过创建thread 方式进行进行数据推送，相关参考代码在 collector/plugin/thread 目录 + +``` +#include "sample_threads.h" +#include +#include + +static volatile pthread_t sample_thread_id = 0; //进程id，停止的时候使用 + +static int sample_thread_func(struct beeQ* q, void * arg); //线程回调函数声明，可以通过arg 向 线程回调函数传参 +int init(void * arg) { + struct beeQ* q = (struct beeQ *)arg; + sample_thread_id = beeQ_send_thread(q, NULL, sample_thread_func); // 创建线程 + printf("start sample_thread_id: %lu\n", sample_thread_id); + return 0; +} + +static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; + while (plugin_is_working()) { + static double value = 1.0; + struct unity_line* line; + struct unity_lines * lines = unity_new_lines(); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl3"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + unity_set_log(line, "log", "hello world."); + beeQ_send(q, lines); // 往队列里面推送数据 + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } + } + return 0; +} + +int call(int t, struct unity_lines* lines) { + static double value = 0.0; + struct unity_line* line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sample_tbl1"); + unity_set_index(line, 0, "mode", "threads"); + unity_set_value(line, 0, "value1", 1.0 + value); + unity_set_value(line, 1, "value2", 2.0 + value); + + value += 0.1; + return 0; +} + +void deinit(void) { + plugin_thread_stop(sample_thread_id); + printf("thread plugin uninstall\n"); +} + +``` + +**在线程回调函数中，必须要判断所有调用到的函数是否被信号打断，用于决定是否需要退出并释放相应资源。** + +如实例代码中需要获取sleep 函数的返回值，根据[sleep函数](https://man7.org/linux/man-pages/man3/sleep.3.html)的返回值说明： + +``` +Zero if the requested time has elapsed, or the number of seconds + left to sleep, if the call was interrupted by a signal handler. +``` + +需要判断是否存在sleep 函数被打断的场景。 + +## 3.3、coolbpf 插件开发 + +关于coolbpf，可以参考[这里](https://gitee.com/anolis/coolbpf) + +`/collector/plugin/bpfsample2` 路径提供了一个基于 eBPF 的监控开发样例。其主要包含三个部分： + +1. Makefile: 用于编译该工具； +2. bpfsample2.bpf.c: 此处编写 eBPF 程序 +3. bpfsmaple2.c: 此处编写用户态程序 + +接下分别介绍这三个部分。 + +### 3.3.1、Makfile + +```Makefile +newdirs := $(shell find ./ -type d) + +bpfsrcs := bpfsample2.bpf.c +csrcs := bpfsample2.c +so := libbpfsample2.so + +include ../bpfso.mk +``` + +1. `bpfsrcs`: 用来指定需要编译的 eBPF 程序源文件 +2. `csrcs`: 用来指定需要编译的用户态程序源文件 +3. `so`: 用来指定生成目标动态库名称 + +开发者只需要关注上述三个变量的修改即可。 + + +### 3.3.2、bpfsample2.bpf.c: eBPF 程序的编写 + +```c +#include +#include +#include "bpfsample2.h" + +BPF_PERF_OUTPUT(perf, 1024); + +SEC("kprobe/netstat_seq_show") +int BPF_KPROBE(netstat_seq_show, struct sock *sk, struct msghdr *msg, size_t size) +{ + struct event e = {}; + + e.ns = ns(); + e.cpu = cpu(); + e.pid = pid(); + comm(e.comm); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &e, sizeof(struct event)); + return 0; +} + +``` + +1. `vmlinux.h` 和 `coolbpf.h` 是coolbpf框架提供的两个头文件，里面包含了类似 `BPF_PERF_OUTPUT` 的helper函数，以及内核结构体的定义 +2. `bpfsample2.h` 是开发者自定义的头文件 + + +### 3.3.3、bpfsample2.c: 用户态程序的编写 + +unity 监控框架提供了三个函数，分别是： + +```c +int init(void *arg) +{ + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + return 0; +} + +void deinit(void) +{ +} +``` + +在 `init` 函数里，需要去 load, attach eBPF程序，如有需要可能还会创建用于接收perf事件的线程。为了开发方便，coolbpf提供了简单的宏定义去完成这一系列的操作，即 `LOAD_SKEL_OBJECT(skel_name, perf);` 。因此，一般 `init` 函数具体形式如下： + +```c +int init(void *arg) +{ + return LOAD_SKEL_OBJECT(bpf_sample2, perf);; +} +``` + +对于 `call` 函数，我们保持不变，即直接 `return 0`。 + +对于 `deinit` 函数，同 `init` 函数里提供的 `LOAD_SKEL_OBJECT` 宏定义一样，我们也提供了类似的销毁宏定义，即：`DESTORY_SKEL_BOJECT`。 因此，一般 `deinit` 函数具体形式如下： + +```c +int deinit(void *arg) +{ + return DESTORY_SKEL_BOJECT(bpf_sample2); +} +``` + + + diff --git a/source/tools/monitor/unity/beaver/guide/image/frame.png b/source/tools/monitor/unity/beaver/guide/image/frame.png new file mode 100644 index 0000000000000000000000000000000000000000..3892a4cb3f167ffb3c1a32be0ad7e1003e66f262 GIT binary patch literal 38680 zcmZ^~19W9gvoIXnoY;0Uv2EM7ZA?6|ZCexD*2J0Ew*8-Z-sipdu5YdXthLwfU8kzL zySlo&x~lr8f}A)!3^oi95D>higoqLl5C{_x5U?>62mm5nIMok$0Xr*+3jx(k>fHF1^_&;zUAT?l||0(MMQ~oc0m?aR{ zzc^-q_uroc-~mwo`v%Pe{$GxHApgY%VafyjZyDJ5FPYl4mnz@|?I5A)3P)M14w!PLb;So zo$YO0|FT!LvvlEO=J^Nt{{{bVZU3Q?cd|4EwD&I=AJe}i{}=ZE;tSi`Iyjj+JO9OJ z{TKiL!v0@;1q*u@dw{H+ER7}YTuhw+#Q)az-v<8wWBkjDhvBcd{*U~92okjeNN;3MS+*hwEGnXq0Yw?;0(I?g=%_0U8yAdZ#(-&VzS-${H`Tyk ze>jEnv%kN;!~LpZ^Z^u@goH#`c!0otUq28?Fn~}12?-wrMM0vwyPJ&-qTgD%QTz5G z+R52@uOsU6^3vx>@*4`6hll`D?Dtq%ROUc|Vx%~TrluyrSp3lB3i9E>SxJ!7>}VjB z>;Mrmn;~dm{abH|tWf3kxBlOf`@~R4U{D}J!U2<5P}rwhwiu|{q{kyc*AxJFD{`IV zubpY)Cm*5S<&9MNEUVCHld130T4etafdK~!K!b(LQ3FGd5&j4A16wHIxCk30=eso0 ze?VkNgk0rdq@+QBe*A@4K>-&eT0wct(f-plLZ~DG00l~19NfR1Eh7T3tRUL97XJ0b zpi`HekliuJF87AgWp;M~wn#uq!-(&FPv>e^bZ9zRv< zM6bUs%N7Rk3hxsKshangsW-wL8N&+XY&=TJl-+km~N8Y4f_qA2< z^*aef_MLD4>850F!k(Cg4t1Aoh&)+d4O8l;Tm@{Qjv2w-s*Sqz#NOBY-SYFzw3ip? z_wOm393mm1PO78NpLizfh`4!oil%DIH4Aiba_}7 zh03MBv|EUG+&_7kg>(6IP0S%1h~RPXo!l8Le<;GgWbw2)jwj`}w-c^)_;4~W$KYQS zm7kQ=bek=dRYk#7Yyc7Da&B~RmsR{382N=UOYX6KJf~wZ7y3hmfzf4RLf&i# zFI53WCv^LJc(}{d^sT&%_GF&oa{#j9*xH9j%h7VdLLms^tGzvErOwc>R!;@Fo8f$& zb9KElwqTbBCsTVQM)9R320weL+gC~nr6|y23kB8ED**`{jB)+t6I=OQMlcKPYIqHf2DN|9gH_37g4u z)?{NNzjzdo<7c*ET$L61Oh#h~`1JHM_8WhA ze7SDtm6eY)GzBZ0T3W7Hok%g}9_h6)kkNvPb?|A_?Q&U4{RTp(?~Y1+-|8wRg!Wbgle)q{DpF(Hk~=l||8G*GyfCulvL zn;I9FR_FCFNv}(+*G=0*zp!~51gW^-Kb~|$r`_G@J~KX6PFmdjqwmsu1}!Pcz!UEG z(&y*HSx7LLD0*_|`g$Kcb|-_;=kw{}dR5g>sl2bV z$(oBlL_f})RTfTUGjU5~GCKlc(7haDf_%xm zipwfjEBDBSE#synq`W>I_886#JkqY+J+5lY>P}66aLa#KS4368Wx!rTa~!)d?y8FD z$>r#=bU{~A+&i4~p&83PaXRiejyg8Ehz^Zelg}J~FI*ngo4(8bn0~|4b$8jE_I&eO zdwM+4#v{qb!ZD{Tr&5GpoUdSaIvMTz1*yC~juNnnF&A6I!}Ym*;|_e)J7SmsYFn9K zrlQFEb>Hdyjv)0!zvX?prQDHwIUGBLiX*49>P|*PEhr{eqPnaG>&{?fu@F~Dq-SSL z4B}I31MA_Sr6fho&MmB|8kwpkRA_i~ASERw$kr}ekZ3`9Y6lYWLrEeauPa;kSF^1l z-3`=8B)7Z4I$RDf4h*!a$-%*!d=6Y?!NMs*BDvzrE1g!El~oCPIx88KHv)m&3ETq& z{C;v0*W(3j2pBV#rlu?}uLXJ0Z>bq%5&m8K^4+&$MUmd1sLd)B{j9@ zP~V`dCOti+xw$!JW&$ChO&-tN$5j@n*!LrLdm(%c35h)`GrJfqf~`(LuID~bH#>Tl z%|aUW2pCPVYOM_}r~QGm<%NrtEq@A_pMilrTZ;AtPQRRQ^g?ZI`KLdp64~#nj#y{y z87-FbR^02&*>Zn=A@v--XRKVWpvTZ8ol&ht*AF}`eyA@;>CD%tbE`(@DmzDKvIErf zttHt0mF}Lkk(H35QM%HuI!@vv0UIBwyKY%%Vl!oyH}z4^iC{Wi)y`zy;&?sZU2c7q z&%Hww5w=iG%oMtJ(fUjJvl61ZVyo^mz!G$>r&xHz;IhUsa+r`tS_K{t9#1$st?pZy z0H`vkLe7TQl5v}OWDK26*s5Rjm)FPa!_+*xeP?}qWwn#tozt(}XWZMH&7_R?pV_I( z8gVn@dvQlcN5wwT1MYm>h+E*ITPp1+=_=L4gZ=SjftmzM+fcB%xLgqQzPn`5 z5S5lm97#w>RajbmcGAuHv<@Lw8lNq1aXJ#SQb{UlVWV1qhew$V?fs#wvd;@OVDU6p z{DY6rlan)|qJpwO#ZpNpDgn0Y}*rmpm#mXY9M>R!+QZ!n+7iN$iG3 z#}Iy`>4AyiS($a^m2bVeWU}!dz7zj`+aJ_siP?|5^{VxuQcQQp^FW{D1o%X@-on2A z;Xn-2lh5Lg)XNxzXgyaObkwb zJ8o=N3Jev52p_qR!VYSt_%hLx&>S+~IkBXYjQaPU3U*+a8(=)!Gd7p0=A#ZMC zLPJqyv^l(M{j zlHGzZfuJJ}**pU?)ecqHXAR+q z5<-Vj|2jJrwz5zLIE{;nhL>j7R3sPLmog-{eFMU+Fp9M3P5yGGu*|w^$*5gPOYBv6 z6mrEpt%i1#0SI!Fp=Be!E)joY6S6e4v(sjrSkU$OXA>kx4~gi0LPCn^FHRPMd(r=> z^^0iFAcOz=F(ziJvcl{y#loK@bZSC79;h-z_5Zpz3d(J2lk7n;_nRX1A&4>B9AN)43KJDDOD6Tl9ErQ zf$@}3IfD%@SQz0&R!C${A3*im5Hpb@x9Gs~YwSjN&nSX+Mn?Gc)xq9F55lAuR1|=X z#-}x;rvpIb`i9uXS%*)Ez? zJz?^zz9(t-oi~={bh)4a-{&TUE+vw_vktrz>UTjxK}!%F0OHt(qX3CT$3JSx1xyc8vlm83Oa}^swgD${c>OtmNcUsMc-KqF zN|{*dlSDUeDXroKd(cu=6MUs zM?AF)<)XU$5Ez{0LMyRt6N!K*lKRc*-K;Aik_sDfGGL=*7;ai@_lZ^$S{z5*QgU z#gG-rcaaA(mHV9*!(4KC8;yMpp37mnV7ff)fM1|h7n-(OGj{V@>YC}f)i436f)gp$ z3_21K(L>mX&0xRjwncrO#0`ih^tz^pMvVF6+MC&$iGP?KmdHL&kBtin@GSRyYI#00 z&eh?^ROv?goGw z@I_IYZ?$jUdY>skRFZ$W<5U|1<^^p-g)5rS^{1zZmR_kc6JU~1CYCgp0EBMpVyu*X3f5! ztY;qFQDD4tCqq$k$0+#wXVe=F+AXx!2h$&pKE~oQa<4RpM;ikqW z+ej>l{k%q8MW!>TuLto|f}J2b*?D%Et*^IF(~lIvfM(vc=|5c7%kR3}M_20yJ&~1d zfBYibxN9Cre|EUvklYqlR~y=1;q>ggyFa-)JJaN9V`Od|+^3I~lsMD+ePY1Ayt+Q@ zQS`W)f=6@KB(HAwU-_>YyL7QG_ZwGHiRfC zNjvW+&!J{P^Aq*3HJG5|VZCOi9#mDBwO%MIhD0zRl6~&K?h@5^e}W;f>}J$-6Oe=#Qn{>P z%5T@Qy+%r-OURXpesrsTyqdpZdMBmm>$*tYtbI9XXs~vrzr_&Pv$#+;=b053pM1l@ z%E$<;KE98~l&r&fYAF=27YpfXOluuT(JKKThJ_R&`3^?O;9%w~ig7nnZ684$@yrPQ zoJmh5Ao-cLeQ?`dXyJj0UIC@;YdNi34#g{?5+5D~j$C+2DA2|^%k}elL)Gq>z%SzK z?nIek{_+UCGZq6``Y;OhxMLf&?Qq~Iz?Ss5`mY1Qd+8{C|hxG<-zmRg`F0wDZ# zqg~U0;Br%4nNva*B>IJvo^ylI;(8rmuXntC`<5gtDh2hI6OdpY3KSA|If?<3B<8w* zUU6PsgRr+5xg;vU=k(XBldVMcSEUpY|0{E z0+2Q)jYSn3=24wSH%!FC-5s5CeAU=lNR>`kU|@-j9I1|CM+|kP1V#hgs^|0f$(b)K zQh^09vRfpu;1Hl?FO=Vs2f+FY8XWI`9+0P^!`Q6V@bM5$iJRhcAU$aW1G87cgoM>! z`P~@5udEJ~40rAKn3|JN*J#2fnqJV+{D#kuo1PhKL&tMwpue=UvC;7gZOXAjMAFvHk*A;kHL@rT8e6wbfWT6SdhW17|bLM`^28M%{Fh} zSpnG)eId$Q70vaZ4XfwTz(2G>^^ICm#2K`I^GLK<@;2cFK|FF@6_Zn}osE>E5_2=7 zpHi%eMI)`HPG0>c`G$>MVQ;W~eoi~pb2rvIKROytNs4W-P2*${5CHYHF%nq^a(+$;rW?jx1NEMjtD&;}(g~+5I42 zI61n@vr>Ie3{__6;<+lqTaSLIvWUUi_e06qIp)58V4_|}OOdK^Ztf+eJa z3C~_p>oAohDLfYR>CQo4kBFD#PNt`O9o+T#Q(A4QLOd2QwlybnmJ#Wi7a4+1jNjMS zH^w*5c7-v`ebJNua7V$Tp@*@u4@2sfDZf&YlOGO_B~C?01L~W0R~6w$oKsRtYb~^w z6taMlkR01*akE3p9*p)0@_9VS53*iHu3z`v-s0`Z12T86@_zJ@flYMZ8}7A@^6v~8 zv2=k=wCCj1(7ttd3_T#~3mcM4!H6yFCnG9J?bbKPk73R*Kck)t8Er*J4`*YaYnLUAk9#mXIQiVi)7ANf1(h^^ z6$VhyYzTN1v&bSkY+z=0abZD}9>M?tDafth{?xwTqt{buiqAtI()Aka2EO3p%m|%6 zrsH_Ipk3ZSN1R|8NR;#a`aetAG9IjPk$81CG+E50VBAx^-j8N%*g6~w#X1QkEuLe5 zZaVh3i*|dF%-aWMv<0&w~W}A#aQ#d@%g>46kDq+;xpf*M6XEh@^&X*>dI(g z3xV;5e+BdQlVn7yh384Rhqk~#@%+$j+?dI#WcczC-QX*VFVE#OH$fZPihN^``1T5| z9pmHU6Brm+RbB0g!!D@kda|ymPz9}zkJRg>BFQsq3%4n zhFf>0x_8qV!U^=Szt7>QHIr3cOL+CCBAGE0W3|~ODe1Z}_*It}?8D`4i7psCgT*{1 zfnypa@|-d<&%ne#0SZA1KHTM1_jwl*v4d^FlhaCM?@wo9hE(6;FLee{6d9#={Os|kaDX1;v)okNFjGNGi(9sN zj=cRLxsUO1t#RtIEGTi?uoS3;)qB`CxwwCzQE!`80BG+gCK||y7Y_BO9!VF>gR-As z-$W{depql64ttD{92y$RyO&;K%7V+3l68@@_ zInRc4%L^FfL3zS&om$pbUJqBXaUC3A&_7JE$VfF|Kx96G)4WyQyL1Kn4$ex;jxDYt zF;!Jn-yY7?VdgU?U=R@z;h-$64Hcb<<5DySuWIP|z&u}Gxb?uR{XdO-CNK<}*hJz2 z#A+(-1_m6EdRz`J)W`tj&`=)tj`QQUOS!IoSuotOyKx%f?gs*u!^zf zB0n!A{OQJA?D7g?BZcTIe*c6Lm+grFc$mc*ivSfHPKgg%qjtwa2|JRzB@=%v7{ zv|$$Pi2lF!&vC2hh~z3YE-^8%-<7bis(OKNB*%33hG!p9iC@T%_8lJi(dh)6Dmwka zwhqJ@^N~r!C{b$pyGgVqLjgt&Lu!r4Up*USBO)U*EZx$u=@{vWnX;M3J|pH>?6*Hw zQ01(Tm>=hlqu$;>JB!>FP*YRG<8}k1n)iE6%8|S9@BiLFc}($8P#O3cXVga`a8BB1 zOlw5fn+p6XN=%*<9&smL8UsbZw>@W?l}TUYs09ww)eteh-sXHD#XT>guIZWd6Pgc6 zVBtPdfQe{wLRq*kUlSU z&z7pGwD4yTY8Cib+3ZY8@`;qrJsbAB7Zf|hQx@g4x!{zZ@Q(9Wusm{g~B1Yf! z=N4X1TotQNc(%%Fe*a1PTmnjRwd(RQA^~4iod7C@?A+YKzWP@BD;2_-e7&Pa;Gf4^ zbpHosDJv@xll|?!pcgW>9c4VQQ76)DHJBNoYza3|7MdZg)2LPCw64hDUt#f2mjlTl`L24f;p?Dt)O zJ(=8}p$oJZZY@J7U_x$p zjUcD?N}A;tU|R}Ws^X;G?=CJj&EY=6!O0}uce9puPN3a&S8uB)c6QhC@QsM&cR8MI zvR;mg<}6)WIu|{mh}GjA7>N}H+X*NdWR=fVl`4YX8%accrw&OI_}R9U_*{~>RLzY| z@bz$S*%D6@@&USkv1Va@wD|c+1Oi_3$UH){^)iUr$-F0ic-cDJetfVvypBuJd&*`r zs;csG50>BQvxy0cVt323pS|D?qwe46^B5DecQOgyEe z)DZCb_&qa%{S=SY15LHxnk$i6T`3xqFg@_wXrJP9KEwxK`d!%arzJNfDT&!?0b5p< zLPnu*d@PZVaA^_mPu)%&fREo-4wKitR#HVJcOWeZNo2*@9_nn@8?LG2?d=^2^taVsjI~spH~iZ{)igab+B#p`G84 zDwCUZnZ{mIvWA8RVtyaW%B1~eD3GiNQZXAMzGG5wa81<2VJ9c%XN8~V=j(FWX6IY} zi*pSxRhmJJrR+9oZ_e6U%D?S2;+I+Oq(XxD zuHHssnhN)ypKtEYmLUZZonbg3ApXqcT&O82NyxmHRP*kiEnh)HW2&g2IXmBBfBy!@ z4fkEWmT!7|oY&W@zR(zR9UF%|gF{i}SwORGwEemVgea>`8r@K?4+}D7Lt+{h zsV*?cKOKPmp4m@^^26H&biHMaxIW&5n*p zP%5L6iU4uR+&iQj1k@N*gG3ZmSHCRtumbHyQ_l9{;?vV}b89OKP_VGd zYw~}LRuU6)A_oZpok@a&+hD^3j+JR?q?7$j?o_Pyn^3a-JO$WxTRFcFW4h7M4~`p) zih@fHXSOU{dTFSr#vaaMvpF0osHjx6v>FTtUR}@=DA9kkjBU}%=P1P|kBu?(ru%IY z@a-QSeXVt^-4o=GNmZ++vScnP8;jO=AM67K)3P1KlIA3r7x%t2k)~iGeE0e|jE;?s zt*)l4PPCGiPvyPdLPE-;rhzSdw6j?+ctgxfzl3w6_IzI?v%hLFG=#@tvsrI-yxQ!t zmWDh_%1Y~=n3#x<-=PHItD;CAtnu?zo$O({Lq%0~h$JJRRu&dc)*P4i8iI+vSeud0 z*hfNfA|%;+&EbvoC|6CjTsSyjtd5UM4{4>vvSB+?rRh^j{3!oCW9BG+^o>ZTy2E7W ziSoai{qd=)VwLYZ_{;vlC`AI!Y6GyFbC{!gDfU z4zoI-q9v*-9Mw%&i2_-3=bb#(*4FCk%j5;_q?`;K-0arl>#kwYr+}0DqSU-(IKh zSimY*gsfLlI-WMb_Zu4JDDl0v|KHYpjAXv=puow*v znylq981S1d=Vg+lFxpm(euoznp}2@f1ke{-lwCDhQv_YypJ?0IVhTKE@w|jZL$ zX=%-q2sMLm$hak9)Z-i#6ogdJ?d!is6~M@<0wvPZ)4iY0PZrAO0r_G3CPPOuPlB?U zG7meQyh%mHm@PDSkYK#-6eM&TTfY^19QUXVBr2fLfDh~a-WwC8V5%!B7K(n-SI0@A zs8B=pxYzNfC<&E8`8#syQt`P)c!*^#g}YPGP;ss{f%qT%(R8+UGC`nOQU|gL{GU$f|8Hmm_Jvf?SwI6`3x0u_nUq$@**y=g5 zVH+tf%L^EIDr-#hm1o_n`hvH&tI}!y2z%k8#X3GaTU%R;)O;E$;Rbo?K*cEgHu5AD->aJ1j=70|U+cW*Brg=XNn} z37NB%M)Zxz2rQj~60$gh&ggaH#+J9-tV2YL0TTF_)MuOssgD?teTkeT#X;QG)~4E) zA3Qi1w_Ga@M^IDx#3Cn^-1gH?^qgV?v3WBVQdYyQn|a8BUt0r34jLkVf&KF)!YBqz=TOqpQtaEMc zFE)vgnfi!0cTi+{zPO=qFB=OkJUyh8k%)5AbISy3YE;=Xt}C3eO~*m0aWDWtj$IGG1uKVucM& za1NmY36xHk7b`I_NDmahn5gAaZLDyBiYg~j)Xut%g@pzQal9Bb$h&C3iMx79w#&gd z2~8zuM1-#c{)>Ijh48tMR`|k#3$E&BYdvfXg}b98EPQl9L_q#e*NedAnFg9yVZJgU)vL7XymY5152aswauuBM&w5mfz-D6 z$Ezkns>f_fY$p8|9ZjTjxWL%J^of>=_F7Bsz|AQP%;R>O%gJR{md-v&0ob=3K414g zjj~=|(^68t*;#%A`7J8wx3Wk`vS9B7yf8j5D1E}KHqEK2%Qd+^)NzIV?hfzt zx-fVUfeG`>wamYVPLQr3h8TWM^>umuB_4x zXCQsfI$U!iaO<}@5zIje-tP#%SE^@4`-Mvc7t|XslX%SnOv&gd7q`p%f(3LVb_YB+ zxn-fD5LoxA%obGhj~x-g!I8Dr8Sp1C+XLDtH`>zmKNe=@9 zVR~fI?T52vj&&%Qp9u_fbW_ut_~!FJ%UVTf0|iu(%T}%~WBCy%X=r%78ezrn@{@^# z+qpTb3o?YuuGsAh>^9de%$M;wQ7IlTW({AzK98EL7FT+EpXlk?)>>vnKNrel7}auk zz1O$|&Ob#=XMa4zJMNMM6NKVZ&_?(eHo z8A$TN0J#weh`1}&qLI73A$)v4PEW@fgBaVw5T|1;79?QVAQ2eYSy*`7r|tWLd%C&^ zcHhHia%UKHIN(t2^HD)}06oCX17%0LhcGqKEqn+(Kf-L&{{QTbn z(y&_@85s_?NZ>?2;lP0ThzY6a5_9!=#C0RKvYXkC_Ph4?zndlAYaV)33#aOIH2=_) z%Jx|3^xV^KVe5PDh4yoBX!W=uN=b^r|A0?&h|D`Ny~o5P5d`ymz|hdq&8M=U^&xQY zg7(nIDaeb#fSCmHKzg|0;Z<=*+#jnL4(dY@#6<=TKci1Mr&&jc#1w?+w2t>c-Gcyg z%68R>oOBlMK~%jZK&wjlzLva$tnBK}3;!9IckZ8<8h zwT!fzlNn7@n|@j>y*X!dD<$oV^z!ad-`S8w6+{*3qh%hQ9_akl*V}3LfnJZt2VhAH2C2#Btba{q#sXHlt3Qiy6Pybw+MK#vF2@i$ zbc~SS@bMwT?D6niTPJ_y@SWpI>nDpGB^GD6|4-ZLtO^}fe4E$0{g+`=aV}r z6GOn+_HVdMU;!y0J1KcUbG%QrT`q&u8AUcdYOTZ==; zN`25?ruuD^a}8@wn{!w%Dx%)@cBm$A`?>7c)!ue zP#MN-OE}Rk_WA=;N6gL*{`4Z||o4$dWs|0|AgCXfUr=9lw^CyJj!GpZ_RH^|_U;eh%2PqdOy z0W8feL1JbD8V$TZ>)$MX5n$kwExcUUxg6fjwdQPeGYaHA1mI{SHJ!BRgyw&BBS0St zYFw@#Un2`Ozg*Lckibt8kxx>RN>C6PP59i6%jY>XY{ZoBmZIvv%47i`xIz#_MI!MT z49^L?ga(n+(4HyO5~M7|V#xw{xuT|&0_{@&xt#EqrkjKii5oghjWBK_fovdz0M`)U z=YT@)@c&ZnCJ_Z(i};6eKpryEs22*z`VOsnuoF__J=qJ%B+cU@al7sqaf0x_aa-aT zAQvS+{GO-D&`>c+Y3aG&s19;331nYPKY9i03&pAh5#EtM-{*gc$G(h@f55?uPiD4| zkOZMhA~FqCS0C;2^s?EKFIFC{)Naj{mCeZKbPARP26nZ*m=NRwkqeu(n@am;`Atj` zop1&Aw%Gr)-{g^#2&bo{EF;k+-yns8f(L%rBqL3Ax5CpWjqm>~A~>SBnYu#Yg3eb& z{uzT(?`TXXB3O*TD}-S7$ka7zhrP?^cCYsk z@R&`u$CG$RHVObZh;Xx8-6 zoh?nA`A<)K35QzDWGtEvLIwRU%-m>qt)ewD>(0(z_y3|3oc?&jw6P`2pGzz5EM;Mt z_wk(}BOAKaE2dXYfYi1q^xydem((5`5k+kNmFaxPWSWzy)h*%g^;yoVp>19!=V0RS zlC(&R4)CQ~Sh!52Ks1^tm)y^cjd8o4(kDB=Kig)xFwa5&9rQ`6c#ev=PpVbmrGxij>XqmY`vW54;J z=ME?tiqZ!jAGcJhbB6^#J0B!!(}Rr$O@W8!*+=G_>_TQ{&(!=u%}GjVd>qw^G{exd z?u^35hE_5A@w~C~4T>%E&RtG7C50RV1CVT8tK7n&Ju_Z`uI7t`#>4XtNQv$c6Q|(u zK=OHegark?IGPzK`$Z~ner_YI+6Rm!vF#yncu^{^U#?waUtxKuh^7Np;4iJ$&>4$N zn>msCAR!@fxz$r)<^Q9zMUIjt1jw_F20Zl`iNZ&E zbYcd_mi^DzxTNOJQ9WE>oC2X63N3op@(3EH+3em({?+D?0Ru8Z*3b_>m6K}inM`&9 zV*^GIX{mwGYCg|7$R6brWs8#m27nn~AF{9*S)V5apZw4+IG;DNgDC@SrI^uaLc^*7 zSl>Zku~^u~#>eN;Wo7=zXEc_0I#2BH&!`->Ij@z!0_G1x@WC4NzkJvWxm;^c~`h zxF~Jn*T0Wabj|JbH0oK=GY$|NAcBXi#m2qa%NMfb6xF$~<8SF&98`mOguyFyzwf-1 zOy2Zyo1a^}J+3ZXa;{c=E&qw|&;;X|IjL-W=PEc@Z{thCF6dfHL6Eg-l7_jbVZ!9dT0!-qx1Sd|;7nB&a3O~g9 zjG-*fDn4N%XF`TmDPVl$oAKGY}=@dy*=j!S8E!`urd9f;aa_m z+w)afc-*l9W2iu{hekMf&V){{19tnI2Y?;^dXKf-bQVA%Un7@k_Hv?38j+BQy4mGj zZ#Fgh_B2zN5(Rf;G2ZSF51E;&I6XNXT)}bTJXg`QXc9Jdqra!C)*)b+SF=UKf9AO~ zv&o0EY`z{)16sq^X2m(h+;E&vT^e5b`zd#GW{Y!ca@J;PTXW2E4D~a;Zq1wBqPxfs z8Vd1Sho6m@8$?8uOO%_}g^SKkIB^z7QEhdsh;j9BU{0$rjZ9C6Q7YKJ+FhVj0E35y zhD;`#ER`CEE;Op+tmO6jH@$)C=Q+Wi+31H#Y>+xm&+0R9mJp zeN9YE6dN7)7An=BfWzo@IsBb;mXBr>OXcmI9MT$0enLi&%V_XAn;iu~ZUCjWJY5C{ z$HHP>Vlnojiv~f?WN_3k9!%h0UYbWmukm@i8L!ldHaBPKCwV{8g7$yCZ5jLeqOSj0 z;b74HGcrH#I~1{-Tb#@9&*O33l%sb;0A(@wNsP$GAPfh8!?d%P6XLBCnZr~Dx$+%9 zmG_)`g@MMa4FBt@Uh=o_fz95!xg~+Sy!#A#&H-AojmzGBbY!Tr+{dTwq;_(B9IWw(ZHus=$y$B?KlR3u`}>XPG(7nHe#%pqhL+UCyQp#+ zwdA=O6>@qF$wE25T(mWyVndy*z`-{dC=Fu2)h8)QM zeg0iYgQiXSN4q7)=lL32H7RM~^?~LSbZOas=LCzIj&5L)%f-#&81@IR@{|J6lO&zpo^$`+abux5G-x3e}&-nC@OX*ei5gVxrg23+2^BW;c$Kwb-Tbzm%i58~~jtlzz;?G+0K1u6; zM4=P}aq3XH*8Xsw#Tj&>fUqL8Ru*4UQK57+owc>mDkF2B&+~ZM)#>}@DJt65?aO_+ z!el=jMVFAUu6Lq>kc}uCis%c70thk29GI78XB-AV5qtR-C=6(P;F@Xd)aDJnKAh*UN$_L2W@Pi83}@kFmTl?`q;ySMHa? zU_@sIa>RWBscYeWn-+Fn8D6(h$;;&Moc@>=NyHW<%TXmZO7|4Pn7>~F<+g`nI70*9 zeI%)8w=0iE4AAY137U zjgLRHe#eTUrd8I|xX-gFvu9wCIEC*}#erF%|32lJ}J1 zRiFrBE^!O!ag~H*rR`?93KHPv!S=Rj|BqMDsr{ZWdZDjqoaM0Td=v%IYHjv>_7$gO zwNp02ZQZNQ6y?&C1_=c!K33_dR2!K|?M{=n!92YNjqUPVX$(}^$=GsBicXW8Kr^=B z)N_DYk;W+zc`Lm%WC~0QxJ)>d##A^oY?sgN9s?>#nP4`>(GIg|=R%cyIA`cmUhl|V zR9Gm?#47GvHv=8x>_87GOsn6wo*&-h^U`P_C_nxXQxgMrADyCRygyHD7)?z%TC2$d zn*rAYcH)9383))#euO>=x)^=0B8YgNPnGAzybnIZb3xRBi_1%blrqGB0e>+pF3l=P zpc8`D^>&J$=6Ak|ib|4^4oXp(W*U~en}3oc`6a6yLyvYV%0P9aOp%XBqaLIA%Zzyu z%Z4)$JV<%6o<`c{(*I6ya8cyGz9t&&oCM62)G|H(KLb}91ijIt%S+RQ!f5I@HuS93Wuo; zlwMx%u$Ic%1||k}b`tYQk&+eX=WgTq?F}2UvzyCE8hRo5>>ngw!>4vusv{CI`Mnd2 zjorXvK6*XRu8kZm6o_`+-8$8Us#rz3k|y%3bi&>H9KFmX%w6AwdU5#dF%mOuV`;10 zM}07T#y;;q>%n?^^563vI9EIsY;t;X_F(d={ImQnWGH^!y>2~}E>nE+{-&5U_sPA< z5KZBwMBLy2!@Q=6t+C6x@-*&4cR~IfT!WrdD(9! z1zC_@+SouPPnI)!eD^DachQ|GD&t#Vm zd+bNcZEt6LZxfH+rDZT=ySnPBwOM5b6(n}3EGy$OpJv0u11+l8S*X!vc)r~MZGC;p z4ZI07(ry}uF8+om1EHm{LO5ju)e_j++1VM2fX8LEP(~siqwsFJ<3dW8o&Nt(_ElkV zENi;~CSiyW++BkQcXxMpcXyZI1ozZZlODIci z(&|RD_b}5YeI@WXPorUc*-*Ka9zIETRU1gRol+3?DF!PC0o%XI$p3X_)dr~d8&Yi zrw>ZBy!_oto2J;d;f3|}!IT^&c=!D-c!vvHr+gqa$k+os10Q znLn3Z)WCPD6{3;1Lk8K7cbRVp=GD7UT8xkhvR75Ulu~<(+i>jw7czSGx*WPa7c_S8 zB*Rp;+n(u1=Z(lnSu2I|^zZ3to50e~EM_LV8~<~|oCckQ-?3u@F%wKHM!v5zaSFt> zyD&AadRJUiTT@y+GHG@il3*fW`kt15d_{b@58rr0R8v#1i&>?Gt)=C$xvA+~+XUch zu(31eldj_mgt`wr29vGWCzBrH3ig0`I{ZiRf>aaDxa7XQ{Uk9c!L?Dsy5`121*#0| zlsfvG@&LY7_E!HAHlG=c$J5RE#Gwf6(I>OVvMRX(kX_Z2u4~ED+z5lc_I^c$DG?uM z$#k%|Nf4OH41$qW`+J=-EVZpxLUL#GmzTPy!FnGbJ3~TwUm{EuR3(msBtkJY^j23T zwLy0d-C6@$)JLvAF+W7cevzA>ZQ)Vn#2rPGIPNFM>kev}Fo8 z#jW#^rT^0ipmrw*?d6$-$^3z%`#)WJfS}*JctB5*9oc_T4MT*T=k~uhd+iBa6t?|{ zr~kuyI$c4b8NIGzaDxAd)r*Os>E?jq-W#V4m$OqEo%)jnMA7j!;SNu(KKjivu!<~cmB5eIjnY~u*DJ0W3VZa^d}>f(H|+$8vT18AqYRLG5hG4XRF ze#1$@8sbRCFd=RXU!M=MRQ62LHk|#fSzb0#PNS_J&R~ob53Pgn^-D7nGICC8gU!+` zzzNZ2aF!K^N*EdFH1nDL1)}%~fQG1u-=>GHX}>*lnKWs`>K`(3{7ClaP7X?~6_wQt zqW62fi?L94e0&_J_U43bPw8~q&r}lS5P=i<6XuuCw2;+%`MdOYEth9a=qAj#-?>n6 zVc~3*?ON024x#<}dKVyy>+kQsP>M(R`x%3fH|nybS25k^IT0+D)7G=sX0sbQ)Y=6S zjs&O@MA^mJke>cR&ACc{e}B`l%!;By6JujzK=mINxAk}M48FfPb`_nCr=%ZB*zDl| zItn-9Mg)|m`|~x1k2lA<9X{T`FU?;zU2q-Q)qDy8BQ%#F0zxA2vB1^Bn<_{LbLNBCZoyx($crH!FcPMn&=V)we4BrpubLw z3Q@ma`{G#yg{aS|Rw^xZz2`X^Vo4fJz~g;>(#V8`i<$XhVv`1+XLWRRlPxSR0=OIy zRDrpxv=SS%38Rvz^2_wPX=G7|h(`&1OyQucbni#%C(Ah-2^*V-De|SBo+=YlM=jw& zVE_OO#fJtA2IDDFUrm7OO8c%eOc+I$i02G;TDS2FrCACSKBQ*T3h41J-AyYTcXH;- zJ^FixztG{d4=6kSVqr^)*XLut|NG&&u$eZ(;U}iyM;KlsGLHZ6$5lZ2)x2S&?6ClFbgmCBAW`?#L#m>;1fibiU`zV1YglV} zd#^}tLnU*K?07$-@h2Ocizm=g##DuV7^HhsSKE$?hW3SWb{AdH{?DdPKQWDrce#V^ z-Zja^NjXFCGU~=10@HWqL!RM3Oos{F!vgVX+2zU9thi04HOtG(Lm@fU)o0df^WR`U z13IgtXs5MVYxRt zMMoBCf_GCRoNfEgjv?LHHSA|Ajb#qI%3bZtu0uTUp@9#Pk&d5GTyrZz3$B+H2dcHUmly6%O;!ZKB?+O+Tp6tb|w!K-Oxz z)*d9_U~fjV5Cn8+g>~5<2`BR;ars#6SJq~ zVy*EZi}hGKyP}+&TwUB*+ci2AR5GnD9;ZurY%JnhDsr$h2jRx#Cg@aV%#QF*Z(4!yi;%*wo~#!LfA9Qot<-4GD?C{le(er%!|pQa_LcL3>S{ ze>MruCW1D5Vpq$NH`2$4s>yFsVf*yrmVywR&T(&FOp;I%Os3 z)9ZX}Vmwo-OhqyHBfV-R|DZR<_k~{(9__Q?7gJ}x0XM4ke z@bVAsa)^YH%GgkgrQeh zApgV>u8=gJ84@QT6x3YSdVaWug~!mdw0zt;-(2y$!C2|<4O2_HG>RF`*7h=}5zpZ-iv&d<(r zYiVid54DuPFkYsiXw%cvC(~*h?L>%ah=kXz2Y&YU9;Z=}CmbxljK!fLrBXj)<=>}d zbWS3Y)h<$>Z3+ypbpSMpK;qWR)BR)F-SG?06ORw+_0 zfS;F_x1o6osnWHc$X4DXE>Wlh%SlQgfrk{-?j~kut4#_@d=ACMFCf{;GFb zpd?s6s6tQRbgD>J_(n0PK35naS?7oqismiq+agEq0{md5iROeFtPh%nzm+MWkDi{g zRjbxPP#tXZQZyR%ez=O1q-%^Dl98BPs;a8;H?D5CCmDd&1d=0|#$r9>3IYSmv!Kpe zdMw|kTOAxH1sV9I0dh($JmkCn10@_U$V`OEQ#jf4 z*UchPFNm+V9Gks1aECNO0zk$zR$Ct#F$@@AShz59*-X8`I`_++j&PNDS6~Iw%d;k* zCu+!CNw!U39l;6RAq`P)*fy@16W7<*t<(x*Vqw`|S<=aI@tA$TD?U78*!I%@M@^o$ zUP&hQGY9h=ENT$woWH0)ApGz>dJ1s8|2l>6LmqAQGVr^$a-w8ZOvqdmyLxqNyxgUq z>jS>xJ>KZU*G0E+L)`ApHh>{Sl##3BF+5RYi>5X&h$1{2Na+9Ya#PiJD88_8M5Rjf z=TB|$M@+I2V?T|DbUpVHL71_S_I4wCFGxLiV6U1(&^D`oTO0F6h}Z`; zL%3e&K5{~OM8p9=>TI!4sl7|9UHw8r;GZ41uG(h%`jVcG;}$qHG-P0|sH>~XH<~!k zLQ6|aL9q|`T^wp_*_)b9dwR%afshBH)={MG2vLbX45%nhx8rO3&@gds*OQE}u>Fx# z@86YerQC%--j`=avf&>De0~V2|I@^CmtFN)PuRzlS)S&GzoCE%pIq`RrQd%ZO z4G)j$f#|clVk}mL2m(0@kVU~7ZtmAkuDC>HY%aIr+RhyX${@oGBAHPNjaUO}i9vDH z$81H)p#)g`t1YXf3f4m*+az=Ah$XlWWCUL!uN#8O%skJv^{`Fyg znJRI4c{!)+iGm_kX8@GZcsLdyK{D)j`$Z3B~18hyVLbx}iq>0Cim1^Ug&jt6P z_WR)rYD<9+v%DGVrL|0q^0E+n?KWDSKv)nd?GYT#EUU$qn3Y4B2JBaSHv8_t>gwu< z)zy?Pm>w<`PBu2R-yHkiLcMcaM$I4gsHs^#eE5(XV;$3;Vj}@P$ZEr}X6DWVQ3oZkppeNnPQ34%iF6Ds6p}Df{Af%PnbSyzBS?HSiu{ zb=?1BJ~t^yqI&~YLhju64(dRSSAH!k@VB$+SgZ9~JMv9W+&#y++a1ac+LHLl2|;f!0r6~8`v}*B zG_TJS8#gy@5$pyhBQY^mnqyZyHqv+&Ycw6dwxOY65W9J>*_vH>rxdPMr!cxJQK}ko zwP!`1a@IbPH95Brj_a$>Q4HbO&o8-QKnj2wFU!oaR%t$E%ZpVKh#VtJNZ;Q|-=-b3 zivurxlMI0v-IAM@E%A-c6nX77STCtK5-I%xCS#>?^4k*KTLe2GtWmN^*n^vbm33{2 z6X!!^W#!b&^z_}Neh$5TT@4}-4^f#DV^u(dZ&aV}FuFZifuguHSfu&8h&5?L4MP~+ z{{$8Z$b;mw=IdOUOV%|AKk-m-aF`v82Z92Fkto~3)#=`RC3=8T3!C;x*T*#-o%MUP06**zR^4SJ)G!8oeS(oP!eg_PJa@QL@ zDFe})xVW(OR_2sH_7DK!=0APmO99(GolH6)ntY$B*6#s?iNs?RX*6dU-{Ii)A8*>i z!zp{caOK6iacn1(_K5xJ2v_@hLBc}Sml4oK5$(M~0s(Jc5$^g<;jxuAH+gZ{kr+(A zr+)v=CqUlG)$GgrFc|N~$=T9iO>;ko*d0b7cZet^Et>_X!?#7FC&P)NhKGNv{j_QG z-B<9oKGs-codkq9wtr7!qBklEW@hGECf$t~Y)L^+Z#bUW*~k_K)X<1QLWVL(ASeQa z5O9>I!==qe3o{m~zoGBWK|IzJtqy%Etw2!Prg5ya-AfY*bLL(b2}{)yf7 zWW@KiErn@NOFWF1@Q{)5@?<5#=lS8o4PI(m+8m;tI4c}tL@<~%eh5XTNx^n(vBNkj*q{7dLtnHxjjI2 zFtJ+YvU&Ev3ioq8rPkIyg?%f}&Pk{BHH%BVMixKK{W5B$S=c!9_7X<=X|BqtVl@J- z371Q3#h}4%s6R4NTZ<_pU2Az+ATQU9Ar2_G0Snf%_`Q3_9*G1zXA2KJJPa52H+-(p zl0_~&nID~#6SEcLxw#}82P**MNJ?6OwNGN6KtS*Z3=$Tp{qVkjaVa^$QELJ~0#@4u z<=wQj?2JgHPWBj%-5h^9oXa{rdZMARK6861rYoc6&2O)ez{m6c{)16v{Y}N zT~P3ad;qrrl}bcjo+3831fM6rR&CNwAIQ|y5*98Xk^M(yWk=`-=eUL9u#L8|Q?qcV zDp8v-`0>XRznvXzn)Pz=M51VP@ra6uh^Wnq$MXh^v?9X>GDdM_`A5sJ${8`q;ia{# zz5F<}SezUHxok0fU`tr6Zf}1{pqOe0fDxRhgP}xL-F6b7Xq_nwsrVVBpb&?N*)%wa z39RHjJU*o{(f_Qs%FWADq{NRC2e6lfaKD;NybDR@;BEs0IU^GzB@1CnMg|_dFk{bK zhY7EK!ck-0Sm1#tDeJ1$6ekW%)7MIB9`J?^TwNDWB#18ogzxDw91pPCgxmU(ogE&U zk{MV#+U%$QX;!duf&Vq3Sc-6PZ~)XQXI+^*dqws2OeF8$zehnqVPawefc%m*kAhj! z)W2XCJ2-{al8l(t`K~|mEiEmI`o-<3D>Ss0#}%8#R)yYbz4_q5v;@+HGZWL<@lTV; zuy&sKk1penjcg|HFD5nbg}1ZOCg)hJ^pMs+b^X`LEwYa0JF zU!WM@QGP2+C{vMf(GXoa=h`RHVA#X^La5~g4=A3&&@=pR4)klk^_)m1H)L>gjxS)A zvj8@n@i1h;(^c->UDLI8Z*)dRBg(;H!zv_`B_m@aJI}*=$I=Ox{(iAQ5m8Qcb$RyT z+mp7t^{$9E&qP$VobK-2yfoBQ1G~xLvEe+kMJ#MUSu9cPzS;kKaBTFL$5aO3)?d+R z^TEGa`TayeD9)aAc8;5|Dd@FB`y@JSNLavgEL#Em#AAc=G4CFTC%B;R-pj9qz&+4-`tL?e+Gy z`1?Z?uu4mM?0Hru z71~ikQxK*U-2CaIEsrOQ=h=FG0?Np~z96l}k+r$|^Zry@GF?`?>*wB%NKD3RO;-M^ zs52B>kXK%=$JJif0FQ)TQfl-OmD4YTE=NCpqL+myWpX@J zIdwkVD|VgNi#RL4qN#%`&e{tMBx`QgifF0M%0B*%#gdRjNg8D=;o)K8(ok7Q7LN*~ z;JAqb>yNp}k(Ki(^c)iKG!ZCUCZ+Pdw&qWd!4x||(IFsJVRBMBVm7kzfr^gmgJw8Y ze0xPv`S;J>l7Gh#xT{rsW~RkK{8zgQPyi^<00C06L6-4vs5DXJn+b6uZZ09Ve-!_by;VS`Pa*Q_}v|IK?-Gb~KqXa`v;4gxT02<=UuG}MWV?r%M)hL0$6&6yY zrlx(6RaXa6)E9E;NK(wz)d5snrYm~H`|&mzih1yV6v%r71lEjt&s&?whif2fkUi;R zPa{84X;2vC(5Gd5p4>O1& zyYEX|Ehw<*6$Jiy9}FEJz*p{sl0Lvz1Bt%Y3W0!tBE+)~pR~P?QsU#|<(o`bRDNM9WD#`o@wGa|fq}1$RfyIw zA0U+EV8B`}F4hOrB{1(k0wK^aH1L$7H z7WYdN<5DCI!1RHDK$Ri%bW(Z&j^P0LJ0Qv_D_@_TvALaPmz5oE4aO@fDh`v92Z8)8 z2nlpJ#6deJrOa6Q4E%gbhlI215w@DYHW_9wT2j!FZ@B`Yhds*1k0v$J#e z;_JB!g)e;GcRs95+q5-F*A^1!grorGhJzA-ErFI$Mxf5t~Ih=~{ahvv+t1a*FxeM^K(~{=RdD@n$d{HQWJstigSf7uk z0^o7BhY}TwWR%Mq6b-Yep_C!w15xCg1wt@Bn%92NF76c z)Is(RNN)(hvanXuzh=T!Ndus@8(sa+pM4SNHYUs=f}^(Ne-H1bWBYW{(%j5ZINJEg z&CS&xjlU)Xx)=-YklM?JfrSU8UDO;Li-3ej7q@p3-~dHM2Sgtm-@W{%d#TfE_rh!| zhJty=^?Wr2I8x+PFKiGKWkHKXewg@rqmLKEbmtMOmC9u*gu4LB&hvEE(*U32L_{FT zhw6m1czHjR_|=2{Q%@PDNSda)#!!Rexr4bm#nF-43y^gQEaEqMLO$bh&9ANDH#CxK zVy4{bJuTIljsq*GoU;B%Y}oDTB3XPM-GcNWSv0IfGF50W+JJOQN(wj)PbSdw<##M` zo9AuPp?Hbi3_&&#$9{BA2puqk@A}uJesG$~h-%}tk_*5AotM2M?iGvDcSR3s@?B2| zFzJqqkB8Z2vR!Qj?m(_6?>VX#k^HRROmUNtkW_<6kl>UOYHH}y1fUTc>7Y>E6GfOx z<-Q1g(0~5{4!+%1aXSOtvTRw^_Rf%_9IZMZaLXL6w4f^nQb-Zz_=ADlpuPWsy*?3g2yDP*4|B$Q)l|6oMU;vXPnDmm%V$J45Hz@%UlneG1B_`5FPaX&8jp~23oe)- zv=HVKwsLJWdAvClB`07a$p;9Z@TNewYe3cg3|x|koUFbtURPoXj1M?Fz1B+&6>2rm zYS{&pl$4M(p=rEK#4SgKk&4`4Q-4BS{3bR)CHZ~6NG79u%dPS7jQm@e7Cl8Q7IY5T zuXw+MJ(5NRpT*kkW8sMU?brLug98gG5jbqzcG_?EU(UU3$xq-%k`fbxs)GJFCzinI z=;?Z2&j5EDXT8L|=>tImUS9kMI9~A2%`HGZFo)>ilD}PoM76ZJuL|E@9JT^8qqIKC4Au`a2h=|7{;tQdErATgP1C5E6 zg)mpQ6K}hmK8|#M$EYY1&;7Mz3<&s?=E?> z=n%}h4G2*Fz}D4;2MQpPZ~=nuyiV#A{I{@AyW0cmUnO53PdiQlR23LdKR>@$E+Kb@ zY-IPdb^Z}MC=M%CI5&OUK>xRSScyfYl)CYxgpx|_k<)xc5H$bL->+^ohZq@wyPQYE zc6I1L?r#E$UXYx|ds)%Qh6X;>{T#swnAq*b-GCAGCRg@ydovn)*0f=U57IUp*sF9Lg% z?{fdZjl~F~zzDoMlUl1cR|LfHnvIDpxaV$iHyFkd7_Gm5i;aN%_3TF+p`HV46d_kq z)47wgS(GB6P`@7ta_KVN`E2|9`>rSXM#d`?)YMaRb4;vrU%wVJGFjC(>|kPwtAh5= z?muzisoc|~1nlY7A@GMNtIVRH&MePF{rd3!Zg8-BdAYE*)&m*Yn~Ew8l}bhb%~4U7 z&-=^dr&7|>xDAsYJ`HL+;tegxYsxa1l9GY~jt&+)732E0*o%k{6x!*F6jLP=Km%WS zZ8=BV*WYhu$-1z>%)vGd`}Ulg+6~&@$?q2f_1=g5eQ_YNwvYp&Hp%A~{7y$k25#2p zIlZEsYx7SoGZR*`%cov2NuQD?eLJk9qodGg%0l5Gmn7JwpNSO>O87+C@Sv^Y_;s1J zmfK(7zi;{ZJ?u}Vksw8#_qN(Yc+{SszM=}?_Fwk)YAv-t2e)|_P2`bRn{YQZ9Q4K^ zJJ9H6eE1v*R^HuFTnh7ZK#IweS!{NuD%AD7)piO?s5}0wO0_rL?w#gwUHZ6}=LYkw z;~hNCo!x+<&EK=?T|hIMT}&3b9MDv3SD%=IND`O%%*HvN{vVWE#!v{+M1W` z7Mn~-`3(b`|4bq)n^{&i9eE{7fX%x1^`{VmkPQ6iOY#0*u?#72x{ICNx&lmbrH~&O z0V)as=ngd0_b;D5CAuUU7`nH(UWP{$*n=Qa;EQh6y^#?&*Ij944y916@!`#!v=CW# zL2P{n@e%FaqY=S*+yi{f!OfRn?uk+C%Ar7eL~o!kX2jW`MBo>eBvq-EmANq5YHQuX z!dmN0-9xebWV$yq)e%2LeeNN`N!O+)5SRJzj&cPzYx+r3o^U3alysEJ38__7E_G+8 zHIv)@ZtO1pWhXZMhj1Sf6fhc%(tIZz;%J{Uk7#Kp6ScEHs!6{gWW3BQ^@A341v$>~ z+tTM5V?t4UeIqclk*r-4_Os7S-2_ZS=4L1|TuW@4O0lLk3Jc}GepellXIST=)NWf? z*D3aQ5W&R^P(cUzvjpc?1Bu{0B(+IQH$~CeQ!*Du~YV!7(x(3M3^sKyyF~Zd%a!@p#DuxU&CWA6G6) z?{Dr$enK8RB`uSQFD4br#s$4=qyKUi)ahcBPwha$#~~hN$FAjsTvd`2)^lA{itHak z4uQ=3ZEr;95oeM>70ikQu3-+}mz<6cC1vG-T{`XBEjG4D>|ZN0&e!oE5o5%^ryT0H z4F2*{<_EXu>rmn@Ue0j}s0!}Yt@YudzvY!z027CY#8vd9z)!L-woeNC#LE*l?QO{W z6loJ%V*yeYNxDRxL*L_su96QY=y7Od%(E=Dqm^_O(HUr-vpYxT@l*1Gd6WiJ^V+on zFE3swcogAW#!d$wch?ui4lcI+i{B&K(Pa-ZTd^FNcXx5fxIHaz3OXf!y8on)Hs6z&7OsfSSq-k znTdrt$^WbC%yP+aQ7vC3i5CBkqI^+I+>Z>7oi=)ro=Bk$qN;tJwAc9&6F`E~AbZrm zWPUSu#E>^!Lp5OJ8e9GbTB0Hz1>%#)6{Mruc0slM7WPM?+)JjH6&fD35ms6URR>3b z#(;X!mB9YYs;cFrg_TY%qGuSRigB;hcYS{&pA2=kVu;oE)CIM=Kbw839y%q>+I=U+ zf#HqeX@)i|2`uw;wZGC0)E$~hJk*>N^RBGdZ#aL&>xfU*PS$$QG_)7E~mg=Wb#zRRjwy zhtmcmrQNXqT{O4PwcB{I$Ld(?C$unEq&!>EI=Lar;h3iUTo6Ux8g`^$VX&;X<)r3M z=bwzKBSucBF4`^5*84jJf}S7yRsrtFUa~(?b;WbkK$4i>2b6GidVjP~@8@@&nrbW> zy&8;wnF>LGHC``Hno|@ziZ>Pv#^Uci*%AyOquHI3 z{wj}Lrr_`#>C_(}g`^-giBf53xk&%x{r{8AzMR%bH zC`&0T$+U4NEPD^!%ZN>ca9%GOn`-X+8Y7MfTX^Pw`fNN%+gb38Q5b;BS z5|qfp`ZR28kaA+^kjw=jGpPN2+mV=cZT0mg6S+K!g?E`e87x-w4I*B(0Rx^VO>Ou- zZj-%2rdjC#E*O0G3v7adl&6YL?{AhoZ?mjql;fmDQ}G zlSk54)ea6D<-SVNAYj+`Yb+Oe(H=ipiX%H`}gI9Ez zh$X7|)ks@AP_XyMOP5{x1{J6Mj~-ZiZ2R>cfoRIP($2(ROUJ*c za;g^2z`(#hG~MO=x_SdPz^zHnn;VPgEb?XWUQ6{hh-LFsy$OK(kRnJONi!jCu*IYO z!1skRzI|yT_bar-L)8AEWs~(18+j3s$5}!6W?*MjbpuDEz4i6(Txb63iYX$L&Hm8^ zC@QCtXygTh8*$jarjGA!POw-n+T{O6dOE5*3;N(mXng)h@2WnDmYd!%IIE!{u0gwD zCz@I?FI!X5s~qD5EvE$H5#?+lL^&<%?C>+VA$U*^`AW$xk_%&~dMLAirDcJ;XYZp+ za&dgI-dcx)LGrxd(RqaEkSE0+j6N#Hd%br~50J2$ixnS93xQbe`R;3&>jkF>Q);9g zl7wgNcZzr%I<&e?dUJw_ooK=VSV(paE=ktTg|fY>+=n=i9z8*8No!nI+?3N&q|)z?n3f0TEb0=kli6ifJ4?Tj^rZEj@11M+ z&*0!l1r|_58dA*sQGEzf-ZQG0=F`Z2v*;?Rne&Whm7(+uZK!a$XD)Hpnc7;7G-X+( zw$~wAmgJ!2sNPJK{VEz9Ldk!FLQ;n9{}i{A$vtJ~WJ&TaZWaMU7AZc;XC^jL^u6Q< zS&tsyx{mM63`jKU4!NpTr_$2A+O2nV^Ct`p z60r*t6QSI_eSLw6g!A)z5kj|kg&7=}#PrNR3os>a5Kv=s(nyI(Ajn?>FyBgUU<6_0 zECuRuIptJSFd>Df3l*SCI%=u`IU2E0d2`{^roD-AnU3 z-}{G4*D>@oUbzHulGzy)-PZ$Q`7Yd;By3|OXc3~i%}e#w-T=KE z4(=Z8rPlV)eX(7X9X>Kz@OdaV*OO5H?b}U*JpM${N9NUtdn9m$5v8ODbou!p}3d*9sWgYx<2z;$ZwGlUgSFa5$t!q7QSpH2{TXIwtg zxescFUg%NB+-XUHqqk#gvUM>9q@e6`MlTiyrtQu~hEvu;Ve%_yh z>MDvKZ?AjYDgVKkvlhrWVLE%PIS^z2)0yTAg`4wn+04%wS{7PW#EvAlK$&YURgdK509r+FVDBss;Y|RgoIJ6t7}5S+gy+3pKokdux84VxdrccvtGBQ z0+2YIzS#ec85>!}z#`(lUFHV0$;kQw5zP87d?yo7qpu9Nv!xooFsgcfZa+33L3pO;2Jk zs1S?5qBf42TA;1|={5^pyXzwB7BG2iB_TCj4Qx_0=LkZK5%5D^0$ zGv##Ec6cF2x=a)7GacKh*p?Od&YBK4q(2Qi)z;lq((RYs@5_NvUZ3N;&%RH7Ga3e; zq9=|L>o$lckt~k28_+MNql<{r-pLa8vKsYtf3mJIY?ZJ+F1R9>;VW@Q(&A61Gp*3| zHNUe{1&xuCph%@u!*n@waQsMP)rE<$St;KzegDZ>>(@zKRFoZrU{^AkL4VoO-~3`L zV>l_O%i@>!5FjyeaL%AYyYJkz$t908Cu%gao zL!&#JQu0z5hmzi4Dj#c3{u2HpmFeo9ypOK_=sG-cpi7VbIHX z$w*&6yDIR|&0%>*ZK7IjYj+e~!!FF;x7`P}_&OQiUa8OCeDQ_&YX_mW;EyOpzi-q? z!oPv_pvl-(4+_8I4$l0~!~Kao?rc?+ayCY{GZjC?h#)x`DRMYZ)3Ig1=o6$|Hb&^; z_40I*^6dW_eF%NiwyqcAehE(~@Yypb1~6>rg6o&rLw>hQClc3MJEAY~w8up6S-U8z z#GEv9zENR0&)FKF+sKg-O=XwpoRz|YnenJ$`Dv38iqcBLk3A@|u)>N>N}1`Wa7Y5O zW29gHnC!X==d&5P3bx?O&Hyr5uty@znpaxLRaEEnljGxS_&R~@8~H~FJ#VB80s)?I zYbznIEWgCvH#zvv#1v-O-%3X)`|aPFf#x7wI!EqJ#S{lF1-R1DzDf4um-q#UT;O#( ztO616e_A92(c&*y$_Oq&Th@)ZP{F2LcpPClWzMZ!UrVzf~AV3Q%Ifx4&`n z__v1AgNub^F9wp zY5-^(_!u8hTysVP_%e3Uwu6H=HytD6w_%#>X5_!`CM5XbJc^T%k(rL2QBo$fq!GVFI>aVYdRpU*~7hK$j85!=-V={x0@M{&Cttl9z zfgl||K32ewLGFoJ%=B;P-$1b+kR;$?VWDAQIBoVv)|pnl-e^`96hKznXD4e1C*w?k zO}mC`;%^$!(ItwDLz6Qim5FJjsW@B478jRSR$M#Z*m864n`)xg1&Rbx?g&NuqIo@* z-`9CZ>!KwBXXn@a7eawn=IZiX*0!*;WXzZJkdj(F-)!pZLnR{NRmXExu0=DDmh_79g1}!`p-5eu3JME# znmm|kS!RL!R-QJ`p2*T=wk6FL=QW?l>0)^x#`>5$oA{zD_I$~y}<4i3&qyzET87JS+f1*9kZDoMqBkW#_m$dmc7PvYFXIITit zm&_+418?O7Ob#?OG#?)yS=ki7Gkj&^Z@+bVryYu&y*_=?#mBd$s{(|kudU8y-|`IA z)KWmi95aR+t%FpA^fl$>)O2)o^z=2Nw?rv^CUSiq6_TimMrB2^>8Tvfl<3&O zGOIxVKbFaT2)N*}kPP(nI{Q?(lroxDWV^vd0E7<5%uN3qO}3}U_a#g>g}6{A^+FEi z%Ox?g-vb>rwGn)XMu4B(qbO}8d;%=gll-!bOfaQ{l$hQAGUg7!N0i>-My|1fKx2pQ z)M+%<@4EX-zijn>*_g^?y_l?U`0{q}cs~$OH&r}Q#*A5K1peXzLV*F|_eMhR4w06y zMLeJtF5C780JG;9j)ZmMpKDz3P$aI&K!dlNJy<>XMOD$CbM*y745V~8qm@&w((6`! zl2s8S5pLxG=`pdfv6i!yefZkZncP27dFrGN!v~3jmN?*qJZ|>v&me?%Uokl_NDqsX zx+pIi*%uYbo@(4anK}x%I6E4$itiynJ6~Vd)YR-@7@Hb9pP#3)na4HX4;t4bCnPN{ zEesagDb3DJVe?q2Rq3<=v|0N@SyjplvH&UjVtPaAsC&lN_O|o-?ON(iSDAl1jr1sF zwl7JPLkGIqX+K~9zaH_o>-mjJ8!wW(ooB6aciH=;C_sB&6DIEE^#tTV(${|>An06K z$)F9X&&n6TPo}45!Q(WMlCe3_^?i1}S#APoP(lLe=yq~XQD;5#^f~dbXzqD$wdv~DJ;e_htZjy zf>2vGjd>wBY&51_nneZPa>8fW|}~4Cgm3M1uQ8 z>-T$_UjAareiYB@~@8J%PS7C=pzG3pivr$4bJB;_CNz@r&ICrlzch z{onb&z3UpU*YA0E6}>bxnBz2d8{fIOx|$g#a3VstkLz^k{NX)^{U0HO^HYKsmWGCM zNmS)ds>R}1;+dxSK!k%H-zP%IbnkCaitJaFUPvcjtS*n@RwQJhudi?6-bCtSD+Web zkQq4e`x+3b3Z&LEnvMzepsJ@1@Fr94k;v_#VYof@!m+X7Wq}jVz@V>_+W<8edgKvk zxF0h>?$=1|q%eGrTip^7ONjGjHCmV2Rnjl*@*1+lunxfd0b>m{HSRV$QYS12 zP|P7z=!RDG=!i_`fDj}Gx#s2i(8hmY2lrP$<2F15tobWmo4D9jSQ{5$r+K8q^Lh80k^TC<3L7`oe-;1wBn43|5;vstvQ3g;!MH+a zb`0srzn%Sztjmu|2wGND?vHPP%`5_TV8Y|JBkxS^@AHW>uyQ>n6n78xJ3wBq^i=kN zSGw1B?)BE2$Ken^U|GW`bZ&Iql>y@0(I3V02Zl7dtokFp2JuT9X+TP~Dj`Zis_AdF;^DS$2NQ3jPl4`P$uyt8Qg;gve z={CuvCdrUZ|G;U7gD^+oM@g%r%n0;3dWL6!Gsrdo>2&}`J%wK~NREj5^+yMX!-;h5 z!>eu*i?Tx_txjj_l2A}}Zv!6Zs_U=DDz(R3pB&OlO0I^%5S@_>bo>tVfrmS(Bx{nb z(X<|qtCZ6d9!ee`U48wyjt(cb+x@AmP$3Z@JKs7sl?7M~%aK6Iq%dt;w2?p`J-fV; z^R%QPL!&4NfqWbRUt`BBO&N2X; zx(293z6AR@?B}<(&PSkkz<+oQHE+$#RK7jA{EYi!4kAY$A*iSrF`gWyiVhBRc21Q0 z8Z!{}?EZ9jJUa^|C9{Lzl#()vhD*lVvG4C+m#Pi(j^JVJY$SC&Mf!pCZ}SNLQAF87 z9KaqA2c7=xGBVG&I1jaK9=ASqtz6g^B^5OiV`DHPQBNl}FTU)7rDYMZ#RUc9DA)bi z?B9`Uf1NHhU4Cuw0u#k{B~bXn<2?mg5e@!r7Pvs%pl?~NDcbR7W(qo<7>MZTO!~ho z)|!U61KTC&{<6l_(bfI5@=c6=q0M@bp1wsUT`d3tr>zYaPfotu-}UX5O9T!MRs*N3 z!uqTGMLSwp{eNs%I7v3jGfA(t?JCZXG*3YC0Q2r$12gkBgfCl8Qxkq+wqamzf>)=b z-FSCwkmkEvsFRr3T^sOB5b;365g2rKmd^4x5LF2Q_WsB(6d@w$?k~g!?NNeJQP~p{ z-4XQu2&UtCvcdg(77#)ZM|)PFdH{>{fuejY5Q{hAxT_*8YHP?}t~P-~z+(m}jU{G1 zS$RxvXxR82iyh;S*?IMR+oqrAxtqaogCN3(fIExL>Ju6DQXrWuex&18s#n74dE?02 zv8$yWl9Xi9Nr#H}vT%K-+3u z!&P{J6OK76Gm&5d?_f(0DX(F`|I$ z2@{e{yFP-UJg=`SV*JtmEjs16{G;Z?KQEb#{Nyl$?AD4B_6?S-*As=LEtMh8G z+RXp2v-AFjv-{e(mT2+JFj1mKAEFy1dJqX=)I1MmAn9a}jFn-_`NK&IdGkNXQi70H`g;svvFt?q z8TS8hsuPFYlJJuRvbJjp9bvO>Nc;rQ&F*Uf`X1qPB}2kzDK7}LPEH-H;8^pFo%RPa zG4G^?pzN_+u+;Qb zrcn!rCvhfe=OO4eB@;ONjdKz&4G`;tPYM1;3A2?WBnWOQdqQ-B3L3$us->@_<;w5w z;N(!+iz-oi5iIZ{ssA41MM(|CBLcIDLJY80TV#s-WH% zJsKNBb8@4dA2a&E!s<^m^Cq4x+2Bjd)9MT_?0@QY7JPuwi_`O8V`s9~_10zGWSiWY zm{n-nq5&b5n)g4gh$`kJQt;CfZ-E6yyEZYUHbGe?cMN_Weop9pSdugD6gj!>mJUJp z;t2RGz0xdJ=k`bHLfgXB3+2*YUC_Ld%WNt0iPVw8R{wO;R?^w*z6nb$uquDGz|($^ zEo|{7EWz7#QA$v8yBYPH^t`AOnpnHyA3-0v(|zQM1iD``)c;7fjR zjvK5MxYqP5{>TrZ)m0%mdQlqASV&s>%nPMHZd_YZU-ILW1SNGxYOa3n9Dkrk4^@CN z?ONExu`|TW?`D0s-L{uoN*k3`{}SBKj_&G-)6`DQkFvQgQfTg9hLPJbdJ6fgiNKr0 zsA4pbOFf)#fa8&okv#$-@7BpeXz=$5SG>C0KSi@oTM3j2&EIK+aEpdPm0tb)cK+cI z$7e(~reW5oe^73Jyt8#t;BUM*QzzTlBwdy!Q$`S^!`R1O!n!Zx>x8b31paH8%}?kerO?`(aJ@h7@2kxA>}R`6V5l5vEWGX0sKHxNbYAc{|Zv5=+b-X_;NMv zom{}GP?jS8a8|IP(&m06mxnQ!03%%Ou(l>68XCduGBxVb5o-v#k(voQnb@W-L^AAh z7!Qfs*!?D~U=WK5{|QJg>O^u4yqa(nMG-TNHeKb zRArx4QFjA}OyDNOf5nS4yYWyl`aw_Ue)nLf$aj^Rl`IfM5*(&y+sk?lnG3$-oekT1TTqSTmUwkIOaOC#NQd%)}3$O`x%(J^d0XhK_1K3wq;ttpr zylabHNixbl#nssFfM>S<8KZg4=`r04(-Jhl+FEBs`_e`VBb(i4hJr`{gk^WSv8mFqwX!{s>)muWk+%-<@K_D$rf~YPy~oSVYs=WL z&!zumr%w2@$zsDziUxvdAh%Hw#JM^rJ(y?N;;!wjjVdnvH`B}e^|!OhL&LC6)N<{f zMH7=xb>(pDulV+0-6do6>#2i%w#n4qWZjUQY+o4t`_Z-6njhg!tigUexpm?t;=$z?nzM8L8aad>;`&<8qwefjSp}``OYW6W z3ziYt9T=Ll1)L5Qmk6XnIU9vZM()EBeBuK{MBHAm9z+LrQ8bMID$T2tDciF^E^H%i zD(eR)osVN1ulon*bJOK28Ut(~!+iFd{}f>SnB^~t0u##F2hMmZ+00`^!FF!n!b?{v zeIK_!cFW5AhJ?o{;rAZaaJhSI#bW#DmE-Oxh;4vs0>02%)s`9{wR@OaM=8|m|u zR>9^Q8N2Lb8XTih(a?A{rUyWO=iTB);R6rb!}-9aqP0&bcva9z^lULC_tPSzSDf-Y_cO@^Q)RgjbhXDbL)|ovq z&fqrRbLvfOCnG>KyUIfaO*vDy)9R`X${|@G;k1B=bJL;&n~x$C?Q}8oU-nPqqr)(P zp|HMMk1<4g-rcO!8#?-+qRXM-bVXHyaG)4=o4$_zkkYcx!z-($6-+@k*8DyVK_|B9 z2K3@C_R;&5>*J4To*F!{b;LAGdX23^X&#JWe1aNn~&X!5Gl3A!Ra;wZg%U4tj zBoVo3@s>8Z!jV6IYTI=2KEHUey9t~uV*7#g6IHToz@FTC4+BNJ8xrQ>Cl<>a!4p?l z*RmZ3+a%8q7Ta3Tv(w3-2@de}cWZKy29PJf7x7){!w$Vz-d6KnTiHIZeV;fPr)IgJ zy>!{NAF#8}8ge18PJRw{MV=T0=2{^L4Rkqy`E5w{Nazvs zQuZ0&;qC54pU*9E@RWfyRz`_;dg?Qe93u*^4^k!kQuchXBNSCPN#2^ki#TR>#X8W@|v>Dr`-JN@RhOmh-K1XC#8%Oc(T1ix!6yXwW!wBz8)*4 zQ!UXZoL^R6_Hp6&Sm(4u(4^gM1i{AOqD`(zE({e;8Tf=*>)KV}M4{F?xQ$Gw!uu^O z_4sv?Wjyb&zZn>GCF8S*kDDg|HAa$|P~gd$F%&<+AgJFcp-9eI3fDuh%?O44hl(ZI>9NKKnejG9&p z;SZa;RhT^PH(76NWNIXD4u|3IW%KUs1=nt5t)Xll+_DM5ET_Cr6yvn&SVjZ5Wu4b6 z1*;pf0`AYsnefcga9V!A*g5c`Jq>m~T%J)1IGc_!>Y#ux%u<6&(ySL zn4)7Mh!`3OMZLP09qS4u%6-$?e==k)6R_1|P;C_UnRbPx#&Y@QH`Ico@vvoou4(;B zI(^s@^II>be;MAw_iz|{)AXX>0?{e>-P2!>(Q+5bvw0j^|8N2}L71qet4yC$oT8Wo zM<}z!6w7P?kJWA32nTmJCD#$jaP==}R_!&Lw;9)RC+LIHSKZ6u9&%r&%aJGum=@I>~%3X|59jzu)rQBXZw9R1PH%o!_$qn3yk8!tn zcPCtWCUe%w`K46-;^5-WsZLNHzU#Yvx~BI%tF<=&b-@SWFW{n6Aw130vMgD0X}cH; zzDH4&v#pOYjAVE+cYkNl2abDyfW!{=P!+P!fGn3LeM($^luP*sZ0GFM5?V=xs#-$U z5&J%a!aL)zD;w6`{NeXu_kb#xMeoNO&=2M+?|s85O0cO#Pv7@(4O!XlWbj?glR@-Z zhcm*=6#ttU+TIGn$~?^|1GTKq`$>z}ws1P=K|lz2{ks(+#B_17C5Cn#^( z4SnEAp!0nB@1)mS*vKoIG6--)Xg)38-Q{avQH#<}AZnlpnY*0UWeSvu_kGD?z;Z(S z_NZv}y2jaZf)TP8C;Wu+23fFmYPs=lu9+Q(vGqn;gpTOfKHLK088fY7;NDTHXvJ4A zKZb4w4i3|I2j2)WTWoB}K1<^$z*I{RYF}vyI#OF(wxr=V=~x>=3d%%M6j@|^H!8B| zp@H-o?q%%CMfJmwGe%pt8x7X1DH%eO@&Cyd&&X^@Pf|*)YEqYNlqhM6D@1Hnn4FyY z4iDUFprw_SA_5x2-~PpX+T2v- z%=Ho?L8%c)> c*@yX+>tHIh+7M~NH4)*_P}6=?p<)sGKdn;7)Bpeg literal 0 HcmV?d00001 diff --git a/source/tools/monitor/unity/beaver/guide/image/queue.png b/source/tools/monitor/unity/beaver/guide/image/queue.png new file mode 100644 index 0000000000000000000000000000000000000000..01fc076bf35716308fccc70cc0c37e9718810a9c GIT binary patch literal 14649 zcmd_RWl&wuvIh!;0Kwf|LvVL@cXtTxuyJ5}ziKQZ2j!sTi-V+= zGZ+{g+TR--EIkwS5(3UrRTH2oC(CVOZ%c3V-QL)g-rd&WuND}uJ2!~5H3b+EyW85> zIdi-7k^UpW4Wj?D8Ayr$5dm29k!s2*5{uY7nG&JzwC&b zI-59IIsh!~?TG){H8Qq$0q~KM{>|t=*T2dMur&Mcoa~(c{aK(7Wcd4rfr*}x;Xk%P zs=R->+{&iT_BJkm&8yj20{EGE{}KM*?ElW~Us?)ImZqTc{#D~=`nSse-y zZT{hc8iya2m*GDR%?~>Vd;Ap)jGabGR7lkw{45LFFzVrBIOjmzbtDGfI83uDcdSQz zqExd;yFj!`SyiQ|F?w#_T%};399IlO>{QX~qRSWxg)GkTLW=d|OI{i0iQ z<f4-yeo#F8Ul18QI`9xxKStAND)TV9|jp*AB9+NN6k^u zI2;V677Tob4NkC-7({PkfM|UTXo^sf5F9eGUIUmYY~cTY6&OVpC-ntr${40Uo||s1 zUu}1GQevI8oa}aaEf#XU%`c|-{75f99U$)VKni)0cNSw2)iGyb=;{l2b#)hZzqb?_ z0PXSoyq+FS=`+PAqsb>J-Vj8JT`b>&7R9aU~# z?;)j>Dc49x7qdB}WheW(1GfrRV}{tH^qd@}jC%SQ{93IV`sUnD_m41jzb!3KX#LHa z1rRM`F%g~~!~?%SZsHYl#*{bQ8GXZky79ZoauH&`(CB+VP{yE35EeVpp$koa78s+M zHVI2lW+e{5UJJOHakiQI!bfS7+2Z&S`X<+5RYCpvNt8WG*VDsNNVvAufMq4GZD_sH zjAe&=%k#>SInI=!W}CBVO+=UQN@y#Gi#=DR>DUgso<)oaac6M~+dnnFV55%arxO2h z)3DNI<~B8wUG|{tB9tfJLTm@_=JEAP(WYSO>r`nqM!TJU>(k5rMonrfL`%#WCzQdW zyaCWX8N7coiHGVweTlu$R^_3%UmMXY^I8)teFENWaS>h_iYZVo+#tOBt)L_}Mvp%^ zE-9~p*X0iV04g7okCl6bqtqNQwTfi#$Mg=@Nd4n8dJvH6NMVzPkA)yw3%fn~LVch_ zayXPAR-e1wZi5?1BeqE3@+2!4m~{sqya;F>7z3y~d|~P{=s_0nF;F1l>+(KzQWsC2 zygn|g$ixd@`F)tzEz~{DS3vZ~m-9XH%Hj_cI-Ec7mg?v-DJ{E9z03%E!?sfaB`1^s zPoLYl z*2Q5#$O5zTHSuyboPx~hPG_X*cU{{sY#UbJx-?T#ygcRJvF!I`kZ*yY_{XNAoKLJE*bIAI z-QJ5v&Hyef-20dok9WPFAfF|_w|#tb0qNa8@9kGpi!(<(;iC+aMRE4hMT-|)fTfg{ z{etqn;-ENl(V!T@OtdQM-f2lg==08C$y)_=Gnt?NbTg)$0(Viz9esFT(^gTr=$ptM zOKMB*3v3-l8o6{_W8`9b$O>F1jWr1I2tw9i2c{=dtjTN@uX^;AY)L-`zbL}WH*;Gv zS4uY;BNGVo+@4}I;!Y6IAk5lCk#N_j3}pAcvsRp?xj=R6Hyh9ZgI| zWagU(1`Q|H#or5*N2ML^P5NS|^m$djVu&0d%s=6-I9hC!`DDOb&9pfl$h6SQX|aj* zX^ZK&F#VVkbeCgLF*fCkw4>cQA`ku=Hcje zIa(;EuF(8a(E;QLQe!Q0DjWOF0I%VKy}Sh+HSnE2jSR1ak?(nKJZ8D8C_A(`;i@~h zCIFiI@}U9Yb+el6zf@BrnrF(CiIxX_#w)wME1LYnga87m6V*Q;RZfx->S1)^uJ0ci zHITQv7i*{uXO!>JzO%fWpR4&j&0cpfKmc^}=DV?`WpQwwuV&ixZF3lEOO!OIfVDZj z2AYN;5fN%U%tvW8AnzV=(omf&1nE!&N{j7nv9Q0WD*veV43+((pFpQ>;7BPrMgC^AlfYNo@IZof{S8d_a;zF|HgZYL+ zp43$>o5|yjn6eeOm4=g6eO_H!??d?h1SGP!#IohyQh7L^dzDbiLEB;U)OosjPF^Yefw1mMIc!i;V#5ANR=J#u<|QWgd%7 zlOC_-Gy|#UmyLe-%3H<*6(EN=NoK?PK3j`qb^PiiJ<&EFf6*F$0gpA@7K}@!6rGTs zumNiF8;0^2dtD?yCaBM${0sjzj$O5Xh?|9IcV&wd+uU2>Guapm=ln2n(=c+?2v)Th z6@z$s4qxLUC4P${+#*&g<_D&PrKmX{Dh7Zc9+~kE#Fc_{0KU_MWLVRpcZj$2N_oLy zDXuG@1O%EF*c1)5nmyZ&AFmU_f2)XrP$l#asvxVK=7C$*$qXHw&Bh<;deHI}@gQu7Y7hghS+$DTmD{GMr{xoE73~p0y6AyoQDw2NJI%|W z0T4ifSY%raLKk9DWw~~s%3QUIe$IV1%w(=MmKXC2@rzz&GQ!t zfAG%oMyd^Ye&xlb#*0>p7~R+6adXlrRzyzloLIR^#qD-+q5Jve zI}Uro=xN?DlY*&BYq6R3=DBCNrxo^u%YM6X0kXULY4Ypa@mo`3Q#PYQt~H5G`1*`d zTeu$1V$yNavh2O{J{Nj4+)}jTi`>1AQFHCywuiMOILdgCvwfHK4&K5%8`Z*>DTI1R z%afX80X44qp=C4!K*oHAf|b1R;^jymaX0)5u|&l20Mr=Nku3LipBG2smnNaYg&5(< zF}E?dc1HU4@XJ~2A$Ds@%i@N*a*|x$o6SHq6Gk$JlVe=e?T`yc68k1lJpp1(QPqXcd4Zs@|rj5$@9IBZmi zE);Kad(DeETUp)ehryrr;8;AKxjctxmUj4Hce%BdVUuba8ai2>k-l znJN>(O&9rT%mc|3>G4JFp61=mx_(-2RL%JsBh`oZOzGmN;5f)m=(ae?2)IOr(l+S1 z?Y95qvma#d)M2i}bM~xg?Z>GkLhCrgIUcNAq(84dv#=8PgU!3MG$Cy_URG*6?~A$Q z+&Ila^l|iIG&hC7pohpTID+y`U~R<-PdZP(YJYP@i$a{-lje1u-fXxwCxrUEo2OeJW)?})sAP6-9313Rre zVLltSQxZg44NDHg*kM7vD%Yx)H0HDFTb5eJBt`>F)*mC**JC}dtZ?JA&@8O=L|l84 zz0ZJ;#!mrR4fNxON=ILT$rxrUXK7ZIU^R4Q>v}8e*%6EacC1`9G&Cl;CP6m+sLQB! zt^rTKmPW#4jKr@3mG+jta+6Ij2`y?S52_*ooM~0*SmT-7lV}2H(rF5<+v1swxiq;r zL!{E^v+M>CHtYzLiL6R39*eGlN;67Q*M*W&InRm{FyWa_1<{kbBodqMH{XuryJz{Psk_pCm-=|6p3HNOXf%^dxHI@Of5}M{i@Ko_k*-S zx%2{?p=v5IhfSS*$h*<{+S-B*&Wh0{5+TO`Y1NV3CA;RDZ}8&i2I-8G%qa{j$kz-XOun2c;?MOCR@S{1z53Bwt~ zZxZ`P5tf|POxp^s#*%6h6{eX~`FQ!#CL`=sD-U#qUTl>9SEpZH{b$NxCvzv5WmxM8 z8#GhNEiw?YQRps>)nnE3=<|VTn>5!n8!|lyXAULQT=CpYW3(#x+KbUyE(mEaTKCR! z&1NE|B2fc695sDNS-5$W$y?fYE9Yb98Q(K*Tl5&GYbG*{HcT41l_W&Mj$EGZ+_gGw z+5H=>po%7=m{-f0Qqf!ATo6VxDKkQk-B?Un%A0T)Sm8rwaswPiTRX(Npk#ivj5Mnz zsZPw+>EUOp=DccNF|nUkkEwT&^jrnqLX*g2Ok!k>k18E_?EWxm2Dnl;qf4h-*|uG8 z(Ux_aK_a?Y7hCOv?SX}_@Wi`a18lsYQ)5FB_hJ)?=Hups1XhTTh#fPlbIGU+p3`GK z!>7(Q3pVG%>J&xg%cfJxePE`!Kvr8-&lrCqh~_{LNfIe3D}E#UfCg&+>9m%>vjIf; zc-C>Y_tbxw$4@Cr;1Lt}n})c`#wX?POYzKMp8S!c;*?l(g+|=Ou$#v| z!)sj&QR?Dm9m@W)CH9fd4`!qKTYKjD(;&1Po7>&lXjB%j9aD=mBLz$dIQTePUk9>; z>{;U3ZfxoL?AnXalZd0(6}iid^s;m%&ZFdYT&ozuUOt~0fq|Ewdx!RNi-$oOX*HP|JPEV2Z!rjMs#oio~ zywwn-5D0VY#ZYTGRmXy;5@Xw;UJreGC;blf^@_F4B4?0=@eg(TfB!&Mw(4LtW)K)h zPN&V_)n#8hS0e%BSE*JFbVfIiHOJYIb59ooi+Qx;@=(5Q1yo;E_{n`h2T1dzV@!`7 z-lX4L&A4|6G(B53M5Y1chph}@ml`mH4SM45@0#)U#0=bRKt-h(!s^hN4+aA?A zm*iQi8qE4BIx}e`s`8QxH0m{UW!104Mc@CHm3aTX%cgf^B5qwsimKyIltR=}Q<_%+ z>7|yeoY=RDrSl~Z4S`CStFkoDe@+@oh41uz%aTWR__J4d!NUG8&pGvl?y9z`1~z^-1B8a1i%ldQmtXeW^GAA_0i^_BHEZ1b$WG8zk1>{QPR!}{n zHv3!oe1mh7L?Cwv2xg40)#>i&s(-`0OZKSuAk`$V1Zro8qRaij+u0xW+&@zGMnuOQ z)V$N2i@!9EOcq_`4AKNfL^SnO_|^Pcc$9utgKPdQ*W}Rv!%S%DwZIFJR#_57bF`A- zAGK=8A>COGT?VD5r21=G9fEIe!JW{#D{?>9q?)xiW^7$`y>@Sn>8Z|;ol}Ezq1LnENp?s?oYbUNVLE_IR?)P006Hq?EruOK7Sv=| z9b6_KNZvYMb)wXw=Pjr>BR_Wh+e1+!1ph`&BV;b<1E(_FfL#f7CEu$$mtk|K3bmbWr3YF zTf=D~>a>*mxXuXXBswfmz9C(5T~&ekeV_J!X+13~6y>ZCJ8DL;39*szQsE!0cK3G& zk;v2fTG3k48ihRL)q%6>q5ryI*1W2IF@)7wzrb#LTeyR39seDYNiNgp%V3dUr*6N% zgi!?kBD&h$utrZ_mm$Mp$`R!?nLIHhi)dZ*AnHRD*4v`gNL}&OOUN0V?7}>25-}06 z(Y4dn$hu4N(-;ft(aENsPilB_4TG&cYHc+*=AR-9vdZMDp{Y&0Jr2_Bj>un%HZVpn z2mM1##yL8wv%J``EY~cY?|tQUCQb~9f`VQ*=JQh%B)(KZEOTH8KqA&F&%t z`ve+iqhr4hqO=B|>X0gH{F;x1*A2_3BUQ?#{CrZ(g(bp#1Y>lc58V9|4s}+-V*Jn+ zbZ!MEv{h-U?|Xc)yR zR=He^f3pz}P+LWQk2D)G11P%+e~dU!IP*9+S==t^!Vf=Vnm+oLD!(&FFE=l3HOJ&5 z?mJ{A<}@lte|ceKP^PmgvZbEif%$>dWr=p1|-4LhPF*9V5B6eeEv- z9^D_Ji=0jta+wJ^DlAquC<&$Vji2OCL_LDq0!&H=76YOY7|klo!f5(1qP7EHg#|1H zaICk!B!wF%c&b2~h9Qf=O!6I4pLrPg5+x9wNFUE6Y6JPJuu;X}8ogO})W++<52#q- z1jL`D0>$1oUnAS<+j!=9en03MDErj|>al=9mVuvvWKamwwZ^ORs2z01B8c`wUKKXh zrc!%>C}V>UN_W!~#}uhSp?vJzGTaI&3MF~|x{O+rV=amI!CfnFv{=UA!l-IKa>H99 zqhYB;{^#bZYnLz6%W!kV^^;8{=_P0AXN(f?2aIQl7mbyrM(slu^M}`5djd+n+k@^# z6x7{I#fhR5D@Xz>c|J%M6*k_FwPq96(Xp`yBKdL({S&`4ysXd9YRiL;^xY4yt*$DV zi)c@qC5J@ZdU|yFt*i(tDKxZvq5CkZG+KKcYMhFtr+LiWB!&hDmn&~SKd98>;6#dk zcjalUMgaaH<$A@>Q>~ShbcHLdUN>%*9;m6G9gz;`nJZ2=gUj%Oq#AA@<=0*tWL&n&w!zxvvV6Yia;8I~>an}54bxfewGsa_;e7qWdK3m250tcjl z-&!wuXVXbp-g$ZO5*H_wmeu7pgI_84o6Ibmw(uq5s5VJsM###{=jDTDvvj(6b|Hs* zBX(#rdAk0x-r$w6QXfaNTuM7LhiKSKCga$q@{>Qlpc%3p0U0Fx-rMM(Qep`D zgWRm92q*WqUUMSer;}Sl3oo}tB{>;4zF>kf6(q1cC&k--LIT{-j zhbj_wd|IkA-5JVSYGY{y9Oa@?sLQ2YpZp}*vzjaY7^NzEIJebrbx^+egNBMW{|!TT zI{=Pa50_!dWVNM#aiK^?4R$ar$H(<%t&ISfoL4|j$xj%&qS0iv-sMX7jZ*x4x6$%f zAdA<-B)2!&6kM`FL$pU`qK;D?pIbewS&#Un*;Yt z=4}3uiMAkU=(#d9#Y)YJZXbJt_S@k?DfgrN<6*u`cc~L^zD}2u>#LpC-Tqn0_F@_= zAl|ut*UH?w9u_^YzQ>?rNKbffTfMq+v6_HBC5BLMWYKLVhM<~(U`Oj0p7H?=BdyK- zR)1^AuS)iSfGK5_4_PV4o7;ieS#Dx59}0&yyL+3FwO~4(ayF}AU@;R@SA~Yb3y^_t zIBGgX1PYqm-<4rlL;p-*qc9!SMwdszgna@j=o8jhKfy6o-Mj$Zc>MAl6DGayo zVwLxL%aUBrOXoCTWH$nf%4zK89ITEYZ$lO<9;a2fqR|Gtl=OG>+8-C7&Zz3WORXP&w z#r=n=P>q6djsU=Jx(_Qsp|WLl3nwtMva;djeoJeLZ0hr`p0hVQCPrHgof7ti7rnj?1B zJ*d!%_wGG4I6sQ))|*7LlE7K1)<5LPT9 z0TvoduAN?#O}o3hH!2>S%&mQ{)zF>gCaQ`ECwgkBYN2REWEc__0@49t6%`VZap(KY zbb&6*_QGIfrpzN>lNX2i{$J=0C55cN=U4rp*X`s*vi;NP*B?rx)6(s^Eo$b;z0i@G zO$l%v$;^aev&(FCcbqw%PlXA*s>JN_Oe2J7L#GKSC-Pk?v|cn>Z8p+$p13`lA`+-p zE}0Aw3A-e84td7}%h+ff^9w`?Z_KY78k+z|>8+5>-q zT8{Hl`HaUD*34=A@!5K^NVLXWBe1$9h3^-X@0OO%HYq%A1~6dAUdUrgh!%bkjB+3eIBA-O)aY&w2T>KQSFQbDm+( z=>;nF>FSWQ6Wzt*<72y{W7{86U?6hu?TUokxuJIg-m|y+K}<l9@|86aCE1W74EQ4VZ@qLo z!7!Na;tr}5L@}`bf=4HjChNuXjUIyv^(sHQLxWY6+GPc)Ds>_r*GkDa(!9L9W?;6S3G+p#TL)zlX$x;a zWo2c#T1B(PwDIXuT{2RZ@Gf*fx6ey)VPU0q(+{H2Iz1*Lvm8;end^<&EnO{5w%d%H$Z)>YE=;>C>6OP8?@V(ysvJo$F zG)fdOK0dxsr4x(CF*z-Y&|2D^++!{t+|7zBXJc~;deLIDjEas<-iObvncNOI>Z?EP z3xOk`-B>wQ_L%^O<9ndp;b63!DU{FRp0Cupx!wb|@sR2SAg2oWy-ycN_&x1m1r#p! zjTSzqhXni}=Iq=GgU6{i>JPP=FOMMRK}-#qEm7dGSz@H7F4JvuLQM7fo_1^+87PIlk z-2?^qpZL;&Qjm$#6i|M@6Bs>jEfRRc=i-)65JblU>P;ncUC&lZYuId8ZnyfwQkN{< zGpaqLHdpO?1~UiZ$Rw_rU7BUkkJ8UH;pmH{lfIp;wjlAbArbP1m)tpbGV}~aVtu?l zY)Hi5$4TOM`9-Ycb^2+aWeMIcvVy8XJR0|~K94A>OJ=9)N@VgkMZ4F-IjB;4=HvR; z8~}Q5sezMz;%~37jP;>uCx;fm$;nAS!QELaNX0yH^TSS$8_7A&p+akQuLV)`+OV+U zVz~@OEs^NW%B{iF=ut>HKx?&)4xXiRC3gM;>HB$=1jQr?wtA8S$t)Kt%MdwWn z+3sQ`uQJaZt?uq41~@?V<7$}bvBULDJJ8q*VsvyAn?keJ5IjP=`m&55od&ahJVUU{ z^RBi4r5wJjtPDE}xcaHB78x8~>J!gBrNy7lwva${9mY9C44sG-P63S?{lUOsLOxG; z0`9pXB_1@chMo);YOrRc2wjaL2Y$Xv1Hymnoi#DwI zvlNHHbB1ll`>V8nur`{_yS}3tfGQ}GfY2veM%Btbb|+{dEG;b$KM;|SB(Pd##jAo_`uf+* z=H|YsS8{sYpSE!4<9UF#7y=`a*IST5zEAel8d9mBnOH+Z11{uSxuHJ{m1H8l9_|UO zMUk!0kK`c)SmD^$bRjV9(D@Zj#}IaE?!k%6BW(GG`uhBWg8sg~TKylQRsN>(?dpR5 zyX)`}rIlN!e;)pLA+3gQba^ep=NP;F_ynK#!ihjdLK1*zN#|~vP>qpE5@HAiJ|{{k zpSc8b_v4F$X@zEw=rt;HbHultoDjn`zr_hKsm+fK8eqg>2W%iBG3olqLlN-URV&md zY^F%B{R1HIulCN)y1bv9tbDc04M8z}>Ui`R4@i)k4y;oGB$&j)K8mJrwI_qr%=EuP zxJ6SF5F?|bH`**Sqq4zaQqC{Z!XyMTXY~H*@&e&LdP~5j&-r|f%f>Z$Pz2G*3Li=# zo!$0FkcQbrx>&y1eSyi`r}@RG?Ys_d4_HL%(H|l$3bn z!Th=_luGb@y^<(wR4I`!Ej3p02Mr%^BwKwlWJl?jW3sZ;L;2@PG#bbD1Tja5no``x z129acXdb^F!4Ly``xH|3baiX*_V3SD#gZl7zlW{ly;8$T$d1mHsZ3*vwBY(4>%J>F zigby4qq$9Yi_>HP$tYb=((=<4d?>gwz^EJa3V;JX@;!q?REOOZfEsd`v2WfXE)(l#>YR z*!;fmk?{C=M_VL!mxg@2tb~N$kdT@Z&~{E=k1JSu5*bTV@8K9Sg{sUh#V~%bY%&bc z^SsP+P#rXp@ae^8)5|HC&(5;p;8+zYpY82!?d|E@i9q#Px92^|%?G?!!XF!b` z2>C>xh$I5{QoVJmb=#Osh75Uc<%YkjIDiWJ*<6xDGZn1d9+@)=&$vp>={j1crFy;N zL6P1Z;#1L|Kd?kZO)VTdI|Xkmrsl{+@gA?5FLE;d22$GTycA*~&MHBHuONJDRcboQ z4g|R$_}pPMvrlso2MvDtS*?Xn{aKS3!h$1v-Zy5w(}Rj! zRKHTwwa!>Jjx?&nDtanQI3z^1RIy1o1QwZ-wxV~)*}~qK z7c${4Q!%Ufr!w}2;!DK#L9ylllWw3-%E0LbwOm(bq zQ6pmVP24!tLL4}{BH?qs?xk#_u>BOj@fbngf4WsppN?(JaCTpDS{3$JSYdLsaR;Ge8eps$5#j{4eh9oYdkYnb0W-Og#T&<5`#~+;T zG{cscocVQ@fM8^oP`(&WR9Ji47vc^EeT!ic`}G+_T*D@7QOMaA zhMj}qfnhk}D^eNKqx2D2)yw4oWKl7u)S4eYeM+-jo{3MO71Uk}!`%CdjJD8p0<+bX zlOUW<&BM_N!iFn)7&jvap^$~v<3G{TF+K!^Ktoj^&dlPRM@m^S zZZe?yACq%%VEb+OXz+-FO%x$1fPO!&3W>ks48aI8!>fMMyGG{mY4dngtl)%-htvxi z!qJi_>Ph^eBk>vUljH7OgM*PRU6hIfxwyShKwaW2U&r5l<8ZD&8gn6uX;Sx-4`ew; zh`OJ*KeIgSsYe`t49+M)$^@$YuHOpqn_(kFhoJan1u0ol8JfHZp0V-PdvdC{4cd!N z3;}+@g&-YdDkB(-G)z;=y{FN8}_I_9kGfZ|9H}kFQ;*!xh5;2W5+hsOMy*i8V>O;rnP=Y3U3e zJK(1z`JwvB%$~y1VHm%W@Xpebb}bHjC8Cvgf7r#32#>|(G+*lHpO1FM&$t^h;WH^n z5Eoroum(QQuWz6pet*$3Um!96@&KJgf7anLm>A`JKzQ6i!tw#~JXbwYUqj5$z8d`TTdt4o>eSudcAYP0+nBO{tue;(Gv9MfsmE~&V5cve_nx?36cTS{hC?SfEk`RsTYL`o#e<3 z6BLDPwX_)d{dn~e9H!v+%vxci)x+;&RjI#Dm|P-X)cInE4Bzv1-!5W%G_z>77WmV4 z1%Lx}eltvTz=*HZ-MP_&d0WE>qv#`ty(d~Wo&eT96hqj>VLlnRKLIoryqPJQQ{aF4 z6qH?ET^|D1G|N`5fh69e)8#p6KEb_x{LOx*a4wF_ZCa5HrcESlzChe39P6l%Xy3d{9PZZqtj}KHq*dsha`) zTpGHlx#?5JqV5Sof=;vSY}2M0gb0TOK*0NlhGH_Omc}I&^;V%$bQaC%hED$Wmy?o` zNRp#Wz~mLWx{}Q_Wl_Imjo^bU2OS3pCgAjQYC|7Oq5*^VG+vRYZZXtqt%d8h?~u8F zrTPg3{`iO}M*nATXcWjNu|R~`fmUFMB|(E}AT7HzE6Bx&|BbZ-h_rzG4Kg|C?28a4 z9z{CnybBp99uw7@rvvgdF!3PoLmU~6DhC$ip~5S6px?;#b#hY zx?~QAm>`pZpj9Bs5#s_;G|7ju+mh zlzfdxurF>vee$|H9G^O1IEG-o+egOYt4f)&{BnI%LIN>MEcZ&0%=po)yxV#0-7#CK z`q_LyVc>WaBbr=0<+u}?eK`i~V4L&7WG0vA&3@KTWRL0mWg{cd2+vvIzvwYJVtBr8 z5}%N!(~|k`@x2lo2L**BrH4N{aS;buV;i*5RxO)!3<2_de&Ht`%q4OM|MGBzgd_cS^((%bWxkdzdckdT%YZI8(RSfTh; z?b{tN7Rp>@3ykk|Fw9g7zlJKw>ZAKr+JQYum)_acwQpGK#qqzUpAP^xw};1voo!)> icFihLI`qqrPdhJEH-J!EUC@z1Fex#4(Q0AC!2bv9VrD@A literal 0 HcmV?d00001 diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 7595760e..2b002141 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -35,9 +35,11 @@ cp beeQ/run.sh ${APP}/beeQ/ mkdir ${APP}/collector mkdir ${APP}/collector/native +mkdir ${APP}/collector/outline cp collector/native/*.so* ${APP}/collector/native/ cp collector/native/*.lua ${APP}/collector/native/ cp collector/*.lua ${APP}/collector/ +cp collector/outline/*.lua ${APP}/collector/outline cp collector/plugin.yaml ${APP}/collector/ mkdir ${APP}/common diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 0221f07c..9c23dcdd 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -9,4 +9,8 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../install/ export LUA_PATH="../../lua/?.lua;../../lua/?/init.lua;" export LUA_CPATH="../../lib/?.so;../../lib/loadall.so;" -./unity-mon +yaml_path=$1 +[ ! $yaml_path ] && yaml_path="/etc/sysak/plugin.yaml" + +echo $yaml_yaml_path +./unity-mon $yaml_path diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index cc3f9066..8e3f3ec8 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -7,7 +7,7 @@ config: mode: specify name: test_specify # mode: hostip - proc_path: /mnt/home/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ + proc_path: /mnt/host/ # in container mode, like -v /:/mnt/host , should use /mnt/host/ # proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ outline: diff --git a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c index ceee0758..169e5062 100644 --- a/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c +++ b/source/tools/monitor/unity/collector/plugin/threads/sample_threads.c @@ -17,6 +17,7 @@ int init(void * arg) { } static int sample_thread_func(struct beeQ* q, void * arg) { + unsigned int ret; while (plugin_is_working()) { static double value = 1.0; struct unity_line* line; @@ -29,7 +30,10 @@ static int sample_thread_func(struct beeQ* q, void * arg) { unity_set_value(line, 1, "value2", 2.0 + value); unity_set_log(line, "log", "hello world."); beeQ_send(q, lines); - sleep(1); + ret = sleep(5); + if (ret > 0) { // interrupt by signal + break; + } } return 0; } -- Gitee From 22208dea742fd8d737493485d03c7ce4b525475c Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 22 Feb 2023 18:05:44 +0800 Subject: [PATCH 12/21] modify link. --- source/tools/monitor/unity/beaver/guide/guide.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/tools/monitor/unity/beaver/guide/guide.md b/source/tools/monitor/unity/beaver/guide/guide.md index d6b61fa3..654a28c5 100644 --- a/source/tools/monitor/unity/beaver/guide/guide.md +++ b/source/tools/monitor/unity/beaver/guide/guide.md @@ -1,8 +1,8 @@ # 目录 - -1. [插件化与热更新](/guide/hotplugin.md) -2. [面向对象设计](/guide/oop.md) -3. [字符串处理](/guide/pystring.md) -4. [页面开发](/guide/webdevel.md) -5. [proc和probe记录表](/guide/proc_probe.md) -6. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file +1. [开发手册](/guide/guide.md) +2. [插件化与热更新](/guide/hotplugin.md) +3. [面向对象设计](/guide/oop.md) +4. [字符串处理](/guide/pystring.md) +5. [页面开发](/guide/webdevel.md) +6. [proc和probe记录表](/guide/proc_probe.md) +7. [采集proc 接口指标](/guide/dev_proc.md) \ No newline at end of file -- Gitee From 42ad0363a9bfd4c393442778cccb509a831bcd4e Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Thu, 23 Feb 2023 07:38:27 +0000 Subject: [PATCH 13/21] proc_schedstat: Add max and min statistics Signed-off-by: Hailong Liu --- .../plugin/proc_schedstat/proc_schedstat.c | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c index f4691264..5d1075f8 100644 --- a/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c +++ b/source/tools/monitor/unity/collector/plugin/proc_schedstat/proc_schedstat.c @@ -21,6 +21,7 @@ struct sched_stats { long nr_cpus; char *real_proc_path; struct unity_line **lines1; +struct sched_stats curr_min, curr_max; struct sched_stats *schstats, *schstats2, *delta, *curr, *oldp; int init(void * arg) @@ -73,6 +74,9 @@ int init(void * arg) printf("WARN: proc_schedstat install FAIL calloc 4\n"); return ret; } + + curr_min.delay = -1ULL; + curr_max.delay = 0; printf("proc_schedstat plugin install.\n"); return 0; } @@ -143,6 +147,16 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline1[cpu], 0, "cpu", cpu_name); unity_set_value(uline1[cpu], 0, "pcount", delta[cpu].pcount); unity_set_value(uline1[cpu], 1, "delay", delta[cpu].delay); + if (delta[cpu].delay > curr_max.delay) { + curr_max.delay = delta[cpu].delay; + curr_max.pcount = delta[cpu].pcount; + strcpy(curr_max.cpu_name, cpu_name); + } + if (delta[cpu].delay < curr_min.delay) { + curr_min.delay = delta[cpu].delay; + curr_min.pcount = delta[cpu].pcount; + strcpy(curr_min.cpu_name, cpu_name); + } } } @@ -158,6 +172,7 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) unity_set_index(uline2, 0, "summary", "avg"); unity_set_value(uline2, 0, "pcount", sum_cnt/nr_cpus); unity_set_value(uline2, 1, "delay", sum_delay/nr_cpus); + tmp = curr; curr = oldp; oldp = tmp; @@ -168,16 +183,28 @@ int full_line(struct unity_line **uline1, struct unity_line *uline2) int call(int t, struct unity_lines* lines) { int i = 0; static double value = 0.0; - struct unity_line* line2; + struct unity_line *line2, *line3, *line4; - unity_alloc_lines(lines, nr_cpus+1); + unity_alloc_lines(lines, nr_cpus+3); for (i = 0; i < nr_cpus; i++) { lines1[i] = unity_get_line(lines, i); - unity_set_table(lines1[i], "sched_moni"); + unity_set_table(lines1[i], "proc_schedstat"); } line2 = unity_get_line(lines, nr_cpus); - unity_set_table(line2, "sched_moni"); + unity_set_table(line2, "proc_schedstat"); full_line(lines1, line2); + + line3 = unity_get_line(lines, nr_cpus+1); + unity_set_table(line3, "proc_schedstat"); + unity_set_index(line3, 0, "max", curr_max.cpu_name); + unity_set_value(line3, 0, "pcount", curr_max.delay); + unity_set_value(line3, 1, "delay", curr_max.pcount); + + line4 = unity_get_line(lines, nr_cpus+2); + unity_set_table(line4, "proc_schedstat"); + unity_set_index(line4, 0, "min", curr_min.cpu_name); + unity_set_value(line4, 0, "pcount", curr_min.delay); + unity_set_value(line4, 1, "delay", curr_min.pcount); return 0; } -- Gitee From d5c5fa17c9a2c2d0182d0ec26ec3cdcb1f82e7db Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Thu, 23 Feb 2023 23:06:53 +0800 Subject: [PATCH 14/21] change insert to index. use yaml to manage lua plugin. --- source/tools/monitor/unity/beaver/export.lua | 18 +++++-- source/tools/monitor/unity/beaver/frame.lua | 4 +- .../monitor/unity/beaver/guide/develop.md | 4 ++ .../tools/monitor/unity/beaver/identity.lua | 4 +- .../tools/monitor/unity/beeQ/proto_queue.lua | 14 ++++-- source/tools/monitor/unity/collector/loop.lua | 38 ++++++--------- .../unity/collector/outline/pipeMon.lua | 16 +++++-- .../tools/monitor/unity/collector/plugin.lua | 12 +++-- .../tools/monitor/unity/collector/plugin.yaml | 3 ++ .../monitor/unity/collector/proc_mounts.lua | 4 +- .../monitor/unity/collector/proc_netdev.lua | 6 +-- .../unity/collector/proc_snmp_stat.lua | 8 +++- .../monitor/unity/collector/proc_sockstat.lua | 4 +- .../monitor/unity/collector/proc_statm.lua | 4 +- .../tools/monitor/unity/collector/vproc.lua | 4 +- .../tools/monitor/unity/common/lineParse.lua | 11 +++-- .../tools/monitor/unity/common/pystring.lua | 24 +++++++--- .../tools/monitor/unity/httplib/httpComm.lua | 7 ++- .../unity/test/curl/beaver/beavers.lua | 4 +- .../tools/monitor/unity/test/host/hostIp.lua | 4 +- source/tools/monitor/unity/test/string/py.lua | 7 +++ source/tools/monitor/unity/tsdb/foxTSDB.lua | 48 +++++++++++++------ 22 files changed, 165 insertions(+), 83 deletions(-) diff --git a/source/tools/monitor/unity/beaver/export.lua b/source/tools/monitor/unity/beaver/export.lua index e1436c60..b6f3b52e 100644 --- a/source/tools/monitor/unity/beaver/export.lua +++ b/source/tools/monitor/unity/beaver/export.lua @@ -24,11 +24,13 @@ local function qFormData(from, tData) local res = {} local len = #tData local last = 0 + local c = 0 for i = len, 1, -1 do local line = tData[i] if from == line.title then if last == 0 or last == line.time then - table.insert(res, line) + c = c + 1 + res[c] = line last = line.time else break @@ -40,8 +42,10 @@ end local function packLine(title, ls, v) local tLs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(tLs, string.format("%s=\"%s\"", k , v)) + c = c + 1 + tLs[c] = string.format("%s=\"%s\"", k , v) end local label = "" if #tLs then @@ -56,15 +60,18 @@ function Cexport:export() self._fox:resize() self._fox:qlast(self._freq, qs) local res = {} + local c = 0 for _, line in ipairs(self._tDescr) do local from = line.from local tFroms = qFormData(from, qs) if #tFroms then local title = line.title local help = string.format("# HELP %s %s", title, line.help) - table.insert(res, help) + c = c + 1 + res[c] = help local sType = string.format("# TYPE %s %s", title, line.type) - table.insert(res, sType) + c = c + 1 + res[c] = sType for _, tFrom in ipairs(tFroms) do local labels = system:deepcopy(tFrom.labels) @@ -74,7 +81,8 @@ function Cexport:export() labels.instance = self._instance for k, v in pairs(tFrom.values) do labels[line.head] = k - table.insert(res, packLine(title, labels, v)) + c = c + 1 + res[c] = packLine(title, labels, v) end end end diff --git a/source/tools/monitor/unity/beaver/frame.lua b/source/tools/monitor/unity/beaver/frame.lua index 09e95484..29b98103 100644 --- a/source/tools/monitor/unity/beaver/frame.lua +++ b/source/tools/monitor/unity/beaver/frame.lua @@ -21,11 +21,13 @@ end local function waitDataRest(fread, rest, tReq) local len = 0 local tStream = {tReq.data} + local c = #tStream while len < rest do local s = fread() if s then len = len + #s - table.insert(tStream, s) + c = c + 1 + tStream[c] = s else return -1 end diff --git a/source/tools/monitor/unity/beaver/guide/develop.md b/source/tools/monitor/unity/beaver/guide/develop.md index 4e78d9e3..11945179 100644 --- a/source/tools/monitor/unity/beaver/guide/develop.md +++ b/source/tools/monitor/unity/beaver/guide/develop.md @@ -59,6 +59,10 @@ config: outline: # 外部数据入口，适合接入外部数据场景 - /tmp/sysom # 外部unix socket 路径，可以指定多个 +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] # 控制lua 插件加载 + + plugins: # 插件列表 对应 /collector/plugin 路径下编译出来的c库文件。 - so: kmsg # 库名 description: "collect dmesg info." # 描述符 diff --git a/source/tools/monitor/unity/beaver/identity.lua b/source/tools/monitor/unity/beaver/identity.lua index 27582ac6..1c4e93e3 100644 --- a/source/tools/monitor/unity/beaver/identity.lua +++ b/source/tools/monitor/unity/beaver/identity.lua @@ -25,8 +25,8 @@ end local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/beeQ/proto_queue.lua b/source/tools/monitor/unity/beeQ/proto_queue.lua index f1988d89..ecece0f1 100644 --- a/source/tools/monitor/unity/beeQ/proto_queue.lua +++ b/source/tools/monitor/unity/beeQ/proto_queue.lua @@ -21,12 +21,14 @@ function CprotoQueue:que() end function CprotoQueue:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -34,12 +36,14 @@ function CprotoQueue:load_label(unity_line, line) end function CprotoQueue:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -56,7 +60,8 @@ function CprotoQueue:load_log(unity_line, line) end function CprotoQueue:_proc(unity_lines, lines) - for i=0, unity_lines.num - 1 do + local c = #lines["lines"] + for i = 0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), ls = {}, @@ -66,7 +71,8 @@ function CprotoQueue:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 1b278181..7e970dc8 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -7,19 +7,7 @@ require("common.class") local CprotoData = require("common.protoData") local procffi = require("collector.native.procffi") - -local CprocStat = require("collector.proc_stat") -local CprocMeminfo = require("collector.proc_meminfo") -local CprocVmstat = require("collector.proc_vmstat") -local CprocNetdev = require("collector.proc_netdev") -local CprocDiskstats = require("collector.proc_diskstats") -local CprocSockStat = require("collector.proc_sockstat") -local CprocSnmpStat = require("collector.proc_snmp_stat") -local CprocMounts = require("collector.proc_mounts") -local CprocStatm = require("collector.proc_statm") -local CprocBuddyinfo = require("collector.proc_buddyinfo") local Cplugin = require("collector.plugin") - local system = require("common.system") local Cloop = class("loop") @@ -27,21 +15,23 @@ local Cloop = class("loop") function Cloop:_init_(que, proto_q, fYaml) local res = system:parseYaml(fYaml) self._proto = CprotoData.new(que) - self._procs = { - CprocStat.new(self._proto, procffi, res.config.proc_path), - CprocMeminfo.new(self._proto, procffi, res.config.proc_path), - CprocVmstat.new(self._proto, procffi, res.config.proc_path), - CprocNetdev.new(self._proto, procffi, res.config.proc_path), - CprocDiskstats.new(self._proto, procffi, res.config.proc_path), - CprocSockStat.new(self._proto, procffi, res.config.proc_path), - CprocSnmpStat.new(self._proto, procffi, res.config.proc_path), - CprocMounts.new(self._proto, procffi, res.config.proc_path), - CprocStatm.new(self._proto, procffi, res.config.proc_path), - CprocBuddyinfo.new(self._proto, procffi, res.config.proc_path), - } + self:loadLuaPlugin(res, res.config.proc_path) self._plugin = Cplugin.new(self._proto, procffi, que, proto_q, fYaml) end +function Cloop:loadLuaPlugin(res, proc_path) + local luas = res.luaPlugins + + self._procs = {} + if res.luaPlugins then + for i, plugin in ipairs(luas) do + local CProcs = require("collector." .. plugin) + self._procs[i] = CProcs.new(self._proto, procffi, proc_path) + end + end + print("add " .. #self._procs .. " lua plugin.") +end + function Cloop:work(t) local lines = self._proto:protoTable() for k, obj in pairs(self._procs) do diff --git a/source/tools/monitor/unity/collector/outline/pipeMon.lua b/source/tools/monitor/unity/collector/outline/pipeMon.lua index 1298192d..81f20696 100644 --- a/source/tools/monitor/unity/collector/outline/pipeMon.lua +++ b/source/tools/monitor/unity/collector/outline/pipeMon.lua @@ -35,11 +35,11 @@ end function CpipeMon:setupPipe(fYaml) local res = system:parseYaml(fYaml) - for _, path in ipairs(res.outline) do + for i, path in ipairs(res.outline) do if unistd.access(path) then unistd.unlink(path) end - table.insert(self._paths, path) + self._paths[i] = path socket.unix = require("socket.unix") local s = socket.unix.udp() @@ -58,14 +58,20 @@ local function trans(title, ls, vs, log) local values = {} local logs = {} + local c = 0 for k, v in pairs(ls) do - table.insert(labels, {name=k, index=v}) + c = c + 1 + labels[c] = {name=k, index=v} end + c = 0 for k, v in pairs(vs) do - table.insert(values, {name=k, value=v}) + c = c + 1 + values[c] = {name=k, value=v} end + c = 0 for k, v in pairs(log) do - table.insert(logs, {name=k, log=v}) + c = c + 1 + logs[c] = {name=k, log=v} end return {line = title, ls = labels, vs = values, log = logs} end diff --git a/source/tools/monitor/unity/collector/plugin.lua b/source/tools/monitor/unity/collector/plugin.lua index 41b21746..4ccf946b 100644 --- a/source/tools/monitor/unity/collector/plugin.lua +++ b/source/tools/monitor/unity/collector/plugin.lua @@ -55,12 +55,14 @@ function Cplugin:setup(plugins, proto_q) end function Cplugin:load_label(unity_line, line) + local c = #line.ls for i=0, 4 - 1 do local name = self._ffi.string(unity_line.indexs[i].name) local index = self._ffi.string(unity_line.indexs[i].index) if #name > 0 then - table.insert(line.ls, {name = name, index = index}) + c = c + 1 + line.ls[c] = {name = name, index = index} else return end @@ -68,12 +70,14 @@ function Cplugin:load_label(unity_line, line) end function Cplugin:load_value(unity_line, line) + local c = #line.vs for i=0, 32 - 1 do local name = self._ffi.string(unity_line.values[i].name) local value = unity_line.values[i].value if #name > 0 then - table.insert(line.vs, {name = name, value = value}) + c = c + 1 + line.vs[c] = {name = name, value = value} else return end @@ -90,6 +94,7 @@ function Cplugin:load_log(unity_line, line) end function Cplugin:_proc(unity_lines, lines) + local c = #lines["lines"] for i=0, unity_lines.num - 1 do local unity_line = unity_lines.line[i] local line = {line = self._ffi.string(unity_line.table), @@ -100,7 +105,8 @@ function Cplugin:_proc(unity_lines, lines) self:load_label(unity_line, line) self:load_value(unity_line, line) self:load_log(unity_line, line) - table.insert(lines["lines"], line) + c = c + 1 + lines["lines"][c] = line end end diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index b682bbe3..ca6170ee 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -13,6 +13,9 @@ config: outline: - /tmp/sysom +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat"] + plugins: - so: kmsg description: "collect dmesg info." diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 4de16c32..6ad8644a 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -23,8 +23,10 @@ local function get_lines(fName) local fName = fName or "/proc/mounts" local f = assert(io.open(fName, "r")) + local c = 0 for line in f:lines() do - table.insert(lines, line) + c = c + 1 + lines[c] = line end return lines end diff --git a/source/tools/monitor/unity/collector/proc_netdev.lua b/source/tools/monitor/unity/collector/proc_netdev.lua index 7a2f0954..2682bfdb 100644 --- a/source/tools/monitor/unity/collector/proc_netdev.lua +++ b/source/tools/monitor/unity/collector/proc_netdev.lua @@ -27,7 +27,7 @@ function CprocNetdev:_getNewValue(ifName, data) local now = {} local index = self:_netdevIndex() for i = 1, #index do - table.insert(now, tonumber(data.value[i - 1])) + now[i] = tonumber(data.value[i - 1]) end self._lastData[ifName] = now self._lastIfNames[ifName] = 1 @@ -43,13 +43,13 @@ function CprocNetdev:_calcIf(ifName, data, res, elapsed) } for i, index in ipairs(index) do local nowValue = tonumber(data.value[i -1]) - table.insert(now, nowValue) + now[i] = nowValue local value = (nowValue - res[i]) / elapsed local cell = { name = index, value = value } - table.insert(protoTable.vs, cell) + protoTable.vs[i] = cell end self:appendLine(protoTable) diff --git a/source/tools/monitor/unity/collector/proc_snmp_stat.lua b/source/tools/monitor/unity/collector/proc_snmp_stat.lua index 199bdbea..917cd7d4 100644 --- a/source/tools/monitor/unity/collector/proc_snmp_stat.lua +++ b/source/tools/monitor/unity/collector/proc_snmp_stat.lua @@ -58,12 +58,14 @@ end function CprocSnmpStat:pack(labels, logs) local vs = {} + local c = 0 for k, v in pairs(labels) do local value = { name = k, value = tonumber(v) } - table.insert(vs, value) + c = c + 1 + vs[c] = value end self:appendLine(self:_packProto("pkt_status", nil, vs)) if #logs > 0 then @@ -82,6 +84,7 @@ function CprocSnmpStat:check(now) local labels = self:createLabels() local logs = {} if self._rec then + local c = 0 for k, v in pairs(now) do if self._rec[k] and self._rec[k] < v then -- local delta = v - self._rec[k] @@ -91,7 +94,8 @@ function CprocSnmpStat:check(now) labels[lk] = lv + delta end end - table.insert(logs, string.format("%s: %d", k, tonumber(delta))) + c = c + 1 + logs[c] = string.format("%s: %d", k, tonumber(delta)) end end end diff --git a/source/tools/monitor/unity/collector/proc_sockstat.lua b/source/tools/monitor/unity/collector/proc_sockstat.lua index d906ae97..bf7eb80d 100644 --- a/source/tools/monitor/unity/collector/proc_sockstat.lua +++ b/source/tools/monitor/unity/collector/proc_sockstat.lua @@ -17,6 +17,7 @@ end function CprocSockStat:proc(elapsed, lines) CvProc.proc(self) local vs = {} + local c = 0 for line in io.lines(self.pFile) do local cells = pystring:split(line, ":", 1) if #cells > 1 then @@ -31,7 +32,8 @@ function CprocSockStat:proc(elapsed, lines) name = title, value = tonumber(bodies[2 * i]) } - table.insert(vs, v) + c = c + 1 + vs[c] = v end end end diff --git a/source/tools/monitor/unity/collector/proc_statm.lua b/source/tools/monitor/unity/collector/proc_statm.lua index d3ebeb60..9a9fff79 100644 --- a/source/tools/monitor/unity/collector/proc_statm.lua +++ b/source/tools/monitor/unity/collector/proc_statm.lua @@ -16,6 +16,7 @@ end function CprocStatm:proc(elapsed, lines) CvProc.proc(self) local heads = {"size", "resident", "shared", "text", "lib", "data", "dt"} + local c = 0 for line in io.lines("/proc/self/statm") do local vs = {} local data = self._ffi.new("var_long_t") @@ -26,7 +27,8 @@ function CprocStatm:proc(elapsed, lines) name = k, value = tonumber(data.value[i - 1]), } - table.insert(vs, cell) + c = c + 1 + vs[c] = cell end self:appendLine(self:_packProto("self_statm", nil, vs)) end diff --git a/source/tools/monitor/unity/collector/vproc.lua b/source/tools/monitor/unity/collector/vproc.lua index 33c016a5..30d8fa3d 100644 --- a/source/tools/monitor/unity/collector/vproc.lua +++ b/source/tools/monitor/unity/collector/vproc.lua @@ -31,8 +31,10 @@ function CvProc:copyLine(line) end function CvProc:push(lines) + local c = #lines for _, v in ipairs(self._lines["lines"]) do - table.insert(lines["lines"], v) + c = c + 1 + lines["lines"][c] = v end self._lines = nil return lines diff --git a/source/tools/monitor/unity/common/lineParse.lua b/source/tools/monitor/unity/common/lineParse.lua index ee8de957..33d6486a 100644 --- a/source/tools/monitor/unity/common/lineParse.lua +++ b/source/tools/monitor/unity/common/lineParse.lua @@ -81,18 +81,23 @@ function module.pack(title, ls, vs) local line = title if system:keyCount(ls) > 0 then local lss = {} + local c = 0 for k, v in pairs(ls) do - table.insert(lss, k .. "=" .. v) + c = c + 1 + lss[c] = table.concat({k, v}, "=") end line = line .. ',' .. pystring:join(",", lss) end local vss = {} + local c = 0 for k, v in pairs(vs) do local tStr = type(v) if tStr == "number" then - table.insert(vss, k .. '=' .. tostring(v)) + c = c + 1 + vss[c] = table.concat({k, tostring(v)}, "=") elseif tStr == "string" then - table.insert(vss, k .. '=' .. json.encode(v)) + c = c + 1 + vss[c] = table.concat({k, json.encode(v)}, "=") else error("bad value type for " .. tStr) end diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index 25278335..ea9a08da 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -55,8 +55,10 @@ end local function setupDelimiter(delimiter) local rt = {} + local i = 0 for c in string.gmatch(delimiter, ".") do - table.insert(rt, checkDelimiter(c)) + i = i + 1 + rt[i] = checkDelimiter(c) end return table.concat(rt) end @@ -78,18 +80,22 @@ function pystring:split(s, delimiter, n) local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(s, delimiter, beg) if (iBeg) then - table.insert(result, string.sub(s, beg, iBeg - 1)) + c = c + 1 + result[c] = string.sub(s, beg, iBeg - 1) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end else - table.insert(result, string.sub(s, beg, string.len(s))) + c = c + 1 + result[c] = string.sub(s, beg, string.len(s)) break end end @@ -112,19 +118,23 @@ function pystring:rsplit(s, delimiter, n) rDel = setupDelimiter(rDel) local nums = 0 local beg = 1 + local c = 0 while (true) do local iBeg, iEnd = string.find(rs, rDel, beg) if (iBeg) then - table.insert(result, string.sub(s, len - (iBeg - 1),len - beg)) + c = c + 1 + result[c] = string.sub(s, len - (iBeg - 1),len - beg)) beg = iEnd + 1 nums = nums + 1 if nums >= n then - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end else - table.insert(result, string.sub(s, 1, len - beg)) + c = c + 1 + result[c] = string.sub(s, 1, len - beg) break end end diff --git a/source/tools/monitor/unity/httplib/httpComm.lua b/source/tools/monitor/unity/httplib/httpComm.lua index 12b691f3..3fdff02b 100644 --- a/source/tools/monitor/unity/httplib/httpComm.lua +++ b/source/tools/monitor/unity/httplib/httpComm.lua @@ -87,12 +87,15 @@ function ChttpComm:packHeaders(headTable, len) -- just for http out. end local origin = originHeader() + local c = 0 for k, v in pairs(origin) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end for k, v in pairs(headTable) do - table.insert(lines, k .. ": " .. v) + c = c + 1 + lines[c] = table.concat({k, v}, ": ") end return pystring:join("\r\n", lines) .. "\r\n" end diff --git a/source/tools/monitor/unity/test/curl/beaver/beavers.lua b/source/tools/monitor/unity/test/curl/beaver/beavers.lua index 619c4334..af296673 100644 --- a/source/tools/monitor/unity/test/curl/beaver/beavers.lua +++ b/source/tools/monitor/unity/test/curl/beaver/beavers.lua @@ -86,13 +86,15 @@ function Cbeavers:poll() if err then print("socket select return " .. err) end + local c = 0 for _, read in pairs(reads) do if type(read) == "number" then break elseif read == self._server then local s = read:accept() print("accept " .. s:getfd()) - table.insert(self._ss, s) + c = c + 1 + self._ss[c] = s local co = coroutine.create(function(o, s) self._proc(o, s) end) self._cos[s:getfd()] = co coroutine.resume(co, self, s) diff --git a/source/tools/monitor/unity/test/host/hostIp.lua b/source/tools/monitor/unity/test/host/hostIp.lua index 1b0266a6..84b4d4a0 100644 --- a/source/tools/monitor/unity/test/host/hostIp.lua +++ b/source/tools/monitor/unity/test/host/hostIp.lua @@ -9,8 +9,8 @@ local socket = require("socket") local function getAdd(hostName) local _, resolved = socket.dns.toip(hostName) local listTab = {} - for _, v in pairs(resolved.ip) do - table.insert(listTab, v) + for i, v in pairs(resolved.ip) do + listTab[i] = v end return listTab end diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index 04cb3142..ee816857 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -14,6 +14,13 @@ assert(#ret == 3) assert(ret[1] == "hello") assert(ret[2] == "lua") assert(ret[3] == "language") +ret = pystring:split("hello lua language lua language") +assert(#ret == 5 ) +assert(ret[1] == "hello") +assert(ret[2] == "lua") +assert(ret[3] == "language") +assert(ret[4] == "lua") +assert(ret[5] == "language") -- 自定符号分割 ret = pystring:split("hello*lua *language", "*") diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index d5efb6fd..8fab6339 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -240,6 +240,7 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday self:curMove(start) -- moveto position + local lenMs = #ms for line in self:loadData(stop) do local time = line.time for _, v in ipairs(line.lines) do @@ -261,37 +262,50 @@ function CfoxTSDB:query(start, stop, ms) -- start stop should at the same mday end tCell.values = values - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell end end return ms end +function CfoxTSDB:_qlast(date, beg, stop, ms) + if not self._man then -- check _man is already installed. + if self:_setupRead(beg) ~= 0 then -- try to create new + return ms + end + end + + if self.cffi.check_pman_date(self._man, date) == 1 then + return self:query(beg, stop, ms) + else + self:_del_() + if self:_setupRead(beg) ~= 0 then -- try to create new + return ms + end + return self:query(beg, stop, ms) + end +end + function CfoxTSDB:qlast(last, ms) assert(last < 24 * 60 * 60) local now = self:get_us() - local date = self:getDateFrom_us(now) local beg = now - last * 1e6; - if not self._man then -- check _man is already installed. - if self:_setupRead(now) ~= 0 then -- try to create new - return ms - end - end + local dStart = self:getDateFrom_us(beg) + local dStop = self:getDateFrom_us(now) - if self.cffi.check_pman_date(self._man, date) == 1 then -- at the same day - return self:query(beg, now, ms) + if self.cffi.check_foxdate(dStart, dStop) ~= 0 then + self:_qlast(dStart, beg, now, ms) else - local dStop = self:getDateFrom_us(now) - local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 local now2 = now - ms = self:query(beg1, now1, ms) - return self:query(beg2, now2, ms) + self:_qlast(dStart, beg1, now1, ms) + self:_qlast(dStop, beg2, now2, ms) end end @@ -307,6 +321,7 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) budget = budget or self._qBudget self:curMove(start) local inc = false + local lenMs = #ms for line in self:loadData(stop) do inc = false local time = line.time @@ -339,7 +354,8 @@ function CfoxTSDB:qDay(start, stop, ms, tbls, budget) end tCell.logs = logs - table.insert(ms, tCell) + lenMs = lenMs + 1 + ms[lenMs] = tCell inc = true end end @@ -364,11 +380,13 @@ function CfoxTSDB:qDayTables(start, stop, tbls) end self:curMove(start) + local lenTbls = #tbls for line in self:loadData(stop) do for _, v in ipairs(line.lines) do local title = v.line if not system:valueIsIn(tbls, title) then - table.insert(tbls, title) + lenTbls = lenTbls + 1 + tbls[lenTbls] = title end end end -- Gitee From 5692aec0fb00e0c02ffff441be38e0559a60d6a2 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 10:25:29 +0800 Subject: [PATCH 15/21] coolbpf: update Signed-off-by: Shuyi Cheng --- source/lib/internal/ebpf/coolbpf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lib/internal/ebpf/coolbpf b/source/lib/internal/ebpf/coolbpf index 0562b139..735ae613 160000 --- a/source/lib/internal/ebpf/coolbpf +++ b/source/lib/internal/ebpf/coolbpf @@ -1 +1 @@ -Subproject commit 0562b1397b8a8997b16d752d874dc5ad74149149 +Subproject commit 735ae6138430822502e1bd5ae2a366a8668ecd7e -- Gitee From 067b990ab7d893b2911d9172f5503a8d9641ff32 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 10:25:48 +0800 Subject: [PATCH 16/21] unity: malloc -> calloc Signed-off-by: Shuyi Cheng --- source/tools/monitor/unity/collector/plugin/bpf_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 6acf256f..745da01f 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -40,7 +40,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ -- Gitee From 0ac2d30fcb5ad533dcc12c4a7342095a4d5bc0c2 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Fri, 24 Feb 2023 14:47:45 +0800 Subject: [PATCH 17/21] unity: fix calloc Signed-off-by: Shuyi Cheng --- source/tools/monitor/unity/collector/plugin/bpf_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 745da01f..e9d76240 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -40,7 +40,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = calloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(1, sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ -- Gitee From fedf3e8b0abef08afcc728bb2cad77ee2c742209 Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Sun, 26 Feb 2023 01:41:23 +0800 Subject: [PATCH 18/21] add net retrans monitor. --- .../monitor/unity/beaver/localBeaver.lua | 4 + .../monitor/unity/collector/native/Makefile | 2 +- .../monitor/unity/collector/native/fastKsym.c | 166 ++++++++++ .../monitor/unity/collector/native/fastKsym.h | 20 ++ .../monitor/unity/collector/native/sig_stop.c | 19 +- .../tools/monitor/unity/collector/plugin.yaml | 25 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../monitor/unity/collector/plugin/bpf_head.h | 70 ++++- .../unity/collector/plugin/cpudist/Makefile | 8 + .../collector/plugin/cpudist/cpudist.bpf.c | 40 +++ .../unity/collector/plugin/cpudist/cpudist.c | 86 ++++++ .../unity/collector/plugin/cpudist/cpudist.h | 8 + .../collector/plugin/net_health/Makefile | 8 + .../plugin/net_health/net_health.bpf.c | 29 ++ .../collector/plugin/net_health/net_health.c | 108 +++++++ .../collector/plugin/net_health/net_health.h | 8 + .../collector/plugin/net_retrans/Makefile | 8 + .../plugin/net_retrans/net_retrans.bpf.c | 283 ++++++++++++++++++ .../plugin/net_retrans/net_retrans.c | 215 +++++++++++++ .../plugin/net_retrans/net_retrans.h | 48 +++ .../unity/collector/plugin/plugin_head.h | 1 + .../tools/monitor/unity/common/pystring.lua | 5 +- .../tools/monitor/unity/httplib/httpBase.lua | 15 +- source/tools/monitor/unity/test/string/py.lua | 2 + source/tools/monitor/unity/tsdb/foxTSDB.lua | 6 +- 25 files changed, 1169 insertions(+), 17 deletions(-) create mode 100644 source/tools/monitor/unity/collector/native/fastKsym.c create mode 100644 source/tools/monitor/unity/collector/native/fastKsym.h create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c create mode 100644 source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_health/net_health.h create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c create mode 100644 source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h diff --git a/source/tools/monitor/unity/beaver/localBeaver.lua b/source/tools/monitor/unity/beaver/localBeaver.lua index 1f253c64..cb062006 100644 --- a/source/tools/monitor/unity/beaver/localBeaver.lua +++ b/source/tools/monitor/unity/beaver/localBeaver.lua @@ -176,12 +176,16 @@ function CLocalBeaver:write(fd, stream) stream = string.sub(stream, sent + 1) sent, err, errno = socket.send(fd, stream) if sent == nil then + if errno == 11 then -- EAGAIN ? + goto continue + end system:posixError("socket send error.", err, errno) return nil end else -- need to read ? may something error or closed. return nil end + ::continue:: end res = self._cffi.mod_fd(self._efd, fd, 0) -- epoll read ev only assert(res == 0) diff --git a/source/tools/monitor/unity/collector/native/Makefile b/source/tools/monitor/unity/collector/native/Makefile index 03d3d2f8..d185cf7f 100644 --- a/source/tools/monitor/unity/collector/native/Makefile +++ b/source/tools/monitor/unity/collector/native/Makefile @@ -1,7 +1,7 @@ CC := gcc CFLAG := -g -fpic LDFLAG := -g -fpic -shared -OBJS := procffi.o sig_stop.o unity_interface.o +OBJS := procffi.o sig_stop.o unity_interface.o fastKsym.o SO := libprocffi.so all: $(SO) diff --git a/source/tools/monitor/unity/collector/native/fastKsym.c b/source/tools/monitor/unity/collector/native/fastKsym.c new file mode 100644 index 00000000..ea0e2db8 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.c @@ -0,0 +1,166 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#include "fastKsym.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int tfd = 0; +static int sym_cnt = 0; +static struct ksym_cell * gCell = NULL; + +static int load_ksyms(int fd, int stack_only) { + int ret = 0; + int count = 0; + struct ksym_cell cell; + void * addr; + char buf[128]; + + FILE *pf = fopen("/proc/kallsyms", "r"); + + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open /proc/kallsyms failed, errno, %d, %s", errno, strerror(errno)); + goto endOpen; + } + + while (!feof(pf)) { + if (!fgets(buf, sizeof(buf), pf)) + break; + + ret = sscanf(buf, "%p %c %64s %31s", &addr, &cell.type, cell.func, cell.module); + if (ret == 3) { + cell.module[0] = '\0'; + } else if (ret < 3) { + fprintf(stderr, "bad kallsyms line: %s", buf); + goto endRead; + } + + if (!addr) + continue; + + if (stack_only && (cell.type != 't') && (cell.type != 'T')) { + continue; + } + cell.addr = (addr_t) addr; + + ret = write(fd, &cell, sizeof (cell)); + if (ret < 0) { + fprintf(stderr, "write file failed, errno, %d, %s", errno, strerror(errno)); + goto endWrite; + } + count ++; + } + + fclose(pf); + return count; + + endWrite: + endRead: + fclose(pf); + endOpen: + return ret; +} + +static int sym_cmp(const void *p1, const void *p2) +{ + return ((struct ksym_cell *)p1)->addr > ((struct ksym_cell *)p2)->addr; +} + +static int sort_ksym(int fd, int count) { + int ret = 0 ; + struct stat sb; + void *pmmap; + + ret = fstat(fd, &sb); + if (ret < 0) { + fprintf(stderr, "fstat file failed, errno, %d, %s", errno, strerror(errno)); + goto endStat; + } + + pmmap = mmap(NULL, sb.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (pmmap == NULL) { + fprintf(stderr, "mmap file failed, errno, %d, %s", errno, strerror(errno)); + ret = -EACCES; + goto endMmap; + } + + qsort(pmmap, count, sizeof (struct ksym_cell), sym_cmp); + + gCell = (struct ksym_cell*)pmmap; + + return ret; + endMmap: + endStat: + return ret; +} + +int ksym_setup(int stack_only) { + int ret; + + FILE *pf = tmpfile(); + if (pf == NULL) { + ret = -errno; + fprintf(stderr, "open file failed, errno, %d, %s", errno, strerror(errno)); + goto endTmpfile; + } + + tfd = fileno(pf); + + ret = load_ksyms(tfd, stack_only); + if (ret < 0) { + goto endLoad; + } + sym_cnt = ret; + + ret = sort_ksym(tfd, ret); + if (ret < 0) { + goto endSort; + } + + return ret; + endSort: + endLoad: + close(tfd); + endTmpfile: + return ret; +} + +struct ksym_cell* ksym_search(addr_t key) { + int start = 0, end = sym_cnt; + int mid; + + if (sym_cnt <= 0) { + printf("sym_cnt: %d", sym_cnt); + return NULL; + } + + while (start < end) { + mid = start + (end - start) / 2; + + if (key < gCell[mid].addr) { + end = mid; + } else if (key > gCell[mid].addr) { + start = mid + 1; + } else { + return &gCell[mid]; + } + } + + if (start > 0) { + if ((gCell[start - 1].addr < key) && (key < gCell[start].addr)) { + return &gCell[start - 1]; + } + } + if (start == sym_cnt) { + return &gCell[end - 1]; + } + return NULL; +} diff --git a/source/tools/monitor/unity/collector/native/fastKsym.h b/source/tools/monitor/unity/collector/native/fastKsym.h new file mode 100644 index 00000000..57bf8be2 --- /dev/null +++ b/source/tools/monitor/unity/collector/native/fastKsym.h @@ -0,0 +1,20 @@ +// +// Created by 廖肇燕 on 2022/12/18. +// + +#ifndef FASTKSYM_FASTKSYM_H +#define FASTKSYM_FASTKSYM_H + +typedef unsigned long addr_t; + +struct ksym_cell { + addr_t addr; + char func[64]; + char module[31]; + char type; +}; + +int ksym_setup(int stack_only); +struct ksym_cell* ksym_search(addr_t key); + +#endif //FASTKSYM_FASTKSYM_H diff --git a/source/tools/monitor/unity/collector/native/sig_stop.c b/source/tools/monitor/unity/collector/native/sig_stop.c index ec3d865b..c006072d 100644 --- a/source/tools/monitor/unity/collector/native/sig_stop.c +++ b/source/tools/monitor/unity/collector/native/sig_stop.c @@ -4,8 +4,10 @@ #include "sig_stop.h" #include - #include +#include +#include +#include "fastKsym.h" static volatile int working = 1; @@ -38,7 +40,22 @@ static void sig_register(void) { sigaction(SIGQUIT, &action, NULL); } +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + void plugin_init(void) { + bump_memlock_rlimit1(); + ksym_setup(1); sig_register(); working = 1; } diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index ca6170ee..96bc4339 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -38,6 +38,10 @@ plugins: - so: unity_nosched description: "nosched:sys hold cpu and didn't scheduling" + - so: net_health + description: "tcp net health." + - so: net_retrans + description: "tcp retrans monitor." metrics: - title: sysak_proc_cpu_total @@ -150,4 +154,23 @@ metrics: head: value help: "nosched:sys hold cpu and didn't scheduling" type: "gauge" - + - title: sysak_cpu_dist + from: cpu_dist + head: value + help: "task cpu sched dist." + type: "gauge" + - title: sysak_net_health_hist + from: net_health_hist + head: value + help: "net_health_hist" + type: "gauge" + - title: sysak_net_health_count + from: net_health_count + head: value + help: "net_health_count" + type: "gauge" + - title: sysak_net_retrans_count + from: net_retrans_count + head: value + help: "net_retrans_count" + type: "gauge" diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index b29bc4e8..af6ab29b 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/bpf_head.h b/source/tools/monitor/unity/collector/plugin/bpf_head.h index 6acf256f..99adf37f 100644 --- a/source/tools/monitor/unity/collector/plugin/bpf_head.h +++ b/source/tools/monitor/unity/collector/plugin/bpf_head.h @@ -4,15 +4,22 @@ #ifndef UNITY_BPF_HEAD_H #define UNITY_BPF_HEAD_H +#include -#define DEFINE_SEKL_OBJECT(skel_name) \ - struct skel_name##_bpf *skel_name = NULL; \ - static pthread_t perf_thread = 0; +#ifdef COOLBPF_PERF_THREAD -#define DESTORY_SKEL_BOJECT(skel_name) \ - if (perf_thread > 0) \ - kill_perf_thread(perf_thread); \ - skel_name##_bpf__destroy(skel_name) +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } #define LOAD_SKEL_OBJECT(skel_name, perf) \ ( \ @@ -40,7 +47,7 @@ DESTORY_SKEL_BOJECT(skel_name); \ goto load_bpf_skel_out; \ } \ - struct perf_thread_arguments *perf_args = malloc(sizeof(struct perf_thread_arguments)); \ + struct perf_thread_arguments *perf_args = calloc(1, sizeof(struct perf_thread_arguments)); \ if (!perf_args) \ { \ __ret = -ENOMEM; \ @@ -57,4 +64,51 @@ __ret; \ }) +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); +#else +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; + +#define LOAD_SKEL_OBJECT(skel_name, perf) \ + ( \ + { \ + __label__ load_bpf_skel_out; \ + int __ret = 0; \ + skel_name = skel_name##_bpf__open(); \ + if (!skel_name) \ + { \ + printf("failed to open BPF object\n"); \ + __ret = -1; \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__load(skel_name); \ + if (__ret) \ + { \ + printf("failed to load BPF object: %d\n", __ret); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + __ret = skel_name##_bpf__attach(skel_name); \ + if (__ret) \ + { \ + printf("failed to attach BPF programs: %s\n", strerror(-__ret)); \ + DESTORY_SKEL_BOJECT(skel_name); \ + goto load_bpf_skel_out; \ + } \ + load_bpf_skel_out: \ + __ret; \ + }) + +#define DESTORY_SKEL_BOJECT(skel_name) \ + skel_name##_bpf__destroy(skel_name); +#endif + +#define coobpf_map_find(OBJ, NAME) bpf_object__find_map_fd_by_name(OBJ, NAME) +#define coobpf_key_next(FD, KEY, NEXT) bpf_map_get_next_key(FD, KEY, NEXT) +#define coobpf_key_value(FD, KEY, VALUE) bpf_map_lookup_elem(FD, KEY, VALUE) + +#include "plugin_head.h" #endif //UNITY_BPF_HEAD_H diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/Makefile b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile new file mode 100644 index 00000000..9fc693eb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := cpudist.bpf.c +csrcs := cpudist.c +so := libcpudist.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c new file mode 100644 index 00000000..1dc01053 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.bpf.c @@ -0,0 +1,40 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#include +#include + +BPF_ARRAY(cpudist, u64, 20); +BPF_HASH(start, u32, u64, 128 * 1024); + +struct sched_switch_args { + u16 type; + u8 flag; + u8 preeempt; + u32 c_pid; + char prev_comm[16]; + u32 prev_pid; + u32 prev_prio; + u64 prev_state; + char next_comm[16]; + u32 next_pid; + u32 next_prio; +}; +SEC("tracepoint/sched/sched_switch") +int sched_switch_hook(struct sched_switch_args *args){ + u64 ts = ns(); + u64 *pv; + u32 prev = args->prev_pid; + u32 next = args->next_pid; + + if (next > 0) { + bpf_map_update_elem(&start, &next, &ts, BPF_ANY); + } + pv = bpf_map_lookup_elem(&start, &prev); + if (pv && ts > *pv) { + hist10_push((struct bpf_map_def *)&cpudist, (ts - *pv) / 1000); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c new file mode 100644 index 00000000..5ea28bac --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.c @@ -0,0 +1,86 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + + +#include "cpudist.h" +#include "../bpf_head.h" +#include "cpudist.skel.h" + +#define CPU_DIST_INDEX 8 +#define DIST_ARRAY_SIZE 20 +DEFINE_SEKL_OBJECT(cpudist); +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("cpudist plugin install.\n"); + ret = LOAD_SKEL_OBJECT(cpudist, perf); + dist_fd = coobpf_map_find(cpudist->obj, "cpudist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < CPU_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[CPU_DIST_INDEX]; + const char *names[] = {"us1", "us10", "us100", "ms1", "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "cpu_dist"); + + cal_dist(values); + for (i = 0; i < CPU_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("cpudist plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(cpudist); +} + diff --git a/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h new file mode 100644 index 00000000..1bcac9a4 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/cpudist/cpudist.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/23. +// + +#ifndef UNITY_CPUDIST_H +#define UNITY_CPUDIST_H + +#endif //UNITY_CPUDIST_H diff --git a/source/tools/monitor/unity/collector/plugin/net_health/Makefile b/source/tools/monitor/unity/collector/plugin/net_health/Makefile new file mode 100644 index 00000000..55801edc --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_health.bpf.c +csrcs := net_health.c +so := libnet_health.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c new file mode 100644 index 00000000..5efee8bb --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -0,0 +1,29 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// +#include +#include + +BPF_ARRAY(outCnt, u64, 2); +BPF_ARRAY(netHist, u64, 20); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +SEC("kprobe/tcp_validate_incoming") +int j_tcp_validate_incoming(struct pt_regs *ctx) { + struct tcp_sock *tp = (struct tcp_sock *)PT_REGS_PARM1(ctx); + u64 ts = BPF_CORE_READ(tp, srtt_us) >> 3; + u64 ms = ts / 1000; + if (ms > 0) { + addCnt(0, ms); + addCnt(1, 1); + hist10_push((struct bpf_map_def *)&netHist, ms); + } + return 0; +} + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c new file mode 100644 index 00000000..ea63d234 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -0,0 +1,108 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_health.h" +#include "../bpf_head.h" +#include "net_health.skel.h" + +#define NET_DIST_INDEX 4 +#define DIST_ARRAY_SIZE 20 + +DEFINE_SEKL_OBJECT(net_health); +static int cnt_fd = 0; +static int dist_fd = 0; + +int init(void *arg) +{ + int ret; + printf("net_health plugin install.\n"); + ret = LOAD_SKEL_OBJECT(net_health, perf); + cnt_fd = coobpf_map_find(net_health->obj, "outCnt"); + dist_fd = coobpf_map_find(net_health->obj, "netHist"); + return ret; +} + +static int get_dist(unsigned long *locals) { + int i = 0; + unsigned long value = 0; + int key, key_next; + + key = 0; + while (coobpf_key_next(dist_fd, &key, &key_next) == 0) { + coobpf_key_value(dist_fd, &key_next, &value); + locals[i ++] = value; + if (i > DIST_ARRAY_SIZE) { + break; + } + key = key_next; + } + return i; +} + +static int cal_dist(unsigned long* values) { + int i, j; + int size; + static unsigned long rec[DIST_ARRAY_SIZE] = {0}; + unsigned long locals[DIST_ARRAY_SIZE]; + + size = get_dist(locals); + for (i = 0; i < NET_DIST_INDEX - 1; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + j = i; + values[j] = 0; + for (; i < size; i ++) { + values[j] += locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +static int get_count(unsigned long* values) { + int key; + static unsigned long rec[2]; + unsigned long now[2]; + + key = 0; + coobpf_key_value(cnt_fd, &key, &now[0]); + key = 1; + coobpf_key_value(cnt_fd, &key, &now[1]); + + values[0] = now[0] - rec[0]; rec[0] = now[0]; + values[1] = now[1] - rec[1]; rec[1] = now[1]; + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + int i; + unsigned long values[NET_DIST_INDEX]; + const char *names[] = { "ms10", "ms100", "s1", "so"}; + struct unity_line* line; + + unity_alloc_lines(lines, 2); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_health_hist"); + + cal_dist(values); + for (i = 0; i < NET_DIST_INDEX; i ++ ) { + unity_set_value(line, i, names[i], values[i]); + } + + get_count(values); + line = unity_get_line(lines, 1); + unity_set_table(line, "net_health_count"); + unity_set_value(line, 0, "sum", values[0]); + unity_set_value(line, 1, "count", values[1]); + return 0; +} + +void deinit(void) +{ + printf("net_health plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_health); +} + + diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.h b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h new file mode 100644 index 00000000..dd3cebc2 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.h @@ -0,0 +1,8 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_HEALTH_H +#define UNITY_NET_HEALTH_H + +#endif //UNITY_NET_HEALTH_H diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile new file mode 100644 index 00000000..09638c6e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := net_retrans.bpf.c +csrcs := net_retrans.c +so := libnet_retrans.so + +include ../bpfso.mk \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c new file mode 100644 index 00000000..70769cba --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.bpf.c @@ -0,0 +1,283 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include +#include +#include "net_retrans.h" + +struct liphdr { + __u8 ver_hdl; + __u8 tos; + __be16 tot_len; + __be16 id; + __be16 frag_off; + __u8 ttl; + __u8 protocol; + __sum16 check; + __be32 saddr; + __be32 daddr; +}; + +#define MAX_ENTRY 128 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +BPF_PERF_OUTPUT(perf, 1024); +BPF_STACK_TRACE(stack, MAX_ENTRY); +BPF_ARRAY(outCnt, u64, NET_RETRANS_TYPE_MAX); + +static inline void addCnt(int k, u64 val) { + u64 *pv = bpf_map_lookup_elem(&outCnt, &k); + if (pv) { + __sync_fetch_and_add(pv, val); + } +} + +static inline int get_tcp_info(struct data_t* pdata, struct tcp_sock *ts) +{ + pdata->rcv_nxt = BPF_CORE_READ(ts, rcv_nxt); + pdata->rcv_wup = BPF_CORE_READ(ts, rcv_wup); + pdata->snd_nxt = BPF_CORE_READ(ts, snd_nxt); + pdata->snd_una = BPF_CORE_READ(ts, snd_una); + pdata->copied_seq = BPF_CORE_READ(ts, copied_seq); + pdata->snd_wnd = BPF_CORE_READ(ts, snd_wnd); + pdata->rcv_wnd = BPF_CORE_READ(ts, rcv_wnd); + + pdata->lost_out = BPF_CORE_READ(ts, lost_out); + pdata->packets_out = BPF_CORE_READ(ts, packets_out); + pdata->retrans_out = BPF_CORE_READ(ts, retrans_out); + pdata->sacked_out = BPF_CORE_READ(ts, sacked_out); + pdata->reordering = BPF_CORE_READ(ts, reordering); + return 0; +} + +static inline int get_skb_info(struct data_t* pdata, struct sk_buff *skb, u32 type) +{ + u16 offset; + u8 ihl; + void* head; + struct liphdr *piph; + struct tcphdr *ptcph; + + addCnt(type, 1); + pdata->type = type; + pdata->sk_state = 0; + + head = (void*)BPF_CORE_READ(skb, head); + offset = BPF_CORE_READ(skb, network_header); + piph = (struct liphdr *)(head + offset); + ihl = _(piph->ver_hdl) & 0x0f; + ptcph = (struct tcphdr *)((void *)piph + ihl * 4); + + pdata->ip_dst = _(piph->daddr); + pdata->dport = BPF_CORE_READ(ptcph, dest); + pdata->ip_src = _(piph->saddr); + pdata->sport = BPF_CORE_READ(ptcph, source); + return 0; +} + +static inline void get_list_task(struct list_head* phead, struct data_t* e) { + struct list_head *next = BPF_CORE_READ(phead, next); + if (next) { + wait_queue_entry_t *entry = container_of(next, wait_queue_entry_t, entry); + struct poll_wqueues *pwq = (struct poll_wqueues *)BPF_CORE_READ(entry, private); + if (pwq) + { + struct task_struct* tsk = (struct task_struct*)BPF_CORE_READ(pwq, polling_task); + if (tsk) { + e->pid = BPF_CORE_READ(tsk, pid); + bpf_probe_read(&e->comm[0], TASK_COMM_LEN, &tsk->comm[0]); + } + } + } +} + +static inline void get_sock_task(struct sock *sk, struct data_t* e) { + struct socket_wq *wq = BPF_CORE_READ(sk, sk_wq); + if (wq) { + struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); + get_list_task(phead, e); + } +} + +static inline void get_task(struct data_t* pdata, struct sock *sk) { + pdata->pid = 0; + pdata->comm[0] = '\0'; + + get_sock_task(sk, pdata); +} + +static inline void get_socket_task(struct data_t* e, struct sock *sk) { +// struct socket* psocket; +// struct socket_wq *wq; + + e->pid = 0; + e->comm[0] = '\0'; + +// psocket = BPF_CORE_READ(sk, sk_socket); +// wq = BPF_CORE_READ(psocket, wq); +// if (wq) { +// struct list_head* phead = (struct list_head*)((char *)wq + offsetof(struct socket_wq, wait.head)); +// get_list_task(phead, e); +// } +} + +static inline int get_info(struct data_t* pdata, struct sock *sk, u32 type) +{ + struct inet_sock *inet = (struct inet_sock *)sk; + + addCnt(type, 1); + pdata->type = type; + pdata->ip_dst = BPF_CORE_READ(sk, __sk_common.skc_daddr); + pdata->dport = BPF_CORE_READ(sk, __sk_common.skc_dport); + pdata->ip_src = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); + pdata->sport = BPF_CORE_READ(inet, inet_sport); + pdata->sk_state = BPF_CORE_READ(sk, __sk_common.skc_state); + return 0; +} + +static inline int check_inner(unsigned int ip) +{ + int i; + const unsigned int array[3][2] = { + {0x0000000A, 0x000000ff}, + {0x000010AC, 0x0000f0ff}, + {0x0000A8C0, 0x0000ffff}, + }; + + if (ip == 0) { + return 1; + } +#pragma unroll 3 + for (i =0; i < 3; i ++) { + if ((ip & array[i][1]) == array[i][0]) { + return 1; + } + } + return 0; +} + +static inline int check_ip(struct data_t* pdata) { + return check_inner(pdata->ip_src) && check_inner(pdata->ip_dst); +} + +SEC("kprobe/tcp_enter_loss") +int j_tcp_enter_loss(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat != 1) { + return 0; + } + get_task(&data, sk); + get_info(&data, sk, NET_RETRANS_TYPE_RTO); + data.stack_id = 0; + get_tcp_info(&data, (struct tcp_sock *)sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_probe0") +int j_tcp_send_probe0(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + u32 stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == 0) { + return 0; + } + + get_info(&data, sk, NET_RETRANS_TYPE_ZERO); + data.stack_id = 0; + get_task(&data, sk); + get_tcp_info(&data, (struct tcp_sock *)sk); + + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} + +SEC("kprobe/tcp_v4_send_reset") +int j_tcp_v4_send_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + if (sk == NULL) { + struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM2(ctx); + get_skb_info(&data, skb, NET_RETRANS_TYPE_RST); + get_task(&data, NULL); + data.stack_id = 0; + } + else { + get_info(&data, sk, NET_RETRANS_TYPE_RST_SK); + get_task(&data, sk); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + } + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_send_active_reset") +int j_tcp_send_active_reset(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_RST_ACTIVE); + data.stack_id = bpf_get_stackid(ctx, &stack, KERN_STACKID_FLAGS); + + get_task(&data, sk); + if (check_ip(&data)) { + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +#define TCP_SYN_SENT 2 +#define TCPF_SYN_SENT (1 << TCP_SYN_SENT) +SEC("kprobe/tcp_retransmit_skb") +int j_tcp_retransmit_skb(struct pt_regs *ctx){ + struct sock *sk; + unsigned char stat; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + + stat = BPF_CORE_READ(sk, __sk_common.skc_state); + if (stat == TCP_SYN_SENT) + { + struct data_t data = {}; + + get_info(&data, sk, NET_RETRANS_TYPE_SYN); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + } + return 0; +} + +SEC("kprobe/tcp_rtx_synack") +int j_tcp_rtx_synack(struct pt_regs *ctx) +{ + struct sock *sk; + struct data_t data = {}; + + sk = (struct sock *)PT_REGS_PARM1(ctx); + get_info(&data, sk, NET_RETRANS_TYPE_SYN_ACK); + get_task(&data, sk); + bpf_perf_event_output(ctx, &perf, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c new file mode 100644 index 00000000..2f5a1969 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.c @@ -0,0 +1,215 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#include "net_retrans.h" +#define COOLBPF_PERF_THREAD +#include "../bpf_head.h" +#include "net_retrans.skel.h" + +#include +#include + +#include +#include +#include + +static volatile int budget = 0; // for log budget +static int cnt_fd = 0; +static int stack_fd = 0; + +const char *net_title[] = {"rto_retrans", "zero_probe", \ + "noport_reset", "bad_sync", \ + "net_proc", "syn_send", "syn_ack"}; + +int proc(int stack_fd, struct data_t *e, struct unity_line *line); +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + int ret; + if (budget > 0) { + struct data_t *e = (struct data_t *)data; + struct beeQ *q = (struct beeQ *)ctx; + struct unity_line *line; + struct unity_lines *lines = unity_new_lines(); + + printf("receive: %d msg.\n", data_sz); + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + ret = proc(stack_fd, e, line); + if (ret >= 0) { + beeQ_send(q, lines); + } + budget --; + } +} + +DEFINE_SEKL_OBJECT(net_retrans); +int init(void *arg) +{ + int ret; + printf("net_retrans plugin install.\n"); + + ret = LOAD_SKEL_OBJECT(net_retrans, perf); + cnt_fd = coobpf_map_find(net_retrans->obj, "outCnt"); + stack_fd = coobpf_map_find(net_retrans->obj, "stack"); + return ret; +} + +static int get_count(unsigned long *locals) { + int i = 0; + + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + coobpf_key_value(cnt_fd, &i, &locals[i]); + } + return i; +} + +static int cal_retrans(unsigned long *values) { + int i; + static unsigned long rec[NET_RETRANS_TYPE_MAX] = {0}; + unsigned long locals[NET_RETRANS_TYPE_MAX]; + + get_count(locals); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + values[i] = locals[i] - rec[i]; + rec[i] = locals[i]; + } + return 0; +} + +int call(int t, struct unity_lines *lines) { + int i; + unsigned long values[NET_RETRANS_TYPE_MAX]; + struct unity_line* line; + + budget = t; //release log budget + + unity_alloc_lines(lines, 1); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "net_retrans_count"); + + cal_retrans(values); + for (i = 0; i < NET_RETRANS_TYPE_MAX; i ++) { + unity_set_value(line, i, net_title[i], values[i]); + } + + return 0; +} + +void deinit(void) +{ + printf("net_retrans plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(net_retrans); +} + +#define LOG_MAX 256 +static char log[LOG_MAX]; + +static char * transIP(unsigned long lip) { + struct in_addr addr; + memcpy(&addr, &lip, sizeof(lip)); + return inet_ntoa(addr); +} + +static const char * resetSock(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_v4_rcv") == 0) { + if (e->sk_state == 12) { + return "bad_ack"; // TCP_NEW_SYN_REC + } else { + return "tw_rst"; + } + } else if (strcmp(cell->func, "tcp_check_req") == 0) { + return "bad_syn"; + } else if (strcmp(cell->func, "tcp_v4_do_rcv") == 0) { + return "tcp_stat"; + } else { + return "unknown_sock"; + } + } + } + return "failure_sock"; +} + +static const char * resetActive(int stack_fd, struct data_t *e){ + unsigned long addr; + int i = 1; //last stack + struct ksym_cell* cell; + + coobpf_key_value(stack_fd, &i, &addr); + if (addr) { + cell = ksym_search(addr); + if (cell) { + if (strcmp(cell->func, "tcp_out_of_resources") == 0) { + return "tcp_oom"; + } else if (strcmp(cell->func, "tcp_keepalive_timer") == 0) { + return "keep_alive"; + } else if (strcmp(cell->func, "inet_release") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_close") == 0) { + return "bad_close"; + } else if (strcmp(cell->func, "tcp_disconnect") == 0) { + return "tcp_abort"; + } else if (strcmp(cell->func, "tcp_abort") == 0) { + return "tcp_abort"; + } else { + return "unknown_active"; + } + } + } + return "failure_active"; +} + +int proc(int stack_fd, struct data_t *e, struct unity_line *line) { + snprintf(log, LOG_MAX, "task:%d|%s, tcp:%s:%d->%s:%d, state:%d, ", e->pid, e->comm, \ + transIP(e->ip_src), htons(e->sport), \ + transIP(e->ip_src), htons(e->sport), \ + e->sk_state); + switch (e->type) { + case NET_RETRANS_TYPE_RTO: + case NET_RETRANS_TYPE_ZERO: + case NET_RETRANS_TYPE_SYN: + case NET_RETRANS_TYPE_SYN_ACK: + { + char buf[LOG_MAX]; + snprintf(buf, LOG_MAX, "rcv_nxt:%d, rcv_wup:%d, snd_nxt:%d, snd_una:%d, copied_seq:%d, " + "snd_wnd:%d, rcv_wnd:%d, lost_out:%d, packets_out:%d, retrans_out:%d, " + "sacked_out:%d, reordering:%d", + e->rcv_nxt, e->rcv_wup, e->snd_nxt, e->snd_una, e->copied_seq, + e->snd_wnd, e->rcv_wnd, e->lost_out, e->packets_out, e->retrans_out, + e->sacked_out, e->reordering + ); + strncat(log, buf, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST: + strncat(log, "noport", LOG_MAX); + break; + case NET_RETRANS_TYPE_RST_SK: + { + const char *type = resetSock(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + case NET_RETRANS_TYPE_RST_ACTIVE: + { + const char *type = resetActive(stack_fd, e); + strncat(log, type, LOG_MAX); + } + break; + default: + break; + } + unity_set_table(line, "net_retrans_log"); + unity_set_index(line, 0, "type", net_title[e->type]); + unity_set_log(line, "log", log); + return 0; +} diff --git a/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h new file mode 100644 index 00000000..5d953924 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/net_retrans/net_retrans.h @@ -0,0 +1,48 @@ +// +// Created by 廖肇燕 on 2023/2/24. +// + +#ifndef UNITY_NET_RETRANS_H +#define UNITY_NET_RETRANS_H + +#define TASK_COMM_LEN 16 + +enum { + NET_RETRANS_TYPE_RTO, + NET_RETRANS_TYPE_ZERO, + NET_RETRANS_TYPE_RST, + NET_RETRANS_TYPE_RST_SK, + NET_RETRANS_TYPE_RST_ACTIVE, + NET_RETRANS_TYPE_SYN, + NET_RETRANS_TYPE_SYN_ACK, + NET_RETRANS_TYPE_MAX, +}; + + +struct data_t { + char comm[TASK_COMM_LEN]; + unsigned int pid; + unsigned int type; + unsigned int ip_src; + unsigned int ip_dst; + unsigned short sport; + unsigned short dport; + unsigned short sk_state; + unsigned short stack_id; + + unsigned int rcv_nxt; + unsigned int rcv_wup; + unsigned int snd_nxt; + unsigned int snd_una; + unsigned int copied_seq; + unsigned int snd_wnd; + unsigned int rcv_wnd; + + unsigned int lost_out; + unsigned int packets_out; + unsigned int retrans_out; + unsigned int sacked_out; + unsigned int reordering; +}; + +#endif //UNITY_NET_RETRANS_H diff --git a/source/tools/monitor/unity/collector/plugin/plugin_head.h b/source/tools/monitor/unity/collector/plugin/plugin_head.h index 202a110f..ee00783d 100644 --- a/source/tools/monitor/unity/collector/plugin/plugin_head.h +++ b/source/tools/monitor/unity/collector/plugin/plugin_head.h @@ -39,6 +39,7 @@ struct unity_lines { #include "../../beeQ/beeQ.h" #include "../native/sig_stop.h" #include "../native/unity_interface.h" +#include "../native/fastKsym.h" inline struct unity_lines *unity_new_lines(void) __attribute__((always_inline)); inline int unity_alloc_lines(struct unity_lines * lines, unsigned int num) __attribute__((always_inline)); diff --git a/source/tools/monitor/unity/common/pystring.lua b/source/tools/monitor/unity/common/pystring.lua index ea9a08da..c38a0d1f 100644 --- a/source/tools/monitor/unity/common/pystring.lua +++ b/source/tools/monitor/unity/common/pystring.lua @@ -75,6 +75,9 @@ end function pystring:split(s, delimiter, n) local result = {} + if not delimiter or delimiter == "" then -- for blank, gsub multi blank to single + s = string.gsub(s, "%s+", " ") + end local delimiter = setupDelimiter(delimiter or " ") local n = n or 2 ^ 63 - 1 @@ -124,7 +127,7 @@ function pystring:rsplit(s, delimiter, n) local iBeg, iEnd = string.find(rs, rDel, beg) if (iBeg) then c = c + 1 - result[c] = string.sub(s, len - (iBeg - 1),len - beg)) + result[c] = string.sub(s, len - (iBeg - 1),len - beg) beg = iEnd + 1 nums = nums + 1 if nums >= n then diff --git a/source/tools/monitor/unity/httplib/httpBase.lua b/source/tools/monitor/unity/httplib/httpBase.lua index 432b622e..3ec1ff4f 100644 --- a/source/tools/monitor/unity/httplib/httpBase.lua +++ b/source/tools/monitor/unity/httplib/httpBase.lua @@ -31,10 +31,19 @@ end local function checkKeep(tReq) local conn = tReq.header["connection"] - if conn and string.lower(conn) == "close" then - return false + if tReq.vers == "1.0" then + if conn and string.lower(conn) == "keep-alive" then + return true + else -- for http 1.0, close as default + return false + end + else + if conn and string.lower(conn) == "close" then + return false + else -- for http 1.1 and newer, for keep-alive as default + return true + end end - return true end function ChttpBase:call(tReq) diff --git a/source/tools/monitor/unity/test/string/py.lua b/source/tools/monitor/unity/test/string/py.lua index ee816857..8befc8ea 100644 --- a/source/tools/monitor/unity/test/string/py.lua +++ b/source/tools/monitor/unity/test/string/py.lua @@ -21,6 +21,8 @@ assert(ret[2] == "lua") assert(ret[3] == "language") assert(ret[4] == "lua") assert(ret[5] == "language") +ret = pystring:split("Node 0, zone DMA 1 0 0 1 2 1 1 0 1 1 3") +assert(#ret == 15) -- 自定符号分割 ret = pystring:split("hello*lua *language", "*") diff --git a/source/tools/monitor/unity/tsdb/foxTSDB.lua b/source/tools/monitor/unity/tsdb/foxTSDB.lua index 8fab6339..807e1ea1 100644 --- a/source/tools/monitor/unity/tsdb/foxTSDB.lua +++ b/source/tools/monitor/unity/tsdb/foxTSDB.lua @@ -291,7 +291,7 @@ function CfoxTSDB:qlast(last, ms) assert(last < 24 * 60 * 60) local now = self:get_us() - local beg = now - last * 1e6; + local beg = now - last * 1e6 local dStart = self:getDateFrom_us(beg) local dStop = self:getDateFrom_us(now) @@ -299,6 +299,7 @@ function CfoxTSDB:qlast(last, ms) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:_qlast(dStart, beg, now, ms) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -405,6 +406,7 @@ function CfoxTSDB:qDate(dStart, dStop, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -433,6 +435,7 @@ function CfoxTSDB:qNow(sec, tbls) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDay(beg, now, ms, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 @@ -461,6 +464,7 @@ function CfoxTSDB:qTabelNow(sec) if self.cffi.check_foxdate(dStart, dStop) ~= 0 then self:qDayTables(beg, now, tbls) else + dStop.hour, dStop.min, dStop.sec = 0, 0, 0 local beg1 = beg local beg2 = self.cffi.make_stamp(dStop) local now1 = beg2 - 1 -- Gitee From 833f3eeb34b5632b368b1aafd26b2feafd9ebe58 Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Sun, 26 Feb 2023 13:43:06 +0800 Subject: [PATCH 19/21] unity: netlink: add netlink to monitor packet drop count Signed-off-by: Shuyi Cheng --- .../monitor/unity/collector/plugin/Makefile | 2 +- .../unity/collector/plugin/netlink/Makefile | 19 ++++ .../unity/collector/plugin/netlink/netlink.c | 90 +++++++++++++++++++ .../unity/collector/plugin/netlink/netlink.h | 11 +++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/netlink.c create mode 100644 source/tools/monitor/unity/collector/plugin/netlink/netlink.h diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index af6ab29b..c2a64f2b 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans netlink all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/netlink/Makefile b/source/tools/monitor/unity/collector/plugin/netlink/Makefile new file mode 100644 index 00000000..cfe0e649 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/Makefile @@ -0,0 +1,19 @@ +CC := gcc +CFLAG := -g -fpic +LDFLAG := -g -fpic -shared +OBJS := netlink.o +SO := libnetlink.so + +all: $(SO) install + +%.o: %.c + $(CC) -c $< -o $@ $(CFLAG) + +$(SO): $(OBJS) + $(CC) -o $@ $(OBJS) $(LDFLAG) + +install: $(SO) + cp $(SO) ../../native/ + +clean: + rm -f $(SO) $(OBJS) \ No newline at end of file diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c new file mode 100644 index 00000000..350bec23 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c @@ -0,0 +1,90 @@ +#include "netlink.h" +#include +#include +#include + +static char buffer[4096]; + +int get_conntrack_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("conntrack -S", "r"); // 将命令ls-l 同过管道读到fp + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, " drop=")) != NULL) + { + buf += strlen(" drop="); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + +int get_tc_drop() +{ + int total_drop = 0, i; + FILE *fp = NULL; + fp = popen("tc -s qdisc", "r"); // 将命令ls-l 同过管道读到fp + + if (!fp) + return -1; + + while (fgets(buffer, 4096, fp) != NULL) + { + char *buf = buffer; + while ((buf = strstr(buf, "dropped ")) != NULL) + { + buf += strlen("dropped "); + for (i = 0;; i++) + { + if (buf[i] > '9' || buf[i] < '0') + { + buf[i] = 0; + break; + } + } + total_drop += atoi(buf); + buf += i + 1; + } + } + pclose(fp); + return total_drop; +} + + +int init(void * arg) { + printf("netlink plugin install\n"); + return 0; +} + +int call(int t, struct unity_lines* lines) { + struct unity_line* line; + + unity_alloc_lines(lines, 3); // 预分配好 + line = unity_get_line(lines, 0); + unity_set_table(line, "netlink"); + unity_set_value(line, 0, "conntrack_drop", get_conntrack_drop()); + unity_set_value(line, 1, "tc_drop", get_tc_drop()); + + return 0; +} + +void deinit(void) { + printf("netlink plugin uninstall\n"); +} diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.h b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h new file mode 100644 index 00000000..4a55af11 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.h @@ -0,0 +1,11 @@ +#ifndef __NET_LINK_H +#define __NET_LINK_H + +#include "../plugin_head.h" + +int init(void * arg); +int call(int t, struct unity_lines* lines); +void deinit(void); + + +#endif \ No newline at end of file -- Gitee From 54af888a5849428feaa8c06fba0015e55e2b317d Mon Sep 17 00:00:00 2001 From: Shuyi Cheng Date: Sun, 26 Feb 2023 14:00:37 +0800 Subject: [PATCH 20/21] unity: netlink: remove comment Signed-off-by: Shuyi Cheng --- .../tools/monitor/unity/collector/plugin/netlink/netlink.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c index 350bec23..4bae1997 100644 --- a/source/tools/monitor/unity/collector/plugin/netlink/netlink.c +++ b/source/tools/monitor/unity/collector/plugin/netlink/netlink.c @@ -9,7 +9,7 @@ int get_conntrack_drop() { int total_drop = 0, i; FILE *fp = NULL; - fp = popen("conntrack -S", "r"); // 将命令ls-l 同过管道读到fp + fp = popen("conntrack -S", "r"); if (!fp) return -1; @@ -40,7 +40,7 @@ int get_tc_drop() { int total_drop = 0, i; FILE *fp = NULL; - fp = popen("tc -s qdisc", "r"); // 将命令ls-l 同过管道读到fp + fp = popen("tc -s qdisc", "r"); if (!fp) return -1; @@ -76,7 +76,7 @@ int init(void * arg) { int call(int t, struct unity_lines* lines) { struct unity_line* line; - unity_alloc_lines(lines, 3); // 预分配好 + unity_alloc_lines(lines, 1); line = unity_get_line(lines, 0); unity_set_table(line, "netlink"); unity_set_value(line, 0, "conntrack_drop", get_conntrack_drop()); -- Gitee From ed8be4e7ca459e805938d979c48065ce84fc4266 Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Sun, 26 Feb 2023 07:18:16 +0000 Subject: [PATCH 21/21] unity: Add irqoff check Signed-off-by: Hailong Liu --- .../tools/monitor/unity/collector/plugin.yaml | 5 +- .../monitor/unity/collector/plugin/Makefile | 2 +- .../collector/plugin/unity_irqoff/Makefile | 8 + .../plugin/unity_irqoff/unity_irqoff.bpf.c | 151 ++++++++++++ .../plugin/unity_irqoff/unity_irqoff.c | 233 ++++++++++++++++++ .../plugin/unity_irqoff/unity_irqoff.h | 48 ++++ 6 files changed, 445 insertions(+), 2 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c create mode 100644 source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h diff --git a/source/tools/monitor/unity/collector/plugin.yaml b/source/tools/monitor/unity/collector/plugin.yaml index 96bc4339..3c5ae4a1 100644 --- a/source/tools/monitor/unity/collector/plugin.yaml +++ b/source/tools/monitor/unity/collector/plugin.yaml @@ -42,6 +42,9 @@ plugins: description: "tcp net health." - so: net_retrans description: "tcp retrans monitor." + - + so: unity_irqoff + description: "irqoff:detect irq turned off and can't response" metrics: - title: sysak_proc_cpu_total @@ -152,7 +155,7 @@ metrics: - title: sched_moni_jitter from: sched_moni_jitter head: value - help: "nosched:sys hold cpu and didn't scheduling" + help: "nosched/irqoff:sys and irqoff hold cpu and didn't scheduling" type: "gauge" - title: sysak_cpu_dist from: cpu_dist diff --git a/source/tools/monitor/unity/collector/plugin/Makefile b/source/tools/monitor/unity/collector/plugin/Makefile index c2a64f2b..cec50dc7 100644 --- a/source/tools/monitor/unity/collector/plugin/Makefile +++ b/source/tools/monitor/unity/collector/plugin/Makefile @@ -4,7 +4,7 @@ LDFLAG := -g -fpic -shared OBJS := proto_sender.o LIB := libproto_sender.a -DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched cpudist net_health net_retrans netlink +DEPMOD=sample threads kmsg proc_schedstat proc_loadavg bpfsample2 unity_nosched unity_irqoff cpudist net_health net_retrans netlink all: $(LIB) $(DEPMOD) diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile new file mode 100644 index 00000000..9e86c359 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/Makefile @@ -0,0 +1,8 @@ + +newdirs := $(shell find ./ -type d) + +bpfsrcs := unity_irqoff.bpf.c +csrcs := unity_irqoff.c +so := libunity_irqoff.so + +include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c new file mode 100644 index 00000000..136445aa --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.bpf.c @@ -0,0 +1,151 @@ +#include +#include +#include "sched_jit.h" +#include "unity_irqoff.h" + +#define PERF_MAX_STACK_DEPTH 127 +#define MAX_ENTRIES 10240 +#define BPF_F_FAST_STACK_CMP (1ULL << 9) +#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP) + +BPF_PERF_OUTPUT(perf, 1024); + +struct bpf_map_def SEC("maps") stackmap = { + .type = BPF_MAP_TYPE_STACK_TRACE, + .key_size = sizeof(u32), + .value_size = PERF_MAX_STACK_DEPTH * sizeof(u64), + .max_entries = 10000, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct arg_info); +} arg_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct tm_info); +} tm_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u64); + __type(value, struct info); +} info_map SEC(".maps"); + +#define _(P) ({typeof(P) val = 0; bpf_probe_read(&val, sizeof(val), &P); val;}) + +static inline u64 get_thresh(void) +{ + u64 thresh, i = 0; + struct arg_info *argp; + + argp = bpf_map_lookup_elem(&arg_map, &i); + if (argp) + thresh = argp->thresh; + else + thresh = -1; + + return thresh; +} + +SEC("perf_event") +int hw_irqoff_event(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.stamp = now; + event.delay = delta; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } + + return 0; +} + +SEC("perf_event") +int sw_irqoff_event1(struct bpf_perf_event_data *ctx) +{ + int ret, i = 0; + struct tm_info *tmifp, tm; + + tmifp = bpf_map_lookup_elem(&tm_map, &i); + if (tmifp) { + tmifp->last_stamp = bpf_ktime_get_ns(); + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = bpf_ktime_get_ns(); + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + return 0; +} + +SEC("perf_event") +int sw_irqoff_event2(struct bpf_perf_event_data *ctx) +{ + int i = 0; + u64 now, delta, thresh, stamp; + struct tm_info *tmifp, tm; + struct event event = {}; + u32 cpu = bpf_get_smp_processor_id(); + + now = bpf_ktime_get_ns(); + tmifp = bpf_map_lookup_elem(&tm_map, &i); + + if (tmifp) { + stamp = tmifp->last_stamp; + tmifp->last_stamp = now; + thresh = get_thresh(); + if (stamp && (thresh != -1)) { + delta = now - stamp; + if (delta > thresh) { + event.cpu = cpu; + event.delay = delta; + event.stamp = now; + event.pid = bpf_get_current_pid_tgid(); + bpf_get_current_comm(&event.comm, sizeof(event.comm)); + event.ret = bpf_get_stackid(ctx, &stackmap, KERN_STACKID_FLAGS); + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, + &event, sizeof(event)); + } + } + } else { + __builtin_memset(&tm, 0, sizeof(tm)); + tm.last_stamp = now; + bpf_map_update_elem(&tm_map, &i, &tm, 0); + } + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c new file mode 100644 index 00000000..81f89ea9 --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.c @@ -0,0 +1,233 @@ +#include +#include +#include +#include +//#include +//#include +//#include +#include +#include +#include +#include +#include +#include +#include "unity_irqoff.h" +#include "sched_jit.h" +#include "unity_irqoff.skel.h" +#include "../../../../unity/beeQ/beeQ.h" + +struct env { + __u64 sample_period; + __u64 threshold; +} env = { + .threshold = 50*1000*1000, /* 10ms */ +}; + +static int nr_cpus; +struct sched_jit_summary summary, *percpu_summary; +struct bpf_link **sw_mlinks, **hw_mlinks= NULL; + +DEFINE_SEKL_OBJECT(unity_irqoff); + +static int +open_and_attach_perf_event(struct perf_event_attr *attr, + struct bpf_program *prog, + struct bpf_link *links[]) +{ + int i, fd; + + for (i = 0; i < nr_cpus; i++) { + fd = syscall(__NR_perf_event_open, attr, -1, i, -1, 0); + if (fd < 0) { + /* Ignore CPU that is offline */ + if (errno == ENODEV) + continue; + fprintf(stderr, "failed to init perf sampling: %s\n", + strerror(errno)); + return -1; + } + links[i] = bpf_program__attach_perf_event(prog, fd); + if (!links[i]) { + fprintf(stderr, "failed to attach perf event on cpu: %d\n", i); + close(fd); + return -1; + } + } + return 0; +} + +/* surprise! return 0 if failed! */ +static int attach_prog_to_perf(struct unity_irqoff_bpf *obj) +{ + int ret = 0; + + struct perf_event_attr attr_hw = { + .type = PERF_TYPE_HARDWARE, + .freq = 0, + .sample_period = env.sample_period*2, /* refer to watchdog_update_hrtimer_threshold() */ + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + + struct perf_event_attr attr_sw = { + .type = PERF_TYPE_SOFTWARE, + .freq = 0, + .sample_period = env.sample_period, + .config = PERF_COUNT_SW_CPU_CLOCK, + }; + + if (!open_and_attach_perf_event(&attr_hw, obj->progs.hw_irqoff_event, hw_mlinks)) { + ret = 1<progs.sw_irqoff_event1, sw_mlinks)) + ret = ret | 1<progs.sw_irqoff_event2, sw_mlinks)) + ret = 1<num++; + summary->total += e->delay; + + if (e->delay < 10) { + summary->less10ms++; + } else if (e->delay < 50) { + summary->less50ms++; + } else if (e->delay < 100) { + summary->less100ms++; + } else if (e->delay < 500) { + summary->less500ms++; + } else if (e->delay < 1000) { + summary->less1s++; + } else { + summary->plus1s++; + } +} + +void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) +{ + struct event e; + e = *((struct event *)data); + e.delay = e.delay/(1000*1000); + if (e.cpu > nr_cpus - 1) + return; + update_summary(&summary, &e); +} + +static int irqoff_handler(void *arg, struct unity_irqoff_bpf *unity_irqoff) +{ + int arg_key = 0, err = 0; + struct arg_info arg_info = {}; + int arg_fd; + + /*memset(summary, 0, sizeof(struct sched_jit_summary)); */ + struct perf_thread_arguments *perf_args = + calloc(sizeof(struct perf_thread_arguments), 1); + if (!perf_args) { + printf("Failed to malloc perf_thread_arguments\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); + return -ENOMEM; + } + perf_args->mapfd = bpf_map__fd(unity_irqoff->maps.events); + perf_args->sample_cb = handle_event; + perf_args->lost_cb = handle_lost_events; + perf_args->ctx = arg; + perf_thread = beeQ_send_thread(arg, perf_args, thread_worker); + + arg_fd = bpf_map__fd(unity_irqoff->maps.arg_map); + arg_info.thresh = env.threshold; + env.sample_period = env.threshold*2/5; + err = bpf_map_update_elem(arg_fd, &arg_key, &arg_info, 0); + if (err) { + fprintf(stderr, "Failed to update arg_map\n"); + return err; + } + + if (!(err = attach_prog_to_perf(unity_irqoff))) + return err; + return 0; +} + +static void bump_memlock_rlimit1(void) +{ + struct rlimit rlim_new = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, + }; + + if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) { + fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n"); + exit(1); + } +} + +int init(void *arg) +{ + int err; + + nr_cpus = libbpf_num_possible_cpus(); + if (nr_cpus < 0) { + fprintf(stderr, "failed to get # of possible cpus: '%s'!\n", + strerror(-nr_cpus)); + return nr_cpus; + } + + bump_memlock_rlimit1(); + + sw_mlinks = calloc(nr_cpus, sizeof(*sw_mlinks)); + if (!sw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc sw_mlinks or rlinks\n"); + return err; + } + + hw_mlinks = calloc(nr_cpus, sizeof(*hw_mlinks)); + if (!hw_mlinks) { + err = errno; + fprintf(stderr, "failed to alloc hw_mlinks or rlinks\n"); + free(sw_mlinks); + return err; + } + + unity_irqoff = unity_irqoff_bpf__open_and_load(); + if (!unity_irqoff) { + err = errno; + fprintf(stderr, "failed to open and/or load BPF object\n"); + return err; + } + + irqoff_handler(arg, unity_irqoff); + + return 0; +} + +int call(int t, struct unity_lines *lines) +{ + struct unity_line *line; + + unity_alloc_lines(lines, 1); + line = unity_get_line(lines, 0); + unity_set_table(line, "sched_moni_jitter"); + unity_set_index(line, 0, "mod", "irqoff"); + unity_set_value(line, 0, "dltnum", summary.num); + unity_set_value(line, 1, "dlttm", summary.total); + unity_set_value(line, 2, "lt10ms", summary.less10ms); + unity_set_value(line, 3, "lt50ms", summary.less50ms); + unity_set_value(line, 4, "lt100ms", summary.less100ms); + unity_set_value(line, 5, "lt500ms", summary.less500ms); + unity_set_value(line, 6, "lt1s", summary.less1s); + unity_set_value(line, 7, "mts", summary.plus1s); + return 0; +} + +void deinit(void) +{ + free(sw_mlinks); + free(hw_mlinks); + printf("unity_irqoff plugin uninstall.\n"); + DESTORY_SKEL_BOJECT(unity_irqoff); +} diff --git a/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h new file mode 100644 index 00000000..1b024f9e --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_irqoff/unity_irqoff.h @@ -0,0 +1,48 @@ +#ifndef __IRQOFF_H +#define __IRQOFF_H + +#define TASK_COMM_LEN 16 +#define CPU_ARRY_LEN 4 +#define CONID_LEN 13 + +struct info { + __u64 prev_counter; +}; + +struct tm_info { + __u64 last_stamp; +}; + +struct arg_info { + __u64 thresh; +}; + +#ifndef __VMLINUX_H__ + +#include "../plugin_head.h" + +#define DEFINE_SEKL_OBJECT(skel_name) \ + struct skel_name##_bpf *skel_name = NULL; \ + static pthread_t perf_thread = 0; \ + int thread_worker(struct beeQ *q, void *arg) \ + { \ + perf_thread_worker(arg); \ + return 0; \ + } \ + void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) \ + { \ + printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); \ + } + +#define DESTORY_SKEL_BOJECT(skel_name) \ + if (perf_thread > 0) \ + kill_perf_thread(perf_thread); \ + skel_name##_bpf__destroy(skel_name); + +int init(void *arg); +int call(int t, struct unity_lines *lines); +void deinit(void); + +#endif +#endif /* __IRQOFF_H */ + -- Gitee