From c59cc204ccfdba7c3c12b8c59771c84ffdc39fff Mon Sep 17 00:00:00 2001 From: liaozhaoyan Date: Wed, 26 Apr 2023 18:29:32 +0800 Subject: [PATCH] add more lib for run, fix alinux3 bugs. --- source/tools/monitor/unity/beeQ/pack.sh | 12 +++ source/tools/monitor/unity/beeQ/run.sh | 5 ++ .../monitor/unity/collector/btfLoader.lua | 89 +++++++++++++++++++ source/tools/monitor/unity/collector/loop.lua | 13 +-- .../plugin/net_health/net_health.bpf.c | 3 +- .../collector/plugin/net_health/net_health.c | 2 +- .../tools/monitor/unity/test/curl/getBtf.lua | 11 +++ source/tools/monitor/unity/yamls/group.yaml | 73 +++++++++++++++ 8 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 source/tools/monitor/unity/collector/btfLoader.lua create mode 100644 source/tools/monitor/unity/test/curl/getBtf.lua create mode 100644 source/tools/monitor/unity/yamls/group.yaml diff --git a/source/tools/monitor/unity/beeQ/pack.sh b/source/tools/monitor/unity/beeQ/pack.sh index 42eea828..5f05e910 100755 --- a/source/tools/monitor/unity/beeQ/pack.sh +++ b/source/tools/monitor/unity/beeQ/pack.sh @@ -12,6 +12,16 @@ mkdir ${DIST}/install cp /usr/local/lib/libyaml-0.so* ${DIST}/install/ cp /usr/local/lib/libluajit-5.1.so* ${DIST}/install/ cp /usr/local/lib/libyaml.so* ${DIST}/install/ +cp /usr/lib64/libssl.so* ${DIST}/install/ +cp /usr/lib64/libcrypto.so* ${DIST}/install/ +cp /usr/lib64/libgssapi_krb5.so* ${DIST}/install/ +cp /usr/lib64/libkrb5.so* ${DIST}/install/ +cp /usr/lib64/libcom_err.so* ${DIST}/install/ +cp /usr/lib64/libk5crypto.so* ${DIST}/install/ +cp /usr/lib64/libkrb5support.so* ${DIST}/install/ +cp /usr/lib64/libkeyutils.so* ${DIST}/install/ +cp /usr/lib64/libresolv.so* ${DIST}/install/ +cp /usr/lib64/libpcre.so* ${DIST}/install/ mkdir ${DIST}/lib cp -r /usr/local/lib/lua/5.1/* ${DIST}/lib/ @@ -45,6 +55,7 @@ mkdir ${APP}/collector/guard mkdir ${APP}/collector/outline mkdir ${APP}/collector/postPlugin mkdir ${APP}/collector/postEngine +mkdir ${APP}/collector/execEngine mkdir ${APP}/collector/io cp collector/native/*.so* ${APP}/collector/native/ cp collector/native/*.lua ${APP}/collector/native/ @@ -53,6 +64,7 @@ cp collector/guard/*.lua ${APP}/collector/guard cp collector/outline/*.lua ${APP}/collector/outline cp collector/postPlugin/*.lua ${APP}/collector/postPlugin cp collector/postEngine/*.lua ${APP}/collector/postEngine +cp collector/execEngine/*.lua ${APP}/collector/execEngine cp collector/io/*.lua ${APP}/collector/io cp collector/plugin.yaml ${APP}/collector/ diff --git a/source/tools/monitor/unity/beeQ/run.sh b/source/tools/monitor/unity/beeQ/run.sh index 23bf4702..18b67ff5 100755 --- a/source/tools/monitor/unity/beeQ/run.sh +++ b/source/tools/monitor/unity/beeQ/run.sh @@ -1,5 +1,10 @@ #!/bin/bash +DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd); +echo $DIR +cd $DIR +ulimit -c unlimited + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:./lib/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../tsdb/native/ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../collector/native/ diff --git a/source/tools/monitor/unity/collector/btfLoader.lua b/source/tools/monitor/unity/collector/btfLoader.lua new file mode 100644 index 00000000..500be2ad --- /dev/null +++ b/source/tools/monitor/unity/collector/btfLoader.lua @@ -0,0 +1,89 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/24 22:09 +--- +require("common.class") + +local unistd = require("posix.unistd") +local posix = require("posix") +local utsname = require("posix.sys.utsname") +local ChttpCli = require("httplib.httpCli") + +local CbtfLoader = class("btfLoader") + +local function regionId() + local url = "http://100.100.100.200/latest/meta-data/region-id" + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + return res.body +end + +local function checkBtf(path) + if unistd.access("/sys/kernel/btf/vmlinux") then + return false + end + if unistd.access(path) then + return false + end + return true +end + +local function checkKo(path) + if unistd.access(path) then + return false + end + return true +end + +local function downBtf(path, region, machine, release) + local url = "https://sysom-".. region ..".oss-".. region .."-internal.aliyuncs.com/home/hive/btf/".. machine .."/vmlinux-" .. release + if not unistd.access("/boot") then + local res, err, errno = posix.mkdir("/boot") + assert(res ~= 0, err .. errno) + end + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + + local file = io.open(path,"wb") + file:write(res.body) + file:close() +end + +local function downKo(path, name, region, machine, release) + local url = "https://sysom-".. region ..".oss-".. region .."-internal.aliyuncs.com/home/hive/sysak/modules/".. machine .."/sysak-" .. release .. ".ko" + if not unistd.access(path) then + local res, err, errno = posix.mkdir(path) + print(res) + assert(res ~= 0, err .. errno) + end + local cli = ChttpCli.new() + local res = cli:get(url) + assert(#res.body > 0) + + print(path..name) + local file = io.open(path .. name,"wb") + file:write(res.body) + file:close() +end + +function CbtfLoader:_init_(root) + local distro = utsname.uname() + if distro then + local release, machine = distro.release, distro.machine + local path = '/boot/vmlinux-' .. release + local region = regionId() + if checkBtf(path) then + downBtf(path, region, machine, release) + end + local ko = root .. "/lib/" .. release .. "/sysak.ko" + if checkKo(ko) then + local path = root .. "/lib/" .. release .. "/" + ko = downKo(path, "sysak.ko", region, machine, release) + end + end +end + +return CbtfLoader diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 933c87b8..8197b3b5 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -8,6 +8,7 @@ require("common.class") local CprotoData = require("common.protoData") local procffi = require("collector.native.procffi") local system = require("common.system") +local CbtfLoader = require("collector.btfLoader") local CpluginManager = require("collector.pluginManager") local calcJiffies = require("collector.guard.calcJiffies") local CguardSched = require("collector.guard.guardSched") @@ -19,9 +20,9 @@ local CforkRun = require("collector.execEngine.forkRun") local Cloop = class("loop") function Cloop:_init_(que, proto_q, fYaml, tid) + CbtfLoader.new("../../../..") -- setup btf local res = system:parseYaml(fYaml) self._daemon = CguardDaemon.new(res) - self._proto = CprotoData.new(que) self._tid = tid self:loadLuaPlugin(res, res.config.proc_path) @@ -54,10 +55,12 @@ end function Cloop:forkRun(res) local runs = res.forkRun local c = system:keyCount(self._procs) - for _, run in ipairs(runs) do - c = c + 1 - self._procs[c] = CforkRun.new(run, self._proto, procffi) - self._names[c] = run.cmd + if runs then + for _, run in ipairs(runs) do + c = c + 1 + self._procs[c] = CforkRun.new(run, self._proto, procffi) + self._names[c] = run.cmd + end end end diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c index 32aacfb7..fedd6b09 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.bpf.c @@ -4,7 +4,7 @@ #include #include -BPF_HASH(outCnt, int, u64, 2); +BPF_ARRAY(outCnt, u64, 2); BPF_ARRAY(netHist, u64, 20); SEC("kprobe/tcp_validate_incoming") @@ -19,4 +19,3 @@ int j_tcp_validate_incoming(struct pt_regs *ctx) { } return 0; } - diff --git a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c index de4a5309..4e77dc4c 100644 --- a/source/tools/monitor/unity/collector/plugin/net_health/net_health.c +++ b/source/tools/monitor/unity/collector/plugin/net_health/net_health.c @@ -71,7 +71,7 @@ static int cal_dist(unsigned long* values) { static int get_count(unsigned long* values) { int key; static unsigned long rec[2]; - unsigned long now[2]; + unsigned long now[2] = {0, 0}; key = 0; coobpf_key_value(cnt_fd, &key, &now[0]); diff --git a/source/tools/monitor/unity/test/curl/getBtf.lua b/source/tools/monitor/unity/test/curl/getBtf.lua new file mode 100644 index 00000000..8e0afd01 --- /dev/null +++ b/source/tools/monitor/unity/test/curl/getBtf.lua @@ -0,0 +1,11 @@ +--- +--- Generated by EmmyLua(https://github.com/EmmyLua) +--- Created by liaozhaoyan. +--- DateTime: 2023/4/26 14:13 +--- + +package.path = package.path .. ";../../?.lua;" + +local CbtfLoader = require("collector.btfLoader") + +local btf = CbtfLoader.new(".") diff --git a/source/tools/monitor/unity/yamls/group.yaml b/source/tools/monitor/unity/yamls/group.yaml new file mode 100644 index 00000000..ec7217b8 --- /dev/null +++ b/source/tools/monitor/unity/yamls/group.yaml @@ -0,0 +1,73 @@ +config: + freq: 20 # unit second + daemon: true + port: 8400 # bind port + bind_addr: 127.0.0.1 # bind ip + backlog: 32 # listen backlog + identity: # support hostip, curl(need url arg), hostname, file(need path arg), specify(need name arg), env(need name arg) + mode: hostip + proc_path: / # in container mode, like -v /:/mnt/host , should use /mnt/host/ + db: + rotate: 7 # tsdb file retention time, unit day + budget: 200 # max query buffer from tsdb. + limit: + cpu: 30 # unit % + mem: 50 # unit mb + tasks: 10 # monitor 10 pid max. + +outline: + - /var/sysom/outline + +pushTo: + to: "Influx" + host: "ld-8vb0s2ih252f53pv4-proxy-tsdb.lindorm.rds.aliyuncs.com" + port: 8242 + url: "/api/v2/write?db=sysom" + +luaPlugins: ["proc_buddyinfo", "proc_diskstats", "proc_meminfo", "proc_mounts", "proc_netdev", + "proc_snmp_stat", "proc_sockstat", "proc_stat", "proc_statm", "proc_vmstat", + "proc_uptime"] + +plugins: + - so: kmsg + description: "collect dmesg info." + - so: net_health + description: "tcp net health." + - so: net_retrans + description: "tcp retrans monitor." + - so: virtout + description: "virt status out put." + - so: sum_retrans + description: "summary retrans out put." + +metrics: + - title: sysak_proc_pkt_status + from: pkt_status + head: counter + help: "net status info from /proc/net/snmp and /proc/net/status." + type: "gauge" + - title: sysak_net_health_hist + from: net_health_hist + head: value + help: "net_health_hist" + type: "gauge" + - title: sysak_net_health_count + from: net_health_count + head: value + help: "net_health_count" + type: "gauge" + - title: sysak_net_retrans_count + from: net_retrans_count + head: value + help: "net_retrans_count" + type: "gauge" + - title: sysak_virtout_dist + from: virtout_dist + head: value + help: "sysak_virtout_dist" + type: "gauge" + - title: sysak_retrans + from: retrans + head: value + help: "sysak_retrans" + type: "gauge" -- Gitee