From 22fcbe9ecf9fc997f599ae98efbadeaa5e54edf7 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 13:45:03 +0800 Subject: [PATCH 1/6] collector: set container collecotr to podMan Signed-off-by: yinbinbin --- source/tools/monitor/unity/collector/loop.lua | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/tools/monitor/unity/collector/loop.lua b/source/tools/monitor/unity/collector/loop.lua index 9156c8a5..47e793de 100644 --- a/source/tools/monitor/unity/collector/loop.lua +++ b/source/tools/monitor/unity/collector/loop.lua @@ -16,8 +16,8 @@ local CguardDaemon = require("collector.guard.guardDaemon") local CguardSelfStat = require("collector.guard.guardSelfStat") local CpostPlugin = require("collector.postPlugin.postPlugin") local CforkRun = require("collector.execEngine.forkRun") -local CpodFilter = require("collector.podMan.podFilter") - +---local CpodFilter = require("collector.podMan.podFilter") +local CpodsAll = require("collector.podMan.podsAll") local Cloop = class("loop") function Cloop:_init_(que, proto_q, fYaml, tid) @@ -51,11 +51,12 @@ function Cloop:loadLuaPlugin(res, proc_path, procffi) c = c + 1 end end - --self._procs[c] = CpodsAll.new(res, self._proto, procffi, proc_path) - --self._names[c] = "podMon" if res.container then - self._procs[c] = CpodFilter.new(res, self._proto, procffi, proc_path) - self._names[c] = "podFilter" + ---self._procs[c] = CpodFilter.new(res, self._proto, procffi, proc_path) + ---self._names[c] = "podFilter" + self._procs[c] = CpodsAll.new(res, self._proto, procffi, proc_path) + self._names[c] = "podMon" + end print("add " .. system:keyCount(self._procs) .. " lua plugin.") end -- Gitee From 83f101984ab2e8569c4b34b8c41b14f3a6cd36bc Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 14:40:55 +0800 Subject: [PATCH 2/6] podMan: use 10250 to get pods & rescan pod Signed-off-by: yinbinbin --- .../unity/collector/podMan/podsAll.lua | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/source/tools/monitor/unity/collector/podMan/podsAll.lua b/source/tools/monitor/unity/collector/podMan/podsAll.lua index 16e0919a..248a1a77 100644 --- a/source/tools/monitor/unity/collector/podMan/podsAll.lua +++ b/source/tools/monitor/unity/collector/podMan/podsAll.lua @@ -11,6 +11,7 @@ local system = require("common.system") local pystring = require("common.pystring") local Cinotifies = require("common.inotifies") local unistd = require("posix.unistd") +local json = require("cjson") local CpodsAll = class("podsApi") @@ -72,12 +73,22 @@ local function setupCons(res) local cli = ChttpCli.new() local cons = {} local c = 0 - + local blacklist = {["arms-prom"] = 1, ["kube-system"] = 1, ["kube-public"] = 1, ["kube-node-lease"] = 1} local content = cli:get("http://127.0.0.1:10255/pods") local obj = cli:jdecode(content.body) + if not obj then + local cmd = ' curl -s -k -XGET https://127.0.0.1:10250/pods --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token) "' + local f = io.popen(cmd,"r") + local podsinfo = f:read("*a") + f:close() + obj = json.decode(podsinfo) + end for _, pod in ipairs(obj.items) do local metadata = pod.metadata + if blacklist[metadata.namespace] then + goto continue + end local lpod = {name = metadata.name, namespace = metadata.namespace, uid = pystring:replace(metadata.uid, "-", "_"), @@ -97,6 +108,7 @@ local function setupCons(res) cons[c] = cell end end + ::continue:: end return cons @@ -109,11 +121,18 @@ function CpodsAll:getAllcons(procfs) local cons = {} local c = 0 local content = cli:get("http://127.0.0.1:10255/pods") - if #content.body == 0 then return cons end local obj = cli:jdecode(content.body) + if not obj then + local cmd = ' curl -s -k -XGET https://127.0.0.1:10250/pods --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token) "' + local f = io.popen(cmd,"r") + local podsinfo = f:read("*a") + f:close() + obj = json.decode(podsinfo) + end for _, pod in ipairs(obj.items) do local metadata = pod.metadata + --print(string.format("podns :%s, pod:%s",metadata.namespace, metadata.name)) local lpod = {name = metadata.name, namespace = metadata.namespace, uid = pystring:replace(metadata.uid, "-", "_"), @@ -144,29 +163,17 @@ local function setupPlugins(res, proto, pffi, mnt, ino) for _, con in ipairs(cons) do local ls = { { - name = "pod_name", + name = "podname", index = con.pod.name, }, { - name = "con_name", - index = con.name, - }, - { - name = "qos", - index = con.pod.qos, + name = "container", + index = con.name.."-"..string.sub(con.id,0,4), }, { - name = "ns", + name = "podns", index = con.pod.namespace, }, - { - name = "uid", - index = con.pod.uid, - }, - { - name = "con_id", - index = con.id, - }, } for _, plugin in ipairs(res.container.luaPlugin) do @@ -189,15 +196,23 @@ function CpodsAll:_init_(resYaml, proto, pffi, mnt) self._ino = Cinotifies.new() self._plugins = setupPlugins(self._resYaml, self._proto, self._pffi, self._mnt, self._ino) + + self._ino:add(mnt .. "sys/fs/cgroup/memory/kubepods.slice") + self._ino:add(mnt .. "sys/fs/cgroup/memory/kubepods.slice/kubepods-besteffort.slice") + self._ino:add(mnt .. "sys/fs/cgroup/memory/kubepods.slice/kubepods-burstable.slice") + print( "pods plugin add " .. #self._plugins) end function CpodsAll:proc(elapsed, lines) local rec = {} - if self._ino:isChange() then + if self._ino:isChange() or #self._plugins == 0 then print("cgroup changed.") self._ino = Cinotifies.new() self._plugins = setupPlugins(self._resYaml, self._proto, self._pffi, self._mnt, self._ino) + self._ino:add(self._mnt .. "sys/fs/cgroup/memory/kubepods.slice") + self._ino:add(self._mnt .. "sys/fs/cgroup/memory/kubepods.slice/kubepods-besteffort.slice") + self._ino:add(self._mnt .. "sys/fs/cgroup/memory/kubepods.slice/kubepods-burstable.slice") end for i, plugin in ipairs(self._plugins) do --local res = plugin:proc(elapsed, lines) -- Gitee From 7c0868fd157431ba3b944c8d94100be3153af292 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 14:05:11 +0800 Subject: [PATCH 3/6] mounts: filter devtmpfs/tmpfs Signed-off-by: yinbinbin --- source/tools/monitor/unity/collector/proc_mounts.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/unity/collector/proc_mounts.lua b/source/tools/monitor/unity/collector/proc_mounts.lua index 2134f8a6..96e77b2b 100644 --- a/source/tools/monitor/unity/collector/proc_mounts.lua +++ b/source/tools/monitor/unity/collector/proc_mounts.lua @@ -47,7 +47,8 @@ end local function get_point(fName) local lines = get_lines(fName) - local lOut = {"devtmpfs", "tmpfs"} + ---local lOut = {"devtmpfs", "tmpfs"} + local lOut = {} local tDev = {} local ret = {} for _, line in ipairs(lines) do @@ -115,4 +116,4 @@ function CprocMounts:proc(elapsed, lines) self:push(lines) end -return CprocMounts \ No newline at end of file +return CprocMounts -- Gitee From eacb4335cb1540140b9602b0592739e4526131bd Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 12:00:22 +0800 Subject: [PATCH 4/6] container: add memory util Signed-off-by: yinbinbin --- .../collector/container/cg_memory_util.lua | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 source/tools/monitor/unity/collector/container/cg_memory_util.lua diff --git a/source/tools/monitor/unity/collector/container/cg_memory_util.lua b/source/tools/monitor/unity/collector/container/cg_memory_util.lua new file mode 100644 index 00000000..54316714 --- /dev/null +++ b/source/tools/monitor/unity/collector/container/cg_memory_util.lua @@ -0,0 +1,80 @@ +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +local root = "sys/fs/cgroup/memory/" +local dfile = "/memory.stat" +local usage = "/memory.usage_in_bytes" +local limit = "/memory.limit_in_bytes" +local system = require("common.system") + +local CgMemUtil = class("cg_memory_util", CvProc) + +--ls{}, (pod_name and docker_name +function CgMemUtil:_init_(proto, pffi, mnt, path, ls) + CvProc._init_(self, proto, pffi, mnt, root .. path .. dfile) + self.ls = ls + self.path = mnt..root..path..dfile + self.limitpath = mnt..root..path..limit + self.usagepath = mnt..root..path..usage + self.limit = 0 + self.usage = 0 +end + +function CgMemUtil:_getLimit_() + local pfile = io.open(self.limitpath, "r") + local line = pfile:read() + self.limit = tonumber(line) + io.close(pfile) +end + +function CgMemUtil:_getUsage_() + local pfile = io.open(self.usagepath, "r") + local line = pfile:read() + self.usage = tonumber(line) + io.close(pfile) +end + +function CgMemUtil:proc(elapsed, lines) + local c = 1 + local k = 1 + CvProc.proc(self) + self:_getLimit_() + self:_getUsage_() + local values = {} + for line in io.lines(self.pFile) do + local name + local cell = pystring:split(line) + local num = #cell + local val = tonumber(cell[num]) + local metrics = {["total_cache"] = 1, ["total_rss"] = 1, ["total_shmem"]=1,["total_dirty"]=1,["total_pgpgin"]=1,["total_pgpgout"]=1, + ["total_inactive_anon"] =1, ["total_active_anon"]=1,["total_inactive_file"] =1, ["total_active_file"] =1,["total_pgfault"]=1} + --we assume that: memory.use_hierarchy is "1" + if metrics[cell[1]] then + name = string.sub(cell[1], 7) + values[k] = { + name = name, + value = val + } + k = k + 1 + if ("total_cache" == cell[1]) or ("total_rss" == cell[1]) then + local ratio = (100.00*val) / tonumber(self.usage) + values[k] = { + name = name.."_ratio", + value = ratio + } + k = k + 1 + end + end + end + values[k] = { + name = "usage", + value = self.usage + } + values[k+1] = { + name = "mem_util", + value = (tonumber(self.usage)*100.0)/ tonumber(self.limit) + } + self:appendLine(self:_packProto("cg_memory_util", self.ls, values)) + self:push(lines) +end +return CgMemUtil -- Gitee From 41cfd25c8b7a6147589d01d67ef04c7b3e432215 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 15:42:01 +0800 Subject: [PATCH 5/6] container: add cfs quota Signed-off-by: yinbinbin --- .../collector/container/cg_cpu_cfs_quota.lua | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 source/tools/monitor/unity/collector/container/cg_cpu_cfs_quota.lua diff --git a/source/tools/monitor/unity/collector/container/cg_cpu_cfs_quota.lua b/source/tools/monitor/unity/collector/container/cg_cpu_cfs_quota.lua new file mode 100644 index 00000000..abf49a31 --- /dev/null +++ b/source/tools/monitor/unity/collector/container/cg_cpu_cfs_quota.lua @@ -0,0 +1,64 @@ +--- +--- Created by Hailong Liu. +--- DateTime: 2023/5/18 +--- + +require("common.class") +local unistd = require("posix.unistd") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +local root = "sys/fs/cgroup/cpu/" +local quotFile = "/cpu.cfs_quota_us" +local periodFile = "/cpu.cfs_period_us" + +local CgCfsQuota = class("cg_cfs_quota", CvProc) + +--ls{}, (pod_name and docker_name +function CgCfsQuota:_init_(proto, pffi, mnt, path, ls) + CvProc._init_(self, proto, pffi, mnt, root .. path .. quotFile) + self.ls = ls + self.period = 0 + self.quota = 0 + self.nr_cpus = unistd.sysconf(84) + self.periodFp = mnt..root..path..periodFile +end + +function CgCfsQuota:proc(elapsed, lines) + -- if pFile not valid ,return -1 + local c = 1 + CvProc.proc(self) + local values = {} + + for line in io.lines(self.pFile) do + local cell = pystring:split(line) + self.quota = tonumber(cell[1]) + values[c] = { + name = "quota_us", + value = self.quota + } + c = c + 1 + end + + for line in io.lines(self.periodFp) do + local cell = pystring:split(line) + self.period = tonumber(cell[1]) + values[c] = { + name = "period_us", + value = self.period + } + c = c + 1 + end + local ratio = self.nr_cpus*100 + if self.quota ~= -1 then + ratio = self.quota*100.0/self.period + end + + values[c] = { + name = "quota_ratio", + value = ratio + } + self:appendLine(self:_packProto("cgCpuQuota", self.ls, values)) + self:push(lines) +end + +return CgCfsQuota -- Gitee From fd10bfb9d9f43287861192756449c6b83817a242 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Mon, 10 Jul 2023 15:55:31 +0800 Subject: [PATCH 6/6] container: add memcg glob_latency Signed-off-by: yinbinbin --- .../container/cg_mem_drcm_glob_latency.lua | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 source/tools/monitor/unity/collector/container/cg_mem_drcm_glob_latency.lua diff --git a/source/tools/monitor/unity/collector/container/cg_mem_drcm_glob_latency.lua b/source/tools/monitor/unity/collector/container/cg_mem_drcm_glob_latency.lua new file mode 100644 index 00000000..8651b277 --- /dev/null +++ b/source/tools/monitor/unity/collector/container/cg_mem_drcm_glob_latency.lua @@ -0,0 +1,43 @@ +--- +--- Created by liaozhaoyan. +--- DateTime: 2023/5/18 +--- + +require("common.class") +local pystring = require("common.pystring") +local CvProc = require("collector.vproc") +local root = "sys/fs/cgroup/memory/" +local dfile = "/memory.direct_reclaim_global_latency" + +local CgMemGlobDrcmLat = class("cgGlbDrcmLatency", CvProc) + +--ls{}, (pod_name and docker_name +function CgMemGlobDrcmLat:_init_(proto, pffi, mnt, path, ls) + CvProc._init_(self, proto, pffi, mnt, root .. path .. dfile) + self.ls = ls +end + +function CgMemGlobDrcmLat:proc(elapsed, lines) + -- if pFile not valid ,return -1 + local c = 1 + CvProc.proc(self) + local values = {} + + for line in io.lines(self.pFile) do + local cell = pystring:split(line) + local tmp = cell[1] + tmp = string.gsub(tmp, ":", "", 1) + tmp = string.gsub(tmp, ">=", "", 1) + tmp = string.gsub(tmp, "-", "to", 1) + tmp = string.gsub(tmp, "%(.*%)", "", 1) + values[c] = { + name = "memDrcm_glb_lat_"..tmp, + value = tonumber(cell[2]) + } + c = c + 1 + end + self:appendLine(self:_packProto("cgGlbDrcmLatency", self.ls, values)) + self:push(lines) +end + +return CgMemGlobDrcmLat -- Gitee