From 33fb1c910e90a14cd5028f6e58631378971eaa95 Mon Sep 17 00:00:00 2001 From: jietaoxiao Date: Tue, 18 Jul 2023 15:39:33 +0800 Subject: [PATCH 1/3] unity/container: add container working_set_bytes in cg_memory_util --- .../collector/container/cg_memory_util.lua | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/source/tools/monitor/unity/collector/container/cg_memory_util.lua b/source/tools/monitor/unity/collector/container/cg_memory_util.lua index 54316714..3958707e 100644 --- a/source/tools/monitor/unity/collector/container/cg_memory_util.lua +++ b/source/tools/monitor/unity/collector/container/cg_memory_util.lua @@ -37,44 +37,73 @@ end function CgMemUtil:proc(elapsed, lines) local c = 1 local k = 1 + local total_inactive_file + local total_active_file CvProc.proc(self) self:_getLimit_() self:_getUsage_() local values = {} for line in io.lines(self.pFile) do - local name - local cell = pystring:split(line) - local num = #cell - local val = tonumber(cell[num]) - local metrics = {["total_cache"] = 1, ["total_rss"] = 1, ["total_shmem"]=1,["total_dirty"]=1,["total_pgpgin"]=1,["total_pgpgout"]=1, + local name + local cell = pystring:split(line) + local num = #cell + local val = tonumber(cell[num]) + local metrics = {["total_cache"] = 1, ["total_rss"] = 1, ["total_shmem"]=1,["total_dirty"]=1,["total_pgpgin"]=1,["total_pgpgout"]=1, ["total_inactive_anon"] =1, ["total_active_anon"]=1,["total_inactive_file"] =1, ["total_active_file"] =1,["total_pgfault"]=1} - --we assume that: memory.use_hierarchy is "1" - if metrics[cell[1]] then - name = string.sub(cell[1], 7) - values[k] = { - name = name, - value = val - } - k = k + 1 - if ("total_cache" == cell[1]) or ("total_rss" == cell[1]) then + --we assume that: memory.use_hierarchy is "1" + if metrics[cell[1]] then + name = string.sub(cell[1], 7) + values[k] = { + name = name, + value = val + } + k = k + 1 + + if ("total_cache" == cell[1]) or ("total_rss" == cell[1]) then local ratio = (100.00*val) / tonumber(self.usage) values[k] = { - name = name.."_ratio", - value = ratio + name = name.."_ratio", + value = ratio } k = k + 1 + end + + if (cell[1] == "total_inactive_file") then + total_inactive_file = val + end + if (cell[1] == "total_active_file") then + total_active_file = val + end + end end + + --[[ + in cAdvisor, working_set_bytes = memory.usage_in_bytes - total_inactive_file + we use working_set_bytes = usage - (total_inactive_file + total_active_file) to + reflect the actual container used memory. + ]] + local workingSet = self.usage + if workingSet < (total_active_file + total_inactive_file) then + workingSet = 0 + else + workingSet = workingSet - (total_active_file + total_inactive_file) end values[k] = { - name = "usage", - value = self.usage + name = "working_set_bytes", + value = workingSet } + values[k+1] = { + name = "usage", + value = self.usage + } + values[k+2] = { name = "mem_util", value = (tonumber(self.usage)*100.0)/ tonumber(self.limit) } self:appendLine(self:_packProto("cg_memory_util", self.ls, values)) self:push(lines) end + return CgMemUtil -- Gitee From 203304838cbb354cc52b3b64916737475df64815 Mon Sep 17 00:00:00 2001 From: jietaoxiao Date: Tue, 18 Jul 2023 21:39:39 +0800 Subject: [PATCH 2/3] unity/container: modify working_set_bytes caculation --- .../unity/collector/container/cg_memory_util.lua | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/source/tools/monitor/unity/collector/container/cg_memory_util.lua b/source/tools/monitor/unity/collector/container/cg_memory_util.lua index 3958707e..839fecdd 100644 --- a/source/tools/monitor/unity/collector/container/cg_memory_util.lua +++ b/source/tools/monitor/unity/collector/container/cg_memory_util.lua @@ -38,7 +38,6 @@ function CgMemUtil:proc(elapsed, lines) local c = 1 local k = 1 local total_inactive_file - local total_active_file CvProc.proc(self) self:_getLimit_() self:_getUsage_() @@ -71,23 +70,15 @@ function CgMemUtil:proc(elapsed, lines) if (cell[1] == "total_inactive_file") then total_inactive_file = val end - if (cell[1] == "total_active_file") then - total_active_file = val - end - end end - --[[ - in cAdvisor, working_set_bytes = memory.usage_in_bytes - total_inactive_file - we use working_set_bytes = usage - (total_inactive_file + total_active_file) to - reflect the actual container used memory. - ]] + local workingSet = self.usage - if workingSet < (total_active_file + total_inactive_file) then + if workingSet < total_inactive_file then workingSet = 0 else - workingSet = workingSet - (total_active_file + total_inactive_file) + workingSet = workingSet - total_inactive_file end values[k] = { name = "working_set_bytes", -- Gitee From 682ff99b5b0fd14dcffd4a338195d09522678f6d Mon Sep 17 00:00:00 2001 From: jietaoxiao Date: Thu, 20 Jul 2023 15:25:43 +0800 Subject: [PATCH 3/3] unity/container: add container network tx/rx bytes and packets metrics --- .../container/container_network_tr.lua | 120 ++++++++++++++++++ source/tools/monitor/unity/etc/k8s.yaml | 8 +- 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 source/tools/monitor/unity/collector/container/container_network_tr.lua diff --git a/source/tools/monitor/unity/collector/container/container_network_tr.lua b/source/tools/monitor/unity/collector/container/container_network_tr.lua new file mode 100644 index 00000000..1460b78d --- /dev/null +++ b/source/tools/monitor/unity/collector/container/container_network_tr.lua @@ -0,0 +1,120 @@ +require("common.class") +local pystring = require("common.pystring") +local system = require("common.system") +local CvProc = require("collector.vproc") + +local cpu_root = "sys/fs/cgroup/cpu/" +local procs_file = "/cgroup.procs" +local proc = "proc/" +local net_dev = "/net/dev" + +local ProcNetStat = class("container_network_tr", CvProc) + +function ProcNetStat:_init_(proto, pffi, mnt, path, ls) + CvProc._init_(self, proto, pffi, mnt, cpu_root..path..procs_file) + self.ls = ls + self.mnt = mnt + self.procnetpath = "" + self.init_pid = "" + +end + +--- open cgroup.procs to get the first pid +function ProcNetStat:_getInitPid_() + local pfile = io.open(self.pFile, "r") + local firstline = pfile:read("*line") + self.init_pid = firstline + io.close(pfile) +end + +--- procnetpath = "/proc/self/net/dev" +function ProcNetStat:_getProcNetPath_() + self:_getInitPid_() + self.procnetpath = self.mnt..proc..self.init_pid..net_dev +end + +local function isIgnoredDevice(devname) + local ignoredDevicePrefixes = {"lo", "veth", "docker"} + for _, value in ipairs(ignoredDevicePrefixes) do + if pystring:startswith(devname, value) then + return true + end + end + return false +end + +local function copyTable(original) + local copy = {} + for key, value in pairs(original) do + copy[key] = value + end + return copy +end + +function ProcNetStat:proc(elapsed, lines) + local c = 1 + local k = 1 + local local_ls = copyTable(self.ls) + local devName + local RxBytes, RxPackets = 0, 0 + local TxBytes, TxPackets = 0, 0 + local values = {} + CvProc.proc(self) + self:_getProcNetPath_() + + for line in io.lines(self.procnetpath) do + local cell + + --- skip first two lines + if c < 3 then + c = c + 1 + goto continue + end + + line = pystring:replace(line, ":", "") + --- "eth0" and "lo" may start with " " + line = pystring:lstrip(line) + cell = pystring:split(line) + if #cell ~= 17 then + print("invalid interface stats line " .. line) + goto continue + end + + devName = cell[1] + --- ignore lo and veth interface + if isIgnoredDevice(devName) then + goto continue + end + + RxBytes = tonumber(cell[2]) + RxPackets = tonumber(cell[3]) + TxBytes = tonumber(cell[10]) + TxPackets = tonumber(cell[11]) + + --- local_ls = self.ls (podname, container, podns) + table.insert(local_ls, {name = "interface", index = devName}) + values[k] = { + name = "network_receive_bytes", + value = RxBytes + } + values[k+1] = { + name = "network_receive_packets", + value = RxPackets + } + values[k+2] = { + name = "network_transmit_bytes", + value = TxBytes + } + values[k+3] = { + name = "network_transmit_packets", + value = TxPackets + } + + self:appendLine(self:_packProto("container_network_tr", local_ls , values)) + self:push(lines) + + ::continue:: + end +end + +return ProcNetStat \ No newline at end of file diff --git a/source/tools/monitor/unity/etc/k8s.yaml b/source/tools/monitor/unity/etc/k8s.yaml index 458b90f4..b2af26d8 100644 --- a/source/tools/monitor/unity/etc/k8s.yaml +++ b/source/tools/monitor/unity/etc/k8s.yaml @@ -29,7 +29,8 @@ container: #"cg_cpuacct_stat" is a substitute of cg_cpuacct_proc_stat luaPlugin: ["cg_memory_fail_cnt", "cg_memory_util", "cg_memory_dcmp_latency", "cg_memory_drcm_latency", "cg_cpuacct_wait_latency", "cg_cpuacct_proc_stat", - "cg_cpu_stat", "cg_pmu_events", "cg_cpu_cfs_quota", "cg_mem_drcm_glob_latency"] + "cg_cpu_stat", "cg_pmu_events", "cg_cpu_cfs_quota", "cg_mem_drcm_glob_latency", + "container_network_tr"] directCgPath: - "/" - "/kubepods.slice" @@ -331,3 +332,8 @@ metrics: head: value help: "pmu events of cgroups" type: "gauge" + - title: sysom_container_network_rx/tx + from: container_network_tr + head: value + help: "container network rx/tx stat" + type: "gauge" -- Gitee