From 596e19ad3d492769ea67e3d25a1fdead0013957c Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Tue, 12 Apr 2022 20:24:01 +0800 Subject: [PATCH 1/7] memgraph:fix use memory usage Signed-off-by: yinbinbin --- source/tools/detect/memgraph/memgraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/tools/detect/memgraph/memgraph.py b/source/tools/detect/memgraph/memgraph.py index 56ef76b8..01befdc1 100644 --- a/source/tools/detect/memgraph/memgraph.py +++ b/source/tools/detect/memgraph/memgraph.py @@ -126,7 +126,7 @@ def is_number(s): return False def get_page_used(meminfo): - user = meminfo["Buffers"] + meminfo["Active(anon)"] + meminfo["Inactive(anon)"] + user = meminfo["Active(anon)"] + meminfo["Inactive(anon)"] user += meminfo["Active(file)"] + meminfo["Inactive(file)"] user += meminfo["Mlocked"] + meminfo["2048"] + meminfo["1048576"] kernelOther = meminfo["Slab"] + meminfo["KernelStack"] + meminfo["PageTables"] @@ -208,7 +208,7 @@ def memgraph_graph(meminfo): res["used"] = memgraph_free(meminfo) user = {} user["anon"] = meminfo["Active(anon)"] + meminfo["Inactive(anon)"] - user["cache"] = meminfo["Active(file)"] + meminfo["Inactive(file)"] + user["cache"] = meminfo["Cached"] user["buffers"] = meminfo["Buffers"] user["mlock"] = meminfo["Mlocked"] user["huge2M"] = meminfo["2048"] -- Gitee From d63d2fe92d035a3b6a95512b06a84f5fee7feecd Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Wed, 20 Apr 2022 15:23:26 +0800 Subject: [PATCH 2/7] skcheck:fix socket check error Signed-off-by: yinbinbin --- source/tools/detect/skcheck/skcheck.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/tools/detect/skcheck/skcheck.py b/source/tools/detect/skcheck/skcheck.py index ebc636e6..faaf7252 100644 --- a/source/tools/detect/skcheck/skcheck.py +++ b/source/tools/detect/skcheck/skcheck.py @@ -181,10 +181,12 @@ def scan_all_proc(inodes): if not os.path.exists(procName+fd): continue link = os.readlink(procName+fd) - if link.find("socket") == -1: + if link.find("socket:[") == -1: continue - inodeNum += 1 inode = link.strip().split("[") + if len(inode) < 2: + continue + inodeNum += 1 inode = inode[1][:-1].strip() #print("fd {} link {} inode {}".format(procName+fd, link, inode)) if inode not in inodes: -- Gitee From a5ca8f9cda5e16c145594476109e3c059c30b1c0 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Tue, 26 Apr 2022 12:35:31 +0800 Subject: [PATCH 3/7] oomcheck:support 4.19 oom Signed-off-by: yinbinbin --- source/tools/detect/oomcheck/oomcheck.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/tools/detect/oomcheck/oomcheck.py b/source/tools/detect/oomcheck/oomcheck.py index 4e005916..38941a56 100644 --- a/source/tools/detect/oomcheck/oomcheck.py +++ b/source/tools/detect/oomcheck/oomcheck.py @@ -124,7 +124,11 @@ def oom_is_host_oom(reason): return reason == OOM_REASON_HOST def oom_get_pid(oom_result, line, num): - pid = line.strip().split("Killed process")[1].strip().split()[0] + if OOM_END_KEYWORD in line: + split_line = OOM_END_KEYWORD + else: + split_line = OOM_END_KEYWORD_4_19 + pid = line.strip().split(split_line)[1].strip().split()[0] oom_result['sub_msg'][num]['pid'] = pid def oom_get_task_mem(oom_result, line, num): @@ -288,9 +292,10 @@ def oom_get_max_task(num, oom_result): continue if not dump_task: continue - if "Kill process" in line: + if OOM_END_KEYWORD in line or OOM_END_KEYWORD_4_19 in line: + break + if line.count('[') !=2 or line.count(']') !=2: break - pid_idx = line.rfind('[') last_idx = line.rfind(']') if pid_idx == -1 or last_idx == -1: @@ -330,7 +335,7 @@ def oom_reason_analyze(num, oom_result): oom_get_slab_unrecl(oom_result, line, num) elif "pages RAM" in line: oom_get_total_mem(oom_result, line, num) - elif "Killed process" in line: + elif OOM_END_KEYWORD in line or OOM_END_KEYWORD_4_19 in line: oom_get_task_mem(oom_result, line, num) oom_get_pid(oom_result, line, num) oom_result['node_num'] = node_num @@ -348,6 +353,9 @@ def oom_dmesg_analyze(dmesgs, oom_result): return dmesg = dmesgs.splitlines() oom_getting = 0 + task_name = "-unknow-" + if task_name not in oom_result['task']: + oom_result['task'][task_name] = 0 for line in dmesg: line = line.strip() if len(line) > 0 and OOM_BEGIN_KEYWORD in line: @@ -357,6 +365,9 @@ def oom_dmesg_analyze(dmesgs, oom_result): oom_result['sub_msg'][oom_result['oom_total_num']]['oom_msg'] = [] oom_result['sub_msg'][oom_result['oom_total_num']]['time'] = 0 oom_result['sub_msg'][oom_result['oom_total_num']]['cg_name'] = 'unknow' + oom_result['sub_msg'][oom_result['oom_total_num']]['task_name'] = task_name + oom_result['sub_msg'][oom_result['oom_total_num']]['pid'] = "0" + oom_result['sub_msg'][oom_result['oom_total_num']]['killed_task_mem'] = 0 if line.find('[') != -1: oom_result['sub_msg'][oom_result['oom_total_num']]['time'] = oom_time_to_normal_time(line.split('[')[1].split(']')[0]) oom_result['time'].append(oom_result['sub_msg'][oom_result['oom_total_num']]['time']) @@ -403,6 +414,7 @@ def oom_read_dmesg(data, mode, filename): def oom_diagnose(sn, data, mode): try: oom_result = {} + oom_result['task'] = "" oom_result['summary'] = "" oom_result['oom_total_num'] = 0 oom_result['cgroup'] = {} -- Gitee From e4382aaa86e4ccc4b00c2cafbdeb535e78e150ae Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Tue, 26 Apr 2022 13:06:45 +0800 Subject: [PATCH 4/7] oomcheck for shmem Signed-off-by: yinbinbin --- source/tools/detect/oomcheck/oomcheck.py | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/source/tools/detect/oomcheck/oomcheck.py b/source/tools/detect/oomcheck/oomcheck.py index 38941a56..2ef6e890 100644 --- a/source/tools/detect/oomcheck/oomcheck.py +++ b/source/tools/detect/oomcheck/oomcheck.py @@ -151,6 +151,29 @@ def oom_get_cgroup_mem(oom_result, line, num): oom_result['sub_msg'][num]['cg_usage'] = memory_usage oom_result['sub_msg'][num]['cg_limit'] = memory_limit +def oom_get_cgroup_shmem(oom_result, line, num): + inanon = "0" + anon = "0" + rss = '0' + oom_result['sub_msg'][num]['cg_inanon'] = inanon; + oom_result['sub_msg'][num]['cg_anon'] = anon; + oom_result['sub_msg'][num]['cg_rss'] = rss; + + if line.find("inactive_anon:") == -1: + return + inanon = line.strip().split("inactive_anon:")[1] + inanon = inanon.split()[0][:-2] + + anon = line.strip().split("active_anon:")[1] + anon = anon.split()[0][:-2] + + rss = line.strip().split("rss:")[1] + rss = rss.split()[0][:-2] + + oom_result['sub_msg'][num]['cg_inanon'] = inanon; + oom_result['sub_msg'][num]['cg_anon'] = anon; + oom_result['sub_msg'][num]['cg_rss'] = rss; + def oom_get_cgroup_name(oom_result, line, num): is_host = False if "limit of host" in line: @@ -255,6 +278,18 @@ def oom_cgroup_output(oom_result, num): summary += "cgroup内存限制:%s\n"%(oom['cg_limit']) return summary +def oom_cgroup_output_ext(oom_result, num): + summary = '' + oom = oom_result['sub_msg'][num] + reason = oom['reason'] + if not oom_is_cgroup_oom(reason): + return summary + anon = int(oom["cg_inanon"]) + int(oom["cg_anon"]) - int(oom["cg_rss"]) + if anon > int(oom['cg_usage'][:-2])*0.3: + summary = ",并且shmem内存使用量达到:%dKB 请及时删除tmpfs文件或ipcs共享内存"%(anon) + return summary + + def oom_check_score(oom, oom_result): res = oom_result['max'] if res['pid'] == 0: @@ -278,6 +313,7 @@ def oom_output_msg(oom_result,num): summary += oom_cgroup_output(oom_result, num) summary += oom_host_output(oom_result, num) summary += "诊断结论:%s"%(oom['reason']) + summary += oom_cgroup_output_ext(oom_result, num) summary += oom_check_score(oom, oom_result) return summary @@ -329,6 +365,8 @@ def oom_reason_analyze(num, oom_result): oom_get_cgroup_name(oom_result, line, num) elif "memory: usage" in line: oom_get_cgroup_mem(oom_result, line, num) + elif "Memory cgroup stats for" in line: + oom_get_cgroup_shmem(oom_result, line, num) elif "Normal free:" in line: oom_get_host_mem(oom_result, line, num) elif "slab_unreclaimable:" in line: -- Gitee From 8f12c3cc132b5ef2648f3d8a10f26e45e77d3841 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Tue, 26 Apr 2022 15:11:32 +0800 Subject: [PATCH 5/7] oomcheck:commit cgroup shmem Signed-off-by: yinbinbin --- source/tools/detect/oomcheck/oomcheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/tools/detect/oomcheck/oomcheck.py b/source/tools/detect/oomcheck/oomcheck.py index 2ef6e890..867a890a 100644 --- a/source/tools/detect/oomcheck/oomcheck.py +++ b/source/tools/detect/oomcheck/oomcheck.py @@ -15,8 +15,8 @@ import json import argparse import getopt -OOM_REASON_CGROUP = 'Cgroup内存使用量达到上限', -OOM_REASON_PCGROUP = '父Cgroup内存使用量达到上限', +OOM_REASON_CGROUP = '内存使用达到cgroup limit', +OOM_REASON_PCGROUP = '父Cgroup内存使用达到limit', OOM_REASON_HOST = '主机内存不足', OOM_REASON_MEMLEAK = '主机内存不足,存在内存泄漏', OOM_REASON_NODE = 'CPUSET 的mems值设置不合理', @@ -286,7 +286,7 @@ def oom_cgroup_output_ext(oom_result, num): return summary anon = int(oom["cg_inanon"]) + int(oom["cg_anon"]) - int(oom["cg_rss"]) if anon > int(oom['cg_usage'][:-2])*0.3: - summary = ",并且shmem内存使用量达到:%dKB 请及时删除tmpfs文件或ipcs共享内存"%(anon) + summary = ",且shmem内存使用量%dKB,需要清理tmpfs文件或ipcs"%(anon) return summary -- Gitee From 201eb408a226bd91d66911eed2ec23bfc4dbfbc3 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Tue, 26 Apr 2022 17:35:59 +0800 Subject: [PATCH 6/7] oomcheck:fix get slab_unreclaimable error Signed-off-by: yinbinbin --- source/tools/detect/oomcheck/oomcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/detect/oomcheck/oomcheck.py b/source/tools/detect/oomcheck/oomcheck.py index 867a890a..7e5a14bd 100644 --- a/source/tools/detect/oomcheck/oomcheck.py +++ b/source/tools/detect/oomcheck/oomcheck.py @@ -199,7 +199,7 @@ def oom_set_node_oom(oom_result, num, node_num): oom_result['sub_msg'][num]['reason'] = OOM_REASON_NODE def oom_get_slab_unrecl(oom_result, line, num): - if "slab_unreclaimable" not in line: + if "min:" in line or "low:" in line: oom_result['sub_msg'][num]['slab'] = 0 return True slab = int(line.strip().split("slab_unreclaimable:")[1]) -- Gitee From a558aeb3fded0f7a25e400dd38bb275ef8e51208 Mon Sep 17 00:00:00 2001 From: yinbinbin Date: Thu, 12 May 2022 16:27:27 +0800 Subject: [PATCH 7/7] memgraph: fix json create failed Signed-off-by: yinbinbin --- source/tools/detect/memgraph/memgraph.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/tools/detect/memgraph/memgraph.py b/source/tools/detect/memgraph/memgraph.py index 01befdc1..8571cfd5 100644 --- a/source/tools/detect/memgraph/memgraph.py +++ b/source/tools/detect/memgraph/memgraph.py @@ -185,6 +185,8 @@ def memgraph_get_meminfo(meminfo): def dump2json(res,filename): jsonStr = json.dumps(res) + if not os.path.exists(os.path.dirname(filename)): + os.popen("mkdir -p "+os.path.dirname(filename)).read() with open(filename, 'w') as jsonFile: jsonFile.write(jsonStr) -- Gitee