From 75f80554a2bf02ac4c81917ded7484f98ec2e449 Mon Sep 17 00:00:00 2001 From: Hang Zhao Date: Thu, 3 Mar 2022 19:19:52 +0800 Subject: [PATCH 1/3] loadtask:add -g option to default collect cpu perf flamegraph by cpu_flamegraph Signed-off-by: Hang Zhao --- source/tools/detect/loadtask/loadtask.sh | 29 ++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/source/tools/detect/loadtask/loadtask.sh b/source/tools/detect/loadtask/loadtask.sh index bd65e196..e31a4f9f 100755 --- a/source/tools/detect/loadtask/loadtask.sh +++ b/source/tools/detect/loadtask/loadtask.sh @@ -7,6 +7,7 @@ # Modify Date: 2021-02-06 10:53 # Function: #***************************************************************# +TOOLS_ROOT="$SYSAK_WORK_PATH/tools" usage() { echo "sysak loadtask: show all tasks of load contribution" @@ -17,6 +18,7 @@ usage() { echo " -d, keep monitoring even if greater than maxload occurs.useful only if the -m option is set" echo " -s, show summary result" echo " -k, terminate running ${selftaskname} which started previously" + echo " -g, default collect cpu perf flamegraph by cpu_flamegraph tool" echo " -r datafile, read datafile created by '-f datafile' or by default(datafile directory /var/log/sysak/loadtask/) and show result" } @@ -159,6 +161,13 @@ show_result() { echo "" } +collect_global_framegraph() { + if [ -f $TOOLS_ROOT/cpu_flamegraph ]; then + $TOOLS_ROOT/cpu_flamegraph -d 5 | xargs -I {} sudo cp {} $global_cpuflamegraph + sudo cp $global_cpuflamegraph $tmp_cpuflamegraph + fi +} + current_analyse() { local high_sirq=0 local exist_sirq_tool=0 @@ -167,8 +176,12 @@ current_analyse() { exist_sirq_tool=1 fi + if [ "$is_cpuflamegraph" == "true" ];then + collect_global_framegraph + fi echo "####################################################################################" > $tmpfile - date >> $tmpfile + echo "Time: `date "+%Y-%m-%d %H:%M:%S"`" >> $tmpfile + echo "$global_cpuflamegraph" >> $tmpfile cat /proc/loadavg >> $tmpfile load=`cat /proc/loadavg | awk '{print $1}'` @@ -202,8 +215,10 @@ current_analyse() { high_cost+="sys " extra_cmd="[sysmonitor]" extra_info="high memory or kernel competition " - if [ -f $TOOLS_ROOT/cpu_flamegraph ]; then - $TOOLS_ROOT/cpu_flamegraph -d 5 + if [ "$is_cpuflamegraph" != "true" ];then + if [ -f $TOOLS_ROOT/cpu_flamegraph ]; then + $TOOLS_ROOT/cpu_flamegraph -d 5 + fi fi fi if [ $(echo "$irq_util > ((100-$cpu_idle)*0.05)" | bc) -eq 1 ]; then @@ -337,8 +352,11 @@ parse_datafile() { } interval=5 +is_cpuflamegraph=false loadtask_dir=/var/log/sysak/loadtask/ datafile=${loadtask_dir}loadtask-`date "+%Y-%m-%d-%H-%M-%S"`.log +global_cpuflamegraph=${loadtask_dir}global_cpuflamegraph-`date "+%Y-%m-%d-%H-%M-%S"`.svg +tmp_cpuflamegraph=${loadtask_dir}.tmp.svg tmpfile=${loadtask_dir}.tmplog rtaskfile=${loadtask_dir}runtask dtaskfile=${loadtask_dir}dtask @@ -353,7 +371,7 @@ selftaskname="`cat /proc/$$/status | grep -w "Name" | awk -F" " '{print $2}'`" parsed_datafile="" tmp_parsed_datafile=${loadtask_dir}.parsedlog -while getopts 'm:f:i:t:r:dskh' OPT; do +while getopts 'm:f:i:t:r:dskgh' OPT; do case $OPT in "h") usage @@ -387,6 +405,9 @@ while getopts 'm:f:i:t:r:dskh' OPT; do parse_datafile exit 0 ;; + "g") + is_cpuflamegraph="true" + ;; *) usage exit -1 -- Gitee From a06e81ba531ad68d494c6dd56fea3ec0b1eafa30 Mon Sep 17 00:00:00 2001 From: Hang Zhao Date: Mon, 7 Mar 2022 15:14:48 +0800 Subject: [PATCH 2/3] configure:add cpu_flamegraph to default_target Signed-off-by: Hang Zhao --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 01953ca3..d45d4428 100755 --- a/configure +++ b/configure @@ -129,7 +129,7 @@ echo "EXTRA_LDFLAGS = --static" >> $config_host_mak fi if [ $default_target = "yes" ]; then -target_list="cpuirq loadtask sysmonitor $target_list" +target_list="cpuirq loadtask sysmonitor cpu_flamegraph $target_list" for i in $target_list;do target_files="$(grep -nrw "target := $i" $(pwd)/source/tools/ | awk -F ":" '{print $1}') $target_files" done -- Gitee From 798be31b14fb0b7b9c4dbcef7758ee1e6b4649e1 Mon Sep 17 00:00:00 2001 From: Hang Zhao Date: Tue, 8 Mar 2022 19:10:49 +0800 Subject: [PATCH 3/3] loadtask:record loadavg, loadavg_r and loadavg_d Signed-off-by: Hang Zhao --- source/lib/internal/kernel_module/Makefile | 28 ++++ .../kernel_module/modules/loadtask/loadtask.c | 133 ++++++++++++++++++ .../lib/internal/kernel_module/sysak_mods.c | 76 +++++++++- source/tools/detect/loadtask/Makefile | 1 + source/tools/detect/loadtask/loadtask.sh | 18 ++- 5 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 source/lib/internal/kernel_module/modules/loadtask/loadtask.c diff --git a/source/lib/internal/kernel_module/Makefile b/source/lib/internal/kernel_module/Makefile index 61440ba2..2ef277f6 100644 --- a/source/lib/internal/kernel_module/Makefile +++ b/source/lib/internal/kernel_module/Makefile @@ -27,19 +27,47 @@ endif ifneq ($(findstring taskctl,$(TARGET_LIST)),) sysak-objs += modules/task_ctl/task_ctrl.o endif +ifneq ($(findstring appscan,$(TARGET_LIST)),) +sysak-objs += modules/appscan/appscan.o +endif +ifneq ($(findstring iosdiag,$(TARGET_LIST)),) +ifneq ($(wildcard $(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION)),) +sysak-objs += modules/iosdiag/virtio_blk.o modules/iosdiag/nvme.o modules/iosdiag/iosdiag.o modules/iosdiag/data_analysis.o +endif +endif +ifneq ($(findstring ulockcheck,$(TARGET_LIST)),) +sysak-objs += modules/ulockcheck/ulockcheck.o +endif +ifneq ($(findstring pgfthrash,$(TARGET_LIST)),) +sysak-objs += modules/pgfthrash/pgfthrash.o +endif ifneq ($(findstring schedtrace,$(TARGET_LIST)),) sysak-objs += modules/schedtrace/schedtrace.o endif +ifneq ($(findstring reclaimhang,$(TARGET_LIST)),) +sysak-objs += modules/reclaimhang/reclaimhang.o +endif ifneq ($(findstring mmaptrace,$(TARGET_LIST)),) sysak-objs += modules/mmaptrace/mmaptrace.o endif +ifneq ($(findstring virtnet,$(TARGET_LIST)),) +sysak-objs += modules/virtnet/virtnet.o +endif +ifneq ($(findstring loadtask,$(TARGET_LIST)),) +sysak-objs += modules/loadtask/loadtask.o +endif + obj-m += sysak.o EXTRA_CFLAGS := -I$(MODULE_SRC) EXTRA_CFLAGS += -I$(MODULE_SRC)/include +EXTRA_CFLAGS += -I$(MODULE_SRC)/include/$(KERNEL_VERSION) EXTRA_CFLAGS += -I$(SRC)/lib/uapi/include +ifneq ($(wildcard $(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION)),) +EXTRA_CFLAGS += -I$(MODULE_SRC)/modules/iosdiag -I$(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION) +endif sysak_mod: make -C $(KERNEL_BUILD_PATH) M=$(MODULE_SRC) diff --git a/source/lib/internal/kernel_module/modules/loadtask/loadtask.c b/source/lib/internal/kernel_module/modules/loadtask/loadtask.c new file mode 100644 index 00000000..dd2416ca --- /dev/null +++ b/source/lib/internal/kernel_module/modules/loadtask/loadtask.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/proc.h" +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +#include +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) +static void *orig_tasklist_lock; +void (*orig_get_avenrun)(unsigned long *loads, unsigned long offset, int shift); +#ifdef CONFIG_RICH_CONTAINER +int *orig_sysctl_rich_container_enable; +bool (*orig_child_cpuacct)(struct task_struct *tsk); +static void (*orig_get_avenrun_r)(unsigned long *loads, unsigned long offset, + int shift); +static void (*orig_get_cgroup_avenrun)(struct task_struct *tsk, + unsigned long *loads, unsigned long offset, + int shift, bool running); +static inline bool orig_in_rich_container(struct task_struct *tsk) +{ + if (*orig_sysctl_rich_container_enable == 0) + return false; + + return (task_active_pid_ns(tsk) != &init_pid_ns) && orig_child_cpuacct(tsk); +} +#else +static inline void (*orig_get_avenrun_r)(unsigned long *loads, unsigned long offset, + int shift) { } +static inline void (*orig_get_cgroup_avenrun)(struct task_struct *tsk, + unsigned long *loads, unsigned long offset, + int shift, bool running) { } +static inline bool orig_in_rich_container(struct task_struct *tsk) +{ + return false; +} +#endif +#endif + +static int loadtask_show(struct seq_file *m, void *v) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + unsigned long avnrun[3], avnrun_r[3]; + + rcu_read_lock(); + if (orig_in_rich_container(current)) { + struct task_struct *init_tsk; + + read_lock(orig_tasklist_lock); + init_tsk = task_active_pid_ns(current)->child_reaper; + get_task_struct(init_tsk); + read_unlock(orig_tasklist_lock); + orig_get_cgroup_avenrun(init_tsk, avnrun, FIXED_1/200, 0, false); + orig_get_cgroup_avenrun(init_tsk, avnrun_r, FIXED_1/200, 0, true); + put_task_struct(init_tsk); + } else { + orig_get_avenrun(avnrun, FIXED_1/200, 0); + orig_get_avenrun_r(avnrun_r, FIXED_1/200, 0); + } + rcu_read_unlock(); + + seq_printf(m, "loadavg: %lu.%02lu %lu.%02lu %lu.%02lu\n", + LOAD_INT(avnrun[0]), LOAD_FRAC(avnrun[0]), + LOAD_INT(avnrun[1]), LOAD_FRAC(avnrun[1]), + LOAD_INT(avnrun[2]), LOAD_FRAC(avnrun[2])); + seq_printf(m, "loadavg_r: %lu.%02lu %lu.%02lu %lu.%02lu\n", + LOAD_INT(avnrun_r[0]), LOAD_FRAC(avnrun_r[0]), + LOAD_INT(avnrun_r[1]), LOAD_FRAC(avnrun_r[1]), + LOAD_INT(avnrun_r[2]), LOAD_FRAC(avnrun_r[2])); + seq_printf(m, "loadavg_d: %lu.%02lu %lu.%02lu %lu.%02lu\n", + LOAD_INT(avnrun[0] - avnrun_r[0]), LOAD_FRAC(avnrun[0] - avnrun_r[0]), + LOAD_INT(avnrun[1] - avnrun_r[1]), LOAD_FRAC(avnrun[1] - avnrun_r[1]), + LOAD_INT(avnrun[2] - avnrun_r[2]), LOAD_FRAC(avnrun[2] - avnrun_r[2])); +#endif + return 0; +} + +DEFINE_PROC_ATTRIBUTE_RO(loadtask); + +int loadtask_init(void) +{ + struct proc_dir_entry *parent_dir; + struct proc_dir_entry *entry_print; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) + orig_tasklist_lock = (void *)kallsyms_lookup_name("tasklist_lock"); + if (!orig_tasklist_lock) + return -1; + orig_get_avenrun = (void *)kallsyms_lookup_name("get_avenrun"); + if (!orig_get_avenrun) + return -1; +#ifdef CONFIG_RICH_CONTAINER + orig_get_avenrun_r = (void *)kallsyms_lookup_name("get_avenrun_r"); + if (!orig_get_avenrun_r) + return -1; + orig_get_cgroup_avenrun = (void *)kallsyms_lookup_name("get_cgroup_avenrun"); + if (!orig_get_cgroup_avenrun) + return -1; + orig_sysctl_rich_container_enable= (void *)kallsyms_lookup_name("sysctl_rich_container_enable"); + if (!orig_sysctl_rich_container_enable) + return -1; +#endif + orig_child_cpuacct = (void *)kallsyms_lookup_name("child_cpuacct"); + if (!orig_child_cpuacct) + return -1; +#endif + parent_dir = sysak_proc_mkdir("loadtask"); + if (!parent_dir) { + goto failed_root; + } + + entry_print = proc_create("loadavg", 0444, parent_dir, &loadtask_fops); + if(!entry_print) + goto failed; + return 0; + +failed: + sysak_remove_proc_entry("loadtask"); +failed_root: + return -1; +} + +int loadtask_exit(void) +{ + return 0; +} \ No newline at end of file diff --git a/source/lib/internal/kernel_module/sysak_mods.c b/source/lib/internal/kernel_module/sysak_mods.c index 5f9fd18f..12bf810c 100644 --- a/source/lib/internal/kernel_module/sysak_mods.c +++ b/source/lib/internal/kernel_module/sysak_mods.c @@ -40,15 +40,59 @@ int __attribute__((weak)) task_ctl_exit(void) return 0; } -int __attribute__((weak)) schedtrace_init(void) +int __attribute__((weak)) appscan_init(void) { return 0; } -int __attribute__((weak)) schedtrace_exit(void) +int __attribute__((weak)) appscan_exit(void) +{ + return 0; +} + +int __attribute__((weak)) ulockcheck_init(void) +{ + return 0; +} + +int __attribute__((weak)) ulockcheck_exit(void) +{ + return 0; +} + +int __attribute__((weak)) __attribute__((weak)) disk_hang_init(void) +{ + return 0; +} + +int __attribute__((weak)) disk_hang_exit(void) +{ + return 0; +} +int __attribute__((weak)) pgfthrash_init(void) +{ + return 0; +} +int __attribute__((weak)) pgfthrash_exit(void) +{ + return 0; +} +int __attribute__((weak)) schedtrace_init(void) { return 0; } +int __attribute__((weak)) schedtrace_exit(void) +{ + return 0; +} +int __attribute__((weak)) reclaimhang_init(void) +{ + return 0; +} +int __attribute__((weak)) reclaimhang_exit(void) +{ + return 0; +} int __attribute__((weak)) mmaptrace_init(void) { return 0; @@ -57,14 +101,36 @@ int __attribute__((weak)) mmaptrace_exit(void) { return 0; } - +int __attribute__((weak)) virtnet_trace_init(void) +{ + return 0; +} +int __attribute__((weak)) virtnet_trace_exit(void) +{ + return 0; +} +int __attribute__((weak)) loadtask_init(void) +{ + return 0; +} +int __attribute__((weak)) loadtask_exit(void) +{ + return 0; +} struct sysak_module sysak_modules[] = { { "trace_sig", trace_sig_init, trace_sig_exit}, { "memleak", memleak_init, memleak_uninit}, { "trace_irqoff", trace_irqoff_init, trace_irqoff_exit}, { "task_ctl", task_ctl_init, task_ctl_exit}, - { "schedtrace", schedtrace_init, schedtrace_exit}, - { "mmap_trace", mmaptrace_init, mmaptrace_exit}, + {"appscan", appscan_init, appscan_exit}, + {"iosdiag", disk_hang_init, disk_hang_exit}, + {"ulockcheck", ulockcheck_init, ulockcheck_exit}, + {"pgfthrash", pgfthrash_init, pgfthrash_exit}, + {"schedtrace", schedtrace_init, schedtrace_exit}, + {"reclaimhang", reclaimhang_init, reclaimhang_exit}, + {"mmap_trace", mmaptrace_init, mmaptrace_exit}, + {"virtnet", virtnet_trace_init, virtnet_trace_exit}, + {"loadtask", loadtask_init, loadtask_exit}, }; const int sysk_module_num = sizeof(sysak_modules) / sizeof(struct sysak_module); diff --git a/source/tools/detect/loadtask/Makefile b/source/tools/detect/loadtask/Makefile index 5ae9f8b2..24de920d 100644 --- a/source/tools/detect/loadtask/Makefile +++ b/source/tools/detect/loadtask/Makefile @@ -1,4 +1,5 @@ target := loadtask +DEPEND := "prev{-s modin};post{-e modun}" include $(SRC)/mk/sh.mk diff --git a/source/tools/detect/loadtask/loadtask.sh b/source/tools/detect/loadtask/loadtask.sh index e31a4f9f..204353c8 100755 --- a/source/tools/detect/loadtask/loadtask.sh +++ b/source/tools/detect/loadtask/loadtask.sh @@ -16,10 +16,11 @@ usage() { echo " -f datafile, file for output" echo " -i interval, the interval checking the load" echo " -d, keep monitoring even if greater than maxload occurs.useful only if the -m option is set" - echo " -s, show summary result" + echo " -s, show summary result. will insmod loadtask.ko" echo " -k, terminate running ${selftaskname} which started previously" echo " -g, default collect cpu perf flamegraph by cpu_flamegraph tool" echo " -r datafile, read datafile created by '-f datafile' or by default(datafile directory /var/log/sysak/loadtask/) and show result" + echo " -e, rmmod loadtask.ko" } uninterrupt_cnt=0 @@ -54,7 +55,7 @@ uninterrupt_dump() { get_container $pid flag=1 fi - + echo "-----" >> $tmpfile echo "`cat /proc/$tid/status | grep Name` $container" >> $tmpfile cat /proc/$tid/stack >> $tmpfile uninterrupt_cnt=$(($uninterrupt_cnt+1)) @@ -180,9 +181,15 @@ current_analyse() { collect_global_framegraph fi echo "####################################################################################" > $tmpfile + echo "Time: `date "+%Y-%m-%d %H:%M:%S"`" >> $tmpfile echo "$global_cpuflamegraph" >> $tmpfile cat /proc/loadavg >> $tmpfile + if [ "$summary" == "true" ];then + if [ -f "/proc/sysak/loadtask/loadavg" ]; then + cat /proc/sysak/loadtask/loadavg >> $tmpfile + fi + fi load=`cat /proc/loadavg | awk '{print $1}'` cal_sirq $exist_sirq_tool 0 @@ -243,6 +250,7 @@ current_analyse() { #extra_cmd+="[lockcheck]" fi + echo "-----" >> $tmpfile if [ $(echo "$load*0.2 > $uninterrupt_cnt" | bc) -eq 1 ]; then echo "load reason: high $high_cost cpu cost" >> $tmpfile else @@ -371,7 +379,7 @@ selftaskname="`cat /proc/$$/status | grep -w "Name" | awk -F" " '{print $2}'`" parsed_datafile="" tmp_parsed_datafile=${loadtask_dir}.parsedlog -while getopts 'm:f:i:t:r:dskgh' OPT; do +while getopts 'm:f:i:t:r:deskgh' OPT; do case $OPT in "h") usage @@ -408,6 +416,10 @@ while getopts 'm:f:i:t:r:dskgh' OPT; do "g") is_cpuflamegraph="true" ;; + "e") + echo "rmmod loadtask.ko" + exit 0 + ;; *) usage exit -1 -- Gitee