From 6f652704aef0aa1b588899c5b4ba7d46d5c00d7b Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Thu, 11 May 2023 14:31:38 +0800 Subject: [PATCH 1/2] offcpu: Take the output as folded format Signed-off-by: Hailong Liu --- source/tools/detect/sched/offcpu/README.md | 41 +++++++++ source/tools/detect/sched/offcpu/offcputime.c | 83 ++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 source/tools/detect/sched/offcpu/README.md diff --git a/source/tools/detect/sched/offcpu/README.md b/source/tools/detect/sched/offcpu/README.md new file mode 100644 index 00000000..f4304bb8 --- /dev/null +++ b/source/tools/detect/sched/offcpu/README.md @@ -0,0 +1,41 @@ +# 功能说明 +采集系统中任务off cpu情况 +offcpu是一款基于ebpf的用于采集系统中任务从on-cpu状态到off-cpu状态的统计分析工具。 +# 使用说明 +``` +USAGE: offcputime [--help] [-p PID | -u | -k] [-f] [-m MIN-BLOCK-TIME] [-M +MAX-BLOCK-TIME] [--state] [--perf-max-stack-depth] [--stack-storage-size] +[duration] + -k, --kernel-threads-only Kernel threads only (no user threads) + -m, --min-block-time=MIN-BLOCK-TIME + the amount of time in microseconds over which we + store traces (default 1) + -M, --max-block-time=MAX-BLOCK-TIME + the amount of time in microseconds under which we + store traces (default U64_MAX) + -p, --pid=PID Trace this PID only + --perf-max-stack-depth=PERF-MAX-STACK-DEPTH + the limit for both kernel and user stack traces + (default 127) + --stack-storage-size=STACK-STORAGE-SIZE + the number of unique stack traces that can be + stored and displayed (default 1024) + --state=STATE filter on this thread state bitmask (eg, 2 == + TASK_UNINTERRUPTIBLE) see include/linux/sched.h + -t, --tid=TID Trace this TID only + -f, --folded output folded format + -u, --user-threads-only User threads only (no kernel threads) + -v, --verbose Verbose debug output + -?, --help Give this help list + --usage Give a short usage message + -V, --version Print program version +``` +# 使用举例 +## 运行说明 +下面的例子使用offcputime采样30秒，采样的结果存放在当前目录offcputime.fl文件中 +``` +$sudo sysak offcputime -f -u 30 > offcputime.fl +git clone https://github.com/brendangregg/FlameGraph.git +./flamegraph.pl --color=io --title="Off-CPU Time Flame Graph" --countname=us offcputime.fl > offcputime.svg + +``` diff --git a/source/tools/detect/sched/offcpu/offcputime.c b/source/tools/detect/sched/offcpu/offcputime.c index 7a513a60..74a00bb4 100644 --- a/source/tools/detect/sched/offcpu/offcputime.c +++ b/source/tools/detect/sched/offcpu/offcputime.c @@ -196,6 +196,87 @@ static void sig_handler(int sig) { } +static void print_map_fold(struct ksyms *ksyms, struct syms_cache *syms_cache, + struct offcputime_bpf *obj) +{ + struct key_t lookup_key = {}, next_key; + const struct ksym *ksym; + const struct syms *syms; + const struct sym *sym; + int err, i, ifd, sfd, kern, usr; + __u64 *ip, *ip_k; + struct val_t val; + + ip = calloc(env.perf_max_stack_depth, sizeof(*ip)); + if (!ip) { + fprintf(stderr, "failed to alloc ip\n"); + return; + } + ip_k = calloc(env.perf_max_stack_depth, sizeof(*ip)); + if (!ip_k) { + fprintf(stderr, "failed to alloc ip_k\n"); + free(ip); + return; + } + kern = usr = 0; + ifd = bpf_map__fd(obj->maps.info); + sfd = bpf_map__fd(obj->maps.stackmap); + while (!bpf_map_get_next_key(ifd, &lookup_key, &next_key)) { + err = bpf_map_lookup_elem(ifd, &next_key, &val); + if (err < 0) { + fprintf(stderr, "failed to lookup info: %d\n", err); + goto cleanup; + } + lookup_key = next_key; + if (val.delta == 0) + continue; + + printf("%s;", val.comm); + if (next_key.user_stack_id == -1) + goto print_kstack; + + if (bpf_map_lookup_elem(sfd, &next_key.user_stack_id, ip) != 0) { + printf("[Missed User Stack];"); + printf("-"); + goto print_kstack; + } + + syms = syms_cache__get_syms(syms_cache, next_key.tgid); + if (!syms) { + //fprintf(stderr, "failed to get syms\n"); + goto print_kstack; + } + for (i = 0; i < env.perf_max_stack_depth && ip[i]; i++) { + } + usr = i - 1; + + for (i = usr; i >= 0; i--) { + sym = syms__map_addr(syms, ip[i]); + if (sym) + printf("%s;", sym->name); + else + printf("[unknown];"); + } + printf("-;"); +print_kstack: + if (bpf_map_lookup_elem(sfd, &next_key.kern_stack_id, ip_k) != 0) { + printf("[Missed Kernel Stack];"); + goto skip_kstack; + } + for (i = 0; i < env.perf_max_stack_depth && ip_k[i]; i++) { + } + kern = i - 1; + for (i = kern; i >= 0; i--) { + ksym = ksyms__map_addr(ksyms, ip_k[i]); + printf("%s;", ksym ? ksym->name : "Unknown"); + } +skip_kstack: + printf(" %lld\n", val.delta); + } +cleanup: + free(ip); +} + static void print_map(struct ksyms *ksyms, struct syms_cache *syms_cache, struct offcputime_bpf *obj) { @@ -341,7 +422,7 @@ int main(int argc, char **argv) */ sleep(env.duration); - print_map(ksyms, syms_cache, obj); + print_map_fold(ksyms, syms_cache, obj); cleanup: offcputime_bpf__destroy(obj); -- Gitee From 475aad7c093461d6586c335d47737ec580e6f74d Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Thu, 11 May 2023 15:03:45 +0800 Subject: [PATCH 2/2] offcputime: Add folded option and fix a miss off ; Signed-off-by: Hailong Liu --- source/tools/detect/sched/offcpu/offcputime.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/source/tools/detect/sched/offcpu/offcputime.c b/source/tools/detect/sched/offcpu/offcputime.c index 74a00bb4..68742a29 100644 --- a/source/tools/detect/sched/offcpu/offcputime.c +++ b/source/tools/detect/sched/offcpu/offcputime.c @@ -16,6 +16,7 @@ static struct env { pid_t pid; pid_t tid; + bool folded; bool user_threads_only; bool kernel_threads_only; int stack_storage_size; @@ -28,6 +29,7 @@ static struct env { } env = { .pid = -1, .tid = -1, + .folded = false, .stack_storage_size = 1024, .perf_max_stack_depth = 127, .min_block_time = 1, @@ -48,6 +50,7 @@ const char argp_program_doc[] = "EXAMPLES:\n" " offcputime # trace off-CPU stack time until Ctrl-C\n" " offcputime 5 # trace for 5 seconds only\n" +" offcputime -f 5 # trace for 5 seconds and foled the output\n" " offcputime -m 1000 # trace only events that last more than 1000 usec\n" " offcputime -M 10000 # trace only events that last less than 10000 usec\n" " offcputime -p 185 # only trace threads for PID 185\n" @@ -62,6 +65,7 @@ const char argp_program_doc[] = static const struct argp_option opts[] = { { "pid", 'p', "PID", 0, "Trace this PID only" }, { "tid", 't', "TID", 0, "Trace this TID only" }, + { "folded-output", 'f', NULL, 0, "Folded output" }, { "user-threads-only", 'u', NULL, 0, "User threads only (no kernel threads)" }, { "kernel-threads-only", 'k', NULL, 0, @@ -113,6 +117,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'k': env.kernel_threads_only = true; break; + case 'f': + env.folded = true; + break; case OPT_PERF_MAX_STACK_DEPTH: errno = 0; env.perf_max_stack_depth = strtol(arg, NULL, 10); @@ -237,7 +244,7 @@ static void print_map_fold(struct ksyms *ksyms, struct syms_cache *syms_cache, if (bpf_map_lookup_elem(sfd, &next_key.user_stack_id, ip) != 0) { printf("[Missed User Stack];"); - printf("-"); + printf("-;"); goto print_kstack; } @@ -266,7 +273,7 @@ print_kstack: for (i = 0; i < env.perf_max_stack_depth && ip_k[i]; i++) { } kern = i - 1; - for (i = kern; i >= 0; i--) { + for (i = kern; i > 0; i--) { ksym = ksyms__map_addr(ksyms, ip_k[i]); printf("%s;", ksym ? ksym->name : "Unknown"); } @@ -421,9 +428,10 @@ int main(int argc, char **argv) * be "handled" with noop by sig_handler). */ sleep(env.duration); - - print_map_fold(ksyms, syms_cache, obj); - + if (env.folded) + print_map_fold(ksyms, syms_cache, obj); + else + print_map(ksyms, syms_cache, obj); cleanup: offcputime_bpf__destroy(obj); syms_cache__free(syms_cache); -- Gitee