From 40662baa35d0ac3e251c1fdb6452350b3a709fca Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Tue, 23 May 2023 11:29:57 +0000 Subject: [PATCH 1/2] unity_nosched: Add stacktrace Signed-off-by: Hailong Liu --- .../collector/plugin/unity_nosched/Makefile | 2 +- .../plugin/unity_nosched/stacktrace.c | 132 ++++++++++++++++++ .../plugin/unity_nosched/unity_nosched.c | 105 +++++++++++++- .../plugin/unity_nosched/unity_nosched.h | 14 +- 4 files changed, 240 insertions(+), 13 deletions(-) create mode 100644 source/tools/monitor/unity/collector/plugin/unity_nosched/stacktrace.c diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile index 3f88651e..7ce32fc5 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/Makefile @@ -2,7 +2,7 @@ newdirs := $(shell find ./ -type d) bpfsrcs := unity_nosched.bpf.c -csrcs := unity_nosched.c +csrcs := unity_nosched.c stacktrace.c so := libunity_nosched.so include ../bpfso.mk diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/stacktrace.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/stacktrace.c new file mode 100644 index 00000000..70b7b2ae --- /dev/null +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/stacktrace.c @@ -0,0 +1,132 @@ +#include +#include +#include +//#include +//#include +//#include +#include +#include +//#include +#include +#include +#include +#include +//#include "unity_nosched.h" + +#define MAX_SYMS 300000 +#define PERF_MAX_STACK_DEPTH 32 +struct ksym { + long addr; + char *name; +}; + +static int sym_cnt; + +static int ksym_cmp(const void *p1, const void *p2) +{ + return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; +} + +int load_kallsyms(struct ksym **pksyms) +{ + struct ksym *syms; + FILE *f = fopen("/proc/kallsyms", "r"); + char func[256], buf[256]; + char symbol; + void *addr; + int i = 0; + + if (!f) + return -ENOENT; + + syms = malloc(MAX_SYMS * sizeof(struct ksym)); + if (!syms) { + fclose(f); + return -ENOMEM; + } + + while (!feof(f)) { + if (!fgets(buf, sizeof(buf), f)) + break; + if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) + break; + if (!addr) + continue; + syms[i].addr = (long) addr; + syms[i].name = strdup(func); + i++; + if (i > MAX_SYMS) { + printf("Warning: no space on ksym array!\n"); + break; + } + } + fclose(f); + sym_cnt = i; + if (sym_cnt == 0) { + printf("/proc/kallsyms has no valid address\n"); + return -ENOENT; + } + qsort(syms, sym_cnt, sizeof(struct ksym), ksym_cmp); + *pksyms = syms; + return 0; +} + +struct ksym *nosched_ksym_search(long key, struct ksym *syms) +{ + int start = 0, end = sym_cnt; + int result; + + /* kallsyms not loaded. return NULL */ + if (sym_cnt <= 0) + return NULL; + + while (start < end) { + size_t mid = start + (end - start) / 2; + + result = key - syms[mid].addr; + if (result < 0) + end = mid; + else if (result > 0) + start = mid + 1; + else + return &syms[mid]; + } + + if (start >= 1 && syms[start - 1].addr < key && + key < syms[start].addr) + /* valid ksym */ + return &syms[start - 1]; + + /* out of range. return _stext */ + return &syms[0]; +} + +static void print_ksym(__u64 addr, struct ksym *psym, FILE *filep) +{ + struct ksym *sym; + + if (!addr) + return; + sym = nosched_ksym_search(addr, psym); + if (sym) + fprintf(filep, "<0x%llx> %s\n", addr, sym->name); + else + fprintf(filep, "<0x%llx>: missing symbols\n", addr); +} + +void print_stack(int fd, __u32 ret, struct ksym *syms, FILE *filep) +{ + int i; + __u64 ip[PERF_MAX_STACK_DEPTH] = {}; + + if (!filep) + filep = stdout; + if (bpf_map_lookup_elem(fd, &ret, &ip) == 0) { + for (i = 0; i < PERF_MAX_STACK_DEPTH - 1; i++) + print_ksym(ip[i], syms, filep); + } else { + if ((int)(ret) < 0) + fprintf(filep, "<0x0000000000000000>:error=%d\n", (int)(ret)); + } +} + diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c index e8915d72..203a3d45 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -5,26 +5,77 @@ #include #include #include +#include #include #include "unity_nosched.h" #include "sched_jit.h" #include "unity_nosched.skel.h" #include "../../../../unity/beeQ/beeQ.h" +#define MAX_NOSCHED_ELEM 1024 #ifdef __x86_64__ #define TIF_NEED_RESCHED 3 #elif defined (__aarch64__) #define TIF_NEED_RESCHED 1 #endif +char log_dir[] = "/var/log/sysak/nosched"; +char filename[] = "/var/log/sysak/nosched/unity_nosched.log"; + +struct evelem { + struct event e; + time_t t; +}; + +struct stackinfo { + int stackfd, fhead, bhead; + struct evelem elem[MAX_NOSCHED_ELEM]; +}; + +FILE *logfp; unsigned int nr_cpus; struct sched_jit_summary summary, prev; +struct stackinfo globEv; + +static struct ksym *ksyms; +void print_stack(int fd, __u32 ret, struct ksym *syms, FILE *fp); +int load_kallsyms(struct ksym **pksyms); + +static int prepare_directory(char *path) +{ + int ret; + + ret = mkdir(path, 0777); + if (ret < 0 && errno != EEXIST) + return errno; + else + return 0; +} + +void flush_to_file(struct stackinfo *ev, int index) +{ + int tmphd; + + if (ev->fhead == ev->bhead) + return; + if (index < 0) + index = ev->bhead; + printf("will flush: fhead=%d, bhead=%d, index=%d\n", + ev->fhead, ev->bhead, index); + tmphd = ev->fhead; + ev->fhead = index; + if (index < tmphd) + index = index + MAX_NOSCHED_ELEM; + for (; tmphd < index; tmphd++) { + int i = tmphd%MAX_NOSCHED_ELEM; + print_stack(globEv.stackfd, globEv.elem[i].e.ret, ksyms, logfp); + } +} static void update_summary(struct sched_jit_summary* summary, const struct event *e) { summary->num++; summary->total += e->delay; - if (e->delay < 10) { summary->less10ms++; } else if (e->delay < 50) { @@ -40,6 +91,23 @@ static void update_summary(struct sched_jit_summary* summary, const struct event } } +void record_stack(struct event *e) +{ + time_t t; + int idex = globEv.bhead; + + time(&t); + globEv.elem[idex].e = *e; + globEv.elem[idex].t = t; + idex++; + if (idex > MAX_NOSCHED_ELEM) { + /*flush_to_file(&globEv, 0);*/ + globEv.bhead = 0; + } else { + globEv.bhead = idex; + } +} + void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) { struct event e; @@ -49,11 +117,15 @@ void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) e.delay = e.delay/(1000*1000); if (e.cpu > nr_cpus - 1) return; + if (e.delay > 10) { /* we use 100ms to control the frequency of output */ + printf("delay=%lld\n", e.delay); + record_stack(&e); + printf("record_stack over\n"); + } if (e.exit != 0) update_summary(&summary, &e); } - DEFINE_SEKL_OBJECT(unity_nosched); static void bump_memlock_rlimit1(void) @@ -71,10 +143,28 @@ static void bump_memlock_rlimit1(void) int init(void *arg) { - int err, argfd, args_key; struct args args; + int err, ret, argfd, args_key; + err = prepare_directory(log_dir); + if (err) { + printf("prepare_dictory fail\n"); + return err; + } + logfp = fopen(filename, "w+"); + if (!logfp) { + int ret = errno; + fprintf(stderr, "%s :fopen %s\n", + strerror(errno), filename); + return ret; + } bump_memlock_rlimit1(); + ksyms = NULL; + err = load_kallsyms(&ksyms); + if (err) { + fprintf(stderr, "Failed to load kallsyms\n"); + return err; + } unity_nosched = unity_nosched_bpf__open(); if (!unity_nosched) { err = errno; @@ -92,8 +182,8 @@ int init(void *arg) argfd = bpf_map__fd(unity_nosched->maps.args_map); args_key = 0; args.flag = TIF_NEED_RESCHED; - args.thresh = 50*1000*1000; /* 50ms */ - + args.thresh = 10*1000*1000; /* 50ms */ + globEv.stackfd = bpf_map__fd(unity_nosched->maps.stackmap); err = bpf_map_update_elem(argfd, &args_key, &args, 0); if (err) { fprintf(stderr, "Failed to update flag map\n"); @@ -124,7 +214,7 @@ int init(void *arg) DESTORY_SKEL_BOJECT(unity_nosched); return err; } - + globEv.bhead = globEv.fhead = 0; printf("unity_nosched plugin install.\n"); return 0; } @@ -146,11 +236,14 @@ int call(int t, struct unity_lines *lines) unity_set_value(line, 4, "gt500ms", delta(summary,less1s)); unity_set_value(line, 5, "gt1s", delta(summary,plus1s)); prev = summary; + flush_to_file(&globEv, -1); return 0; } void deinit(void) { printf("unity_nosched plugin uninstall.\n"); + if (ksyms) + free(ksyms); DESTORY_SKEL_BOJECT(unity_nosched); } diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h index 4f411ad9..0d6173a8 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.h @@ -1,11 +1,10 @@ - - -#ifndef BPF_SAMPLE_H -#define BPF_SAMPLE_H +#ifndef BPF_UNITY_NOSCHED_H +#define BPF_UNITY_NOSCHED_H #define MAX_MONI_NR 1024 - +#define MAX_NOSCHED_ELEM 1024 #define PERF_MAX_STACK_DEPTH 32 + struct args { int flag; unsigned long long thresh; @@ -88,5 +87,8 @@ int call(int t, struct unity_lines *lines); void deinit(void); #endif - +struct ksym { + long addr; + char *name; +}; #endif -- Gitee From 4130674882133ac3a40c2f7c06be62aa70c0bd5d Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Wed, 24 May 2023 02:44:05 +0000 Subject: [PATCH 2/2] unity_nosched: record log to file Signed-off-by: Hailong Liu --- .../collector/plugin/unity_nosched/unity_nosched.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c index 203a3d45..e3414b71 100644 --- a/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c +++ b/source/tools/monitor/unity/collector/plugin/unity_nosched/unity_nosched.c @@ -67,7 +67,15 @@ void flush_to_file(struct stackinfo *ev, int index) if (index < tmphd) index = index + MAX_NOSCHED_ELEM; for (; tmphd < index; tmphd++) { + char ts[64]; + struct tm *tm; int i = tmphd%MAX_NOSCHED_ELEM; + tm = localtime(&globEv.elem[i].t); + strftime(ts, sizeof(ts), "%F %H:%M:%S", tm); + fprintf(logfp, "%-18.6f %-5d %-15s %-8d %-10llu %-21s\n", + ((double)globEv.elem[i].e.stamp/1000000000), globEv.elem[i].e.cpu, + globEv.elem[i].e.comm, globEv.elem[i].e.pid, globEv.elem[i].e.delay, + (globEv.elem[i].e.exit==globEv.elem[i].e.stamp)?"(EOF)":ts); print_stack(globEv.stackfd, globEv.elem[i].e.ret, ksyms, logfp); } } @@ -117,7 +125,7 @@ void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) e.delay = e.delay/(1000*1000); if (e.cpu > nr_cpus - 1) return; - if (e.delay > 10) { /* we use 100ms to control the frequency of output */ + if (e.delay > 100) { /* we use 100ms to control the frequency of output */ printf("delay=%lld\n", e.delay); record_stack(&e); printf("record_stack over\n"); @@ -182,7 +190,7 @@ int init(void *arg) argfd = bpf_map__fd(unity_nosched->maps.args_map); args_key = 0; args.flag = TIF_NEED_RESCHED; - args.thresh = 10*1000*1000; /* 50ms */ + args.thresh = 50*1000*1000; /* 50ms */ globEv.stackfd = bpf_map__fd(unity_nosched->maps.stackmap); err = bpf_map_update_elem(argfd, &args_key, &args, 0); if (err) { -- Gitee