From 25e2fb9c2f1be06b82e8b41b447d7496bebcee6b Mon Sep 17 00:00:00 2001 From: ZouTao Date: Mon, 22 May 2023 18:08:04 +0800 Subject: [PATCH 1/2] tasktop: add bpf collect sched delay info. --- .../detect/sched/tasktop/bpf/tasktop.bpf.c | 58 ++++++++++++++++++ source/tools/detect/sched/tasktop/tasktop.c | 1 + .../detect/sched/tasktop/tasktopSelftest/test | Bin 8528 -> 8920 bytes .../sched/tasktop/tasktopSelftest/test.c | 26 ++++++-- 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c b/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c index 60f86d45..6a636016 100644 --- a/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c +++ b/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c @@ -29,6 +29,20 @@ struct { __type(value, u64); } cnt_map SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1024); + __type(key, u32); + __type(value, u64); +} sched_delay_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 4096); + __type(key, u64); + __type(value, u64); +} sched_info_map SEC(".maps"); + struct trace_event_sched_process_fork_args { struct trace_entry ent; char parent_comm[16]; @@ -77,4 +91,48 @@ int handle__sched_process_fork(struct trace_event_sched_process_fork_args *ctx) return 0; } +SEC("kprobe/sched_info_map_queued") +int handle_enqueue() { + u64 now, pid; + + pid = bpf_get_current_pid_tgid() & (((u64)1 << 32) - 1); + now = bpf_ktime_get_ns(); + bpf_map_update_elem(&sched_info_map, &pid, &now, BPF_ANY); + return 0; +} + +static void acc_delay() { + u32 cpu_id; + u64 *delay; + u64 zero = 0; + u64 *enqueue_ts, now, pid, delta; + pid = bpf_get_current_pid_tgid() & (((u64)1 << 32) - 1); + enqueue_ts = bpf_map_lookup_elem(&sched_info_map, &pid); + + if (enqueue_ts) { + cpu_id = bpf_get_smp_processor_id(); + now = bpf_ktime_get_ns(); + delta = now - *enqueue_ts; + + delay = bpf_map_lookup_elem(&sched_delay_map, &cpu_id); + if (delay) { + __sync_fetch_and_add(delay, delta); + } else { + bpf_map_update_elem(&sched_delay_map, &cpu_id, &delta, BPF_ANY); + } + } +} + +SEC("kprobe/rq_sched_info_map_dequeued") +int handle_dequeue() { + acc_delay(); + return 0; +} + +SEC("kprobe/rq_sched_info_map_arrive") +int handle_arrive() { + acc_delay(); + return 0; +} + char LICENSE[] SEC("license") = "GPL"; diff --git a/source/tools/detect/sched/tasktop/tasktop.c b/source/tools/detect/sched/tasktop/tasktop.c index c40b2eb9..ab0a5f80 100644 --- a/source/tools/detect/sched/tasktop/tasktop.c +++ b/source/tools/detect/sched/tasktop/tasktop.c @@ -784,6 +784,7 @@ int main(int argc, char** argv) { fprintf(stderr, "Failed read all pids.\n"); goto cleanup; } + printf("procnum=%lu\n", proc_num); rec->tasks = calloc(proc_num, sizeof(struct task_record_t*)); /* if prev process info exist produce record*/ diff --git a/source/tools/detect/sched/tasktop/tasktopSelftest/test b/source/tools/detect/sched/tasktop/tasktopSelftest/test index 01333afa8e8b448cf3ccc881ab436b8c3a54eb0e..e1bb58981f6da1a537e86ca3763a2f7bb4723753 100755 GIT binary patch literal 8920 zcmeHMeQaCR6~DIg)i!lfRv?tF_4rcD7SonC3uSC}61OiXEm@l|K9;_?vC|lm*s`A? zWfX$kcJoYF2ip`)lVE>r;t!@VhEUc?txGyGw*FBn0om3`S+$O&4O7;35Osy@cka8# zd4914o3#IStoP16zkANP=YIO$`<@K6Y;igqg2^eqEJ(_h`6Oh%g}Co}DJra9ED}Dk zN_r;;W=U1^HU zh44s#>?l^@mLhnknNZwNaSSNVb|jKrjk2pz`phR(V9dFGR5yCsl)pBo4-&~fVxmdD z=Q-&TwVpLXl&)MNMR1=?FmE$q!JO^B3OiXpmH$bsS}E*M@_Fm|HO=*csLlMPFW|EpmzZ>!a4DwXry0sN8=BRM}rX~773AI7n#(F@Ib^6 zv80g@K_eOwiQaHHCK5)xt1l*Wo%HmC5sVvpUl2}oeNSJsUm?2K+S1(6sBiFY@NPng z7)CDnqZFrUOgQk{3{K<4gD1HnvWP~;Q%Dn0K8*t&uV%h8mp@z8Nou)uND6q>RCmf` zIo@Q?-BT2FX#U z1z(&)h3K*1YUz-=D|5tWi4oAYd(ZykgWep4;R^*4)h{cvr3VAw_Zz zWll}Zfo{IM2r@O{)JZ8{AeowODl6r4BvX@3y)5N3BvVsOy&&a3l1xoBbx6vmNT#Nl z8kF+yNTw#4ib?r(k}F8=l=5McsR^c9rThxXRP`wzsByQhRHuyS`Nh1!Gk%gyro=tMM-b{kfAJQ{R9|)oL-704E-4cpD2`A?@?cle5syv|mBF>HXiIAt7)@8_w)MKxxEr<5*yp zRL26BXxwH3mon|Mj+5!Y%u#|Thh=G39I1U*o_yjL?Dun4FL}|`TlN2t>j^ErZ&r3% zM=neZdvZA35bC zR^`*@pOej#nyyaoyMq3oL4};gvZE6i`jd@N9l@BHJO2>c9YZ9GbwC`MqiKO+n~;sB zs+1L&{1WLkSoKPmn0hPo^!m`X=T}(t$kiQC0n^b+Z;txE=1>o`lT*Km1Qz@4B-UX0+H`x;!`cBk)%dU@!QOQOV=r&!LtyAwG|msKn^n z4E_c1SHN4qo6wTI;ID$e3QqeRBFAGpgkzxEaeZZZ#UV#|HQ}_k^}+wGq@bvRUws=ihDmp!}Eclp%k>nonsE<)6T_3;E9y!1m4| z>40LK#&Z(+VajiCSAWMDaMwKJ3b;K(B?0%Er%N^Wx`Sn!yFOLE&F$-T*ZbY;{O&ak zZcl@|2Ko)|3faCJ(ay9cV3GJg{f8|uSniV+or<$LnpBdVZ&rAv;uEDlnOdsk<5*va zu9I)P)Om@Ik39ZXsQe0*PveD%^Z)trT$JQtr;k|f%s1{bW!_tjI8B-NLPO9cT4FL$ zqxjW|Gt2Gq@b^x{W2Zt@nCBty)vlFVLX4OAWQy^uk~wZJkJb<(ma#m@`V1<4EDuuV za%Ys>t@2Il|NWcKvG$U#Df3C_7RBon-=g@vigzpih~fto|AFE^Rs1!@$?4X{#?78J zIQgyhZ1mQ9H+eR!Tfb>t-Gl=#B&S%wNY3XF%9 zz^_94(`k{2*J0Yrp>-C}5O5DxbNl$wis(N8yc+uUadnH-UpViA5??ClY(OOUT;Q~q zEEG>h`M1Y2TtxpR;2v^pqhAAG=ByIqw)y^B!Vv|p10~AtUEtJibi5`C%eN4xz^n1E zPse2|eOKvcZR7tv;G}PFpNmSL&h13c$~X3R3jdiJ&leP4j*ctTJ}ZmhbwzNk2>t-@ zWzL!f_3D=TOU3gQJ{ifdiYHdY&tGKx3)k(G^0P(NcR=}hridTW6*m$_vb)>cg&ViK z8+Yg}%{$w4U4+8%@SaG*2*-7!PnYMK1T;cXeNS(6cd%Cv8PRw`4<-jhSF|tI8#clr zZ{6*iHWgOVyCeM(Js6J%_v+z(BfeL3$Af)gJ(TS0+Y1wmq(jrlQ>D|EjBm>h|F(c0 z*xrNy&1gBDXA=^7)BW51+nO7pWyz7Zi-53I&;y!odefSA2z_hIJq`XA{hlpbb_Uw? zHh)7)fJ%^el3j_Uj6QeAX1O*zbKWg+u`uDKE`8!UVfldbI zg{Qg}wJYS^h@vwRQN1VFALFZL*AW8DA)3K)vSXai$}Y{ zi3DHP=9kA;ymY6VpF{Vz`3T+Z<|De^wyjb1Rc|yJ)8zqKUi#)M%FOwLbAG1qCieCj z!QJ3S+~hr+iu7Z2#)KE`8}|Aenrn^V9wqJRPkMJJBlw(v8)z>{J;6ke@P_vG!=K5G zxS8`vIG%_^`||`HdGT;>kPK8R)@ul_Y)r2a9zdE#H1eXdo4w&4HHLdauw}`#FbA_~ z!Rd#(pa$>3zDO5}j=~{cURjFi5j`fnm|K0AFKQP5_l))jSTb_|OT543?`3>m;u1*L zN;g8FGYXawNxTo{?@=CwGvoE0<@HEd?Rme<-^p%bMaAv*O~60EnTE?BcKW2|umU*# zQi;x=`;;i?`v{%k*nY(6gM|B71$b12%Yad`f-*rhCKLC9(;1KLd7nP64t{)2RO-?P zr-|>vLuWL$=ly)QswkJk@v|QDhha}=K9+gkKd9_|$S0!sNr&SnjEMGcq{{YuKFAg+ zpY1uW{mQ-poQTh@q@ z!>lRg-u!)s`S*}xwa+U1tg?673S>KW_(PjLpL33@ivm8Mv%bxpctP27`}02Nq_W@4 zhKh6hG5n&(C#_ z6VRbODBJUSt>RPKkHeNiWP3grReyp#OeFmoZ|Z8c=kp(bho4aO=kdvL^Emh$Y^~+< zIdN8PtXW?ch8N4s&)e*C=SfvSj13iMJLZ>c_PkGzDSLZ=+qzR+hCTV>`tv^7_>}f^ zKqa5+@K}lMYpo#CT)vg_*te>_sLHQU7htrGd+~6+dHtdJ#j#uMUse}sFW8)OxvWdy R@eA3vs)dKwFV?r){~NiAB5VKv literal 8528 zcmeHMYitzP6~4QRUtrb?q2K}yLlYu{>WwjvI)ulp?Zsncz%?d)q-4B!*Y?tVWp@@# zTONra)ygGxB_a{ERs5r>lBlZq(ME0(0WJiqFO~dgld7mfX{l^XB1c8YQk`tixpU5X zX1rFb{^&2xYUbSY-E+=8_c{08Jr!)*?RL2YlUwW*DAkp!5;EUHw6ByRVGW`}sA9EP zB}xG)fk%}&$jub!fHob)`v7r@itHh~7WDih3Cn=q7N5=V#}%bO*Yk|w{m&=i>q zp-2GjFxGOf6oGfi1oJ+MBSoC;P(-_Gva2RL=C>#?=3GCl8$O-nuhXr9gtCf^qMGHX zi`0JG2ur&PQ|_!#WrBH|2@B?I_a(5yIR9JnkzxmxH($GOzcHukipOKo!&^5$9t+pT zqKRyOZGXeo+O3=YnWVp7)*Jn!Zp*%dvLtRtMM^?o|6^B`!Gmo^)rFKvsLy3#9kI{R z=0DYO3Fs_^$4RXK6vHvPV)(v!@Ls?bc$`!ZKruhcJb3dwcNHUrbJ<0Sp#IaD+NJYbF*o`JzMWjD!h*Z|dh>Vf$j;DmKW2T-lLTN*f zhrm+Td*jIjA-ZU3Yi(@O>;3iqEfAL*dH{cz!nD~(F0l;|>~960?8+!~rAHivG?GjH z%zTv1e0L##x~dD|!a5=acvbtlWD>^*j{^i03reX0qjF73$g7aL#yao$yT~AaC zZXIiy1;?0}+ATQGea!oc1-FiEZQyRXHdu0Ep(3=iSB#SJr$Jd8xK@7E^qAk0hOGQL z1Kw3@R7j!RhnW+%@<8j3po~p2aYM?7QO2g2xFY4Rp^Qy1aZ$=0C}Y(phNZj*WvueV zkdzxy#;Q)7mhv`~v5FHZDc7TnRh#GndETz2oHy1%}y*gIezP_(=%lddYVHaUC)>o%)k6 zqN4WPT?LwOEI5NPOzs&k0mIy{27@!&h1~hGw?4_|WqR<8OrO1yePk?n4|T_aQ`i@S z2k#9+>P9Yj_cFrcKmRnJ9|+F422RYJI(cy{IF0D!+QHzoc7eRSiO?l{<;KRtQhyrs zr%#>y^F$iTmx1L@Ov^eSx=L~Spes2}zjp4Wsee&y+UPB4rW?ORhj0F%Of_9=;ClqZ zFU0aYj^$w)i#B-X9dse%K;=oPOo7~-JTHqKxa-TEn9*|CDX5RzSaP`rI(<9^sv{6+ ze)50@wbur&z|a<#T5v#sr)MD>TB=gUKfVL?9<%E0CB0TtZ=tCdhlY@;Uqe@^BJkxv zE+2TRHFrO7ur>F`Kxf;zb?qgJIH;YgMSh@d{q2o^(MCV<;LIBT(M&$Co&C^wn8#CF z?oM0oelz&UFT1G?Ty<$rOlI%kgnRl>;Ol`yfoB5x)uGw(_fMFIa*ooJbjei?22>{0 z8`-89k&K~q$C8PNl1!VDTx=y|uDh^at@1pnK`-E>*$x4n2YwTH82HbivTp-#fok3a z{uFQ=biW6_8b(qZG}CtAZv#IAd>P~+xxU#UT>U=Rs>-tRAy=6X@zwC8!0!fBAS$je z#G{ap`oqxI!>H@^?e(LGMsIunofxtk<{4?-$K>jgnKBj>%|HtsW2KnDYfc?9VN(Wf>HF(OQJtCOj==FWi z9rRYe=m~n2a|?prn(vlq-nth`HE+W}*I~y&&oHRx4L9% z70bj|68BL)t_w)Gh9E64ndo<`GKCu!q*pzv4*Dn>t`ppBUTkdbWe!1XrW5+85k1spEurSZ;_#!bx?Qh2`3->p6e6is9X~(@n zp?Pb^s|4RW*m0j2E~>wzFyHO^O9jt!JN^*dt8`f*nfAcfo6m*TOpkzBO!L)_FBe@! z@f8Bk5l-?5o|UsXxXvvQALnNgGllD%8& zSt#}A&X=IXD+L~~kmMd1c>6s5ljPqyKEFMW{t(~_I(E=2fG=?`7k!TL{8Plm?EGCo zc5eZW?S{v4q=*IOw zMbM$e?#(_2IM&x$?s}4k;f z{MCJdy{%359C^715PWj=phh0F<_@8^wC!&UwCVeI?>-Rh)H?%>Z9()ZuLQd@Ss8KR zUeUZ}R8{MZAYLWf5j_$P86g2Jlt>zV(L^u%o4sOm80ctV-Z#>%q*K9mAqj1pN$P!} zL^u`^dh33$3`Y}sHWPvR&xVdI1M=ZahFr_bO}Y`a7r^(ZcyVga!Rt~xqU)V|o2dC? z$z)2GXMdTRy#TPA2!G~y+z1^8Zlq1#$Ej!nhE_`Wp{*kRKx1pI5$YBGzEGx5_`}B& zU}$n9E&RQStp8{vorxwBHbIBHbR-r+14^Z0hVaW4@EehSNaN6hyrk?Kf25BFTVEJ# zSu!omQD<6k8XSk>(QYU;3BKUvmj#&4poN7W27esJv~9lsuL7PEU@8~ZdEU?Q|3rLF zVE?FVrAI(0g9oO!B;NP&|1b*S%y_+Ic_SoB;9>iIw+fPgeWGMyyUE~NYZCAKS;p@? zo;}!}_mLw6OpzUyfw8ka??`q5hG!GD=kw01bYOWN@~q^ne+y*d*@EqPU%G(;>J4;?}smt!TW@x zh-W&dJ+ANACZ+Ho@%c1`0R!>5)0AewoOb1}THSI7|mr?L{aA7kH#1jf(yy#Mbc z`!-f2?kxWmhdrMYhRMDYom #include +void create_process(int n) { + int pid[n]; + int i; + for (i = 0; i < n; i++) { + if ((pid[i] = fork()) == 0) { + sleep(120); + } + } + + for (i = 0; i < n; i++) { + waitpid(pid[i], 0, 0); + } +} + void loop_fork() { while (1) { + int i; int pid[128]; - for (int i = 0; i < 128; i++) { + for (i = 0; i < 128; i++) { if ((pid[i] = fork()) == 0) { int a = 0; int b = 1; @@ -19,7 +34,7 @@ void loop_fork() { // printf("fork.\n"); } - for (int i = 0; i < 128; i++) { + for (i = 0; i < 128; i++) { waitpid(pid[i], 0, 0); } } @@ -30,12 +45,13 @@ void *do_nothing(void *arg) { return 0; } void loop_clone() { while (1) { pthread_t pid[128]; - for (int i = 0; i < 128; i++) { + int i; + for (i = 0; i < 128; i++) { pthread_create(&pid[i], 0, do_nothing, 0); // printf("fork.\n"); } - for (int i = 0; i < 128; i++) { + for (i = 0; i < 128; i++) { pthread_join(pid[i], 0); } } @@ -51,5 +67,7 @@ int main(int argc, char **argv) { loop_clone(); } else if (!strcmp(argv[1], "fork")) { loop_fork(); + } else { + create_process(atoi(argv[1])); } } \ No newline at end of file -- Gitee From b98ddbc83609c3d7a8b029a03941a6d90c1be49d Mon Sep 17 00:00:00 2001 From: ZouTao Date: Tue, 23 May 2023 15:23:55 +0800 Subject: [PATCH 2/2] tasktop: supoort collect cpu sched delay metric --- .../detect/sched/tasktop/bpf/tasktop.bpf.c | 58 ------ source/tools/detect/sched/tasktop/tasktop.c | 194 ++++++++++++------ source/tools/detect/sched/tasktop/tasktop.h | 3 + 3 files changed, 138 insertions(+), 117 deletions(-) diff --git a/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c b/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c index 6a636016..60f86d45 100644 --- a/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c +++ b/source/tools/detect/sched/tasktop/bpf/tasktop.bpf.c @@ -29,20 +29,6 @@ struct { __type(value, u64); } cnt_map SEC(".maps"); -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 1024); - __type(key, u32); - __type(value, u64); -} sched_delay_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 4096); - __type(key, u64); - __type(value, u64); -} sched_info_map SEC(".maps"); - struct trace_event_sched_process_fork_args { struct trace_entry ent; char parent_comm[16]; @@ -91,48 +77,4 @@ int handle__sched_process_fork(struct trace_event_sched_process_fork_args *ctx) return 0; } -SEC("kprobe/sched_info_map_queued") -int handle_enqueue() { - u64 now, pid; - - pid = bpf_get_current_pid_tgid() & (((u64)1 << 32) - 1); - now = bpf_ktime_get_ns(); - bpf_map_update_elem(&sched_info_map, &pid, &now, BPF_ANY); - return 0; -} - -static void acc_delay() { - u32 cpu_id; - u64 *delay; - u64 zero = 0; - u64 *enqueue_ts, now, pid, delta; - pid = bpf_get_current_pid_tgid() & (((u64)1 << 32) - 1); - enqueue_ts = bpf_map_lookup_elem(&sched_info_map, &pid); - - if (enqueue_ts) { - cpu_id = bpf_get_smp_processor_id(); - now = bpf_ktime_get_ns(); - delta = now - *enqueue_ts; - - delay = bpf_map_lookup_elem(&sched_delay_map, &cpu_id); - if (delay) { - __sync_fetch_and_add(delay, delta); - } else { - bpf_map_update_elem(&sched_delay_map, &cpu_id, &delta, BPF_ANY); - } - } -} - -SEC("kprobe/rq_sched_info_map_dequeued") -int handle_dequeue() { - acc_delay(); - return 0; -} - -SEC("kprobe/rq_sched_info_map_arrive") -int handle_arrive() { - acc_delay(); - return 0; -} - char LICENSE[] SEC("license") = "GPL"; diff --git a/source/tools/detect/sched/tasktop/tasktop.c b/source/tools/detect/sched/tasktop/tasktop.c index ab0a5f80..3f181946 100644 --- a/source/tools/detect/sched/tasktop/tasktop.c +++ b/source/tools/detect/sched/tasktop/tasktop.c @@ -28,6 +28,7 @@ time_t btime = 0; u_int64_t pidmax = 0; char* log_path = 0; int nr_cpu; +int* prev_delay; static volatile sig_atomic_t exiting; struct env { @@ -51,7 +52,8 @@ const char argp_program_doc[] = "A light top, display the process/thread cpu utilization in peroid.\n" "\n" - "USAGE: tasktop [--help] [-t] [-p TID] [-d DELAY] [-i ITERATION] [-s SORT] [-f LOGFILE] [-l " + "USAGE: tasktop [--help] [-t] [-p TID] [-d DELAY] [-i ITERATION] [-s SORT] " + "[-f LOGFILE] [-l " "LIMIT]\n" "\n" @@ -62,7 +64,8 @@ const char argp_program_doc[] = " tasktop -d 5 # modify the sample interval.\n" " tasktop -i 3 # output 3 times then exit.\n" " tasktop -l 20 # limit the records number no more than 20.\n" - " tasktop -f a.log # log to a.log (default to /var/log/sysak/tasktop/tasktop.log)\n"; + " tasktop -f a.log # log to a.log (default to " + "/var/log/sysak/tasktop/tasktop.log)\n"; static const struct argp_option opts[] = { {"human", 'H', 0, 0, "Output human-readable time info."}, @@ -73,7 +76,8 @@ static const struct argp_option opts[] = { {"logfile", 'f', "LOGFILE", 0, "Logfile for result, default /var/log/sysak/tasktop/tasktop.log"}, {"sort", 's', "SORT", 0, - "Sort the result, available options are user, sys and cpu, default is cpu"}, + "Sort the result, available options are user, sys and cpu, default is " + "cpu"}, {"limit", 'l', "LIMIT", 0, "Specify the top-LIMIT tasks to display"}, {NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help"}, {}, @@ -97,7 +101,8 @@ static int parse_long(const char* str, long* retval) { long val = strtol(str, &endptr, 10); /* Check for various possible errors */ - if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || (errno != 0 && val == 0)) { + if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || + (errno != 0 && val == 0)) { fprintf(stderr, "Failed parse val.\n"); err = errno; return err; @@ -219,7 +224,44 @@ int swap(void* lhs, void* rhs, size_t sz) { return 0; } -static int read_stat(struct sys_cputime_t* prev_sys, struct sys_cputime_t* now_sys, +static int check_delay(struct sys_record_t* sys_rec) { + FILE* fp = fopen(SCHEDSTAT_PATH, "r"); + int err = 0; + if (!fp) { + fprintf(stderr, "Failed open stat file.\n"); + err = errno; + goto cleanup; + } + + unsigned long long ph; + unsigned long long delay; + char name[64]; + char buf[BUF_SIZE]; + while (fscanf(fp, "%s ", name) != EOF) { + if (!strncmp(name, "cpu", 3)) { + fscanf(fp, "%llu %llu %llu %llu %llu %llu %llu %llu %llu", &ph, &ph, + &ph, &ph, &ph, &ph, &ph, &delay, &ph); + + int cpu_id = atoi(name + 3); + // #ifdef DEBUG + // fprintf(stderr, "cpu_id = %d delay=%llu\n", cpu_id, + // delay); + // #endif + sys_rec->percpu_sched_delay[cpu_id] = delay - prev_delay[cpu_id]; + prev_delay[cpu_id] = delay; + } else { + fgets(buf, BUF_SIZE, fp); + } + } + +cleanup: + if (fp) fclose(fp); + return err; + return 0; +} + +static int read_stat(struct sys_cputime_t* prev_sys, + struct sys_cputime_t* now_sys, struct sys_record_t* sys_rec) { int err = 0; int i = 0; @@ -231,29 +273,37 @@ static int read_stat(struct sys_cputime_t* prev_sys, struct sys_cputime_t* now_s } for (i = 0; i <= nr_cpu; i++) { /*now only read first line, maybe future will read more info*/ - fscanf(fp, "%s %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n", now_sys[i].cpu, &now_sys[i].usr, - &now_sys[i].nice, &now_sys[i].sys, &now_sys[i].idle, &now_sys[i].iowait, - &now_sys[i].irq, &now_sys[i].softirq, &now_sys[i].steal, &now_sys[i].guest, - &now_sys[i].guest_nice); + fscanf(fp, "%s %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n", + now_sys[i].cpu, &now_sys[i].usr, &now_sys[i].nice, + &now_sys[i].sys, &now_sys[i].idle, &now_sys[i].iowait, + &now_sys[i].irq, &now_sys[i].softirq, &now_sys[i].steal, + &now_sys[i].guest, &now_sys[i].guest_nice); if (prev_sys[i].usr == 0) continue; - int now_time = now_sys[i].usr + now_sys[i].sys + now_sys[i].nice + now_sys[i].idle + - now_sys[i].iowait + now_sys[i].irq + now_sys[i].softirq + now_sys[i].steal + + int now_time = now_sys[i].usr + now_sys[i].sys + now_sys[i].nice + + now_sys[i].idle + now_sys[i].iowait + now_sys[i].irq + + now_sys[i].softirq + now_sys[i].steal + now_sys[i].guest + now_sys[i].guest_nice; - int prev_time = prev_sys[i].usr + prev_sys[i].sys + prev_sys[i].nice + prev_sys[i].idle + - prev_sys[i].iowait + prev_sys[i].irq + prev_sys[i].softirq + - prev_sys[i].steal + prev_sys[i].guest + prev_sys[i].guest_nice; + int prev_time = prev_sys[i].usr + prev_sys[i].sys + prev_sys[i].nice + + prev_sys[i].idle + prev_sys[i].iowait + + prev_sys[i].irq + prev_sys[i].softirq + + prev_sys[i].steal + prev_sys[i].guest + + prev_sys[i].guest_nice; int all_time = now_time - prev_time; - // int all_time = (sysconf(_SC_NPROCESSORS_ONLN) * env.delay * sysconf(_SC_CLK_TCK)); + // int all_time = (sysconf(_SC_NPROCESSORS_ONLN) * env.delay * + // sysconf(_SC_CLK_TCK)); - /* all_time can't not calculate by delay * ticks * online-cpu-num, because there is error - * between process waked up and running, when sched delay occur , the sum of cpu rates more - * than 100%. */ + /* all_time can't not calculate by delay * ticks * online-cpu-num, + * because there is error between process waked up and running, when + * sched delay occur , the sum of cpu rates more than 100%. */ - sys_rec->cpu[i].usr = (double)(now_sys[i].usr - prev_sys[i].usr) * 100 / all_time; - sys_rec->cpu[i].sys = (double)(now_sys[i].sys - prev_sys[i].sys) * 100 / all_time; - sys_rec->cpu[i].iowait = (double)(now_sys[i].iowait - prev_sys[i].iowait) * 100 / all_time; + sys_rec->cpu[i].usr = + (double)(now_sys[i].usr - prev_sys[i].usr) * 100 / all_time; + sys_rec->cpu[i].sys = + (double)(now_sys[i].sys - prev_sys[i].sys) * 100 / all_time; + sys_rec->cpu[i].iowait = + (double)(now_sys[i].iowait - prev_sys[i].iowait) * 100 / all_time; } cleanup: if (fp) fclose(fp); @@ -315,7 +365,8 @@ static int read_all_pids(struct id_pair_t* pids, u_int64_t* num) { } while ((task_de = readdir(task_dir)) != NULL) { - if (task_de->d_type != DT_DIR || !strcmp(task_de->d_name, ".") || + if (task_de->d_type != DT_DIR || + !strcmp(task_de->d_name, ".") || !strcmp(task_de->d_name, "..")) continue; err = parse_long(task_de->d_name, &val); @@ -382,11 +433,13 @@ static int read_proc(pid_t pid, pid_t tid, struct task_cputime_t** prev, fscanf(fp, "%d %s %c %d %d %d %d %d %u %lu %lu %lu %lu %lu %lu %ld %ld %ld " "%ld %ld %ld %llu", - &proc_info.pid, &proc_info.comm[0], &proc_info.state, &proc_info.ppid, &proc_info.pgrp, - &proc_info.session, &proc_info.tty_nr, &proc_info.tpgid, &proc_info.flags, - &proc_info.minflt, &proc_info.cminflt, &proc_info.majflt, &proc_info.cmajflt, - &proc_info.utime, &proc_info.stime, &proc_info.cutime, &proc_info.cstime, - &proc_info.priority, &proc_info.nice, &proc_info.num_threads, &proc_info.itrealvalue, + &proc_info.pid, &proc_info.comm[0], &proc_info.state, + &proc_info.ppid, &proc_info.pgrp, &proc_info.session, + &proc_info.tty_nr, &proc_info.tpgid, &proc_info.flags, + &proc_info.minflt, &proc_info.cminflt, &proc_info.majflt, + &proc_info.cmajflt, &proc_info.utime, &proc_info.stime, + &proc_info.cutime, &proc_info.cstime, &proc_info.priority, + &proc_info.nice, &proc_info.num_threads, &proc_info.itrealvalue, &proc_info.starttime); data->utime = proc_info.utime; @@ -397,7 +450,8 @@ static int read_proc(pid_t pid, pid_t tid, struct task_cputime_t** prev, strcpy(data->comm, proc_info.comm); - time_t run_time = time(0) - btime - (now[pid]->starttime / sysconf(_SC_CLK_TCK)); + time_t run_time = + time(0) - btime - (now[pid]->starttime / sysconf(_SC_CLK_TCK)); if (prev[pid] && !strcmp(prev[pid]->comm, now[pid]->comm)) { long udelta = now[pid]->utime - prev[pid]->utime; @@ -411,7 +465,8 @@ static int read_proc(pid_t pid, pid_t tid, struct task_cputime_t** prev, (*rec)->pid = now[pid]->pid; (*rec)->ppid = now[pid]->ppid; (*rec)->runtime = run_time; - (*rec)->begin_ts = btime + now[pid]->starttime / sysconf(_SC_CLK_TCK); + (*rec)->begin_ts = + btime + now[pid]->starttime / sysconf(_SC_CLK_TCK); (*rec)->user_cpu_rate = (double)udelta * 100 / base; (*rec)->system_cpu_rate = (double)sdelta * 100 / base; (*rec)->all_cpu_rate = (double)(udelta + sdelta) * 100 / base; @@ -425,7 +480,8 @@ cleanup: return err; } -static void sort_records(struct record_t* rec, int proc_num, enum sort_type sort) { +static void sort_records(struct record_t* rec, int proc_num, + enum sort_type sort) { struct task_record_t** records = rec->tasks; int i, j; for (i = 0; i < proc_num; i++) { @@ -457,7 +513,8 @@ static void sort_records(struct record_t* rec, int proc_num, enum sort_type sort } if (lth < rth) { - swap(&records[i], &records[j], sizeof(struct task_record_t*)); + swap(&records[i], &records[j], + sizeof(struct task_record_t*)); } } } @@ -521,18 +578,23 @@ static void output(struct record_t* rec, int proc_num, FILE* dest) { fprintf(dest, "%s\n", ts2str(now, stime_str, BUF_SIZE)); fprintf(dest, "UTIL&LOAD\n"); - fprintf(dest, "%6s %6s %6s %6s %6s %6s %6s :%5s \n", "usr", "sys", "iowait", "load1", "R", "D", - "fork", "proc"); + fprintf(dest, "%6s %6s %6s %6s %6s %6s %6s :%5s \n", "usr", "sys", "iowait", + "load1", "R", "D", "fork", "proc"); - fprintf(dest, "%6.1f %6.1f %6.1f %6.1f %6d %6d %6d", sys->cpu[0].usr, sys->cpu[0].sys, - sys->cpu[0].iowait, sys->load1, sys->nr_R, sys->nr_D, sys->nr_fork); - fprintf(dest, " : %s(%d) ppid=%d cnt=%lu \n", info->comm, info->pid, info->ppid, info->fork); + fprintf(dest, "%6.1f %6.1f %6.1f %6.1f %6d %6d %6d", sys->cpu[0].usr, + sys->cpu[0].sys, sys->cpu[0].iowait, sys->load1, sys->nr_R, + sys->nr_D, sys->nr_fork); + fprintf(dest, " : %s(%d) ppid=%d cnt=%lu \n", info->comm, info->pid, + info->ppid, info->fork); #ifdef DEBUG + fprintf(dest, "[ cpu ] %6s %6s %6s %10s\n", "usr", "sys", "iowait", + "delay(ns)"); for (i = 1; i <= nr_cpu; i++) { - fprintf(dest, "%6.1f %6.1f %6.1f\n", sys->cpu[i].usr, sys->cpu[i].sys, sys->cpu[i].iowait); + fprintf(dest, "[cpu-%d] %6.1f %6.1f %6.1f %10.1d\n", i - 1, + sys->cpu[i].usr, sys->cpu[i].sys, sys->cpu[i].iowait, + sys->percpu_sched_delay[i - 1]); } - #endif for (i = 0; i < proc_num; i++) { if (!records[i]) break; @@ -540,26 +602,30 @@ static void output(struct record_t* rec, int proc_num, FILE* dest) { if (env.human) { if (i == 0) { fprintf(dest, "TASKTOP\n"); - fprintf(dest, "%18s %6s %6s %20s %15s %6s %6s %6s\n", "COMMAND", "PID", "PPID", - "START", "RUN", "%UTIME", "%STIME", "%CPU"); + fprintf(dest, "%18s %6s %6s %20s %15s %6s %6s %6s\n", "COMMAND", + "PID", "PPID", "START", "RUN", "%UTIME", "%STIME", + "%CPU"); } if (i >= env.limit) break; - fprintf(dest, "%18s %6d %6d %20s %15s %6.1f %6.1f %6.1f\n", records[i]->comm, - records[i]->pid, records[i]->ppid, + fprintf(dest, "%18s %6d %6d %20s %15s %6.1f %6.1f %6.1f\n", + records[i]->comm, records[i]->pid, records[i]->ppid, ts2str(records[i]->begin_ts, stime_str, BUF_SIZE), - second2str(records[i]->runtime, rtime_str, BUF_SIZE), records[i]->user_cpu_rate, - records[i]->system_cpu_rate, records[i]->all_cpu_rate); + second2str(records[i]->runtime, rtime_str, BUF_SIZE), + records[i]->user_cpu_rate, records[i]->system_cpu_rate, + records[i]->all_cpu_rate); } else { if (i == 0) { fprintf(dest, "TASKTOP\n"); - fprintf(dest, "%18s %6s %6s %10s %10s %6s %6s %6s\n", "COMMAND", "PID", "PPID", - "START", "RUN", "%UTIME", "%STIME", "%CPU"); + fprintf(dest, "%18s %6s %6s %10s %10s %6s %6s %6s\n", "COMMAND", + "PID", "PPID", "START", "RUN", "%UTIME", "%STIME", + "%CPU"); } if (i >= env.limit) break; - fprintf(dest, "%18s %6d %6d %10ld %10ld %6.1f %6.1f %6.1f\n", records[i]->comm, - records[i]->pid, records[i]->ppid, records[i]->begin_ts, records[i]->runtime, + fprintf(dest, "%18s %6d %6d %10ld %10ld %6.1f %6.1f %6.1f\n", + records[i]->comm, records[i]->pid, records[i]->ppid, + records[i]->begin_ts, records[i]->runtime, records[i]->user_cpu_rate, records[i]->system_cpu_rate, records[i]->all_cpu_rate); } @@ -568,8 +634,10 @@ static void output(struct record_t* rec, int proc_num, FILE* dest) { } static void now_to_prev(struct id_pair_t* pids, int proc_num, int pidmax, - struct task_cputime_t** prev_task, struct task_cputime_t** now_task, - struct sys_cputime_t* prev_sys, struct sys_cputime_t* now_sys) { + struct task_cputime_t** prev_task, + struct task_cputime_t** now_task, + struct sys_cputime_t* prev_sys, + struct sys_cputime_t* now_sys) { int i; for (i = 0; i < pidmax; i++) { if (prev_task[i]) { @@ -590,8 +658,9 @@ static void now_to_prev(struct id_pair_t* pids, int proc_num, int pidmax, swap(prev_sys, now_sys, sizeof(struct sys_cputime_t) * (nr_cpu + 1)); } -static int make_records(struct id_pair_t* pids, int proc_num, struct record_t* rec, - struct task_cputime_t** prev_task, struct task_cputime_t** now_task) { +static int make_records(struct id_pair_t* pids, int proc_num, + struct record_t* rec, struct task_cputime_t** prev_task, + struct task_cputime_t** now_task) { struct task_record_t** records = rec->tasks; int err = 0; u_int64_t i; @@ -636,7 +705,8 @@ static FILE* open_logfile() { return stat_log; } -static int libbpf_print_fn(enum libbpf_print_level level, const char* format, va_list args) { +static int libbpf_print_fn(enum libbpf_print_level level, const char* format, + va_list args) { return vfprintf(stderr, format, args); } @@ -649,7 +719,7 @@ static int bump_memlock_rlimit(void) { return setrlimit(RLIMIT_MEMLOCK, &rlim_new); } -static int look_fork(int cnt_map_fd, int fork_map_fd, struct sys_record_t* sys_rec) { +static int check_fork(int fork_map_fd, struct sys_record_t* sys_rec) { int fd; int err; u_int64_t total = 0; @@ -663,7 +733,7 @@ static int look_fork(int cnt_map_fd, int fork_map_fd, struct sys_record_t* sys_r err = bpf_map_delete_elem(fd, &next_key); if (err < 0) { - fprintf(stderr, "failed to delete elem: %d\n", err); + fprintf(stderr, "Failed to delete elem: %d\n", err); return -1; } lookup_key = next_key; @@ -685,7 +755,7 @@ static void sigint_handler(int signo) { exiting = 1; } int main(int argc, char** argv) { int err = 0; - int cnt_map_fd = -1, fork_map_fd = -1; + int fork_map_fd = -1; FILE* stat_log = 0; struct tasktop_bpf* skel = 0; struct id_pair_t* pids = 0; @@ -693,6 +763,8 @@ int main(int argc, char** argv) { struct sys_cputime_t *prev_sys = 0, *now_sys = 0; struct record_t* rec = 0; + prev_delay = calloc(nr_cpu, sizeof(int)); + nr_cpu = sysconf(_SC_NPROCESSORS_ONLN); if (signal(SIGINT, sigint_handler) == SIG_ERR) { @@ -731,6 +803,8 @@ int main(int argc, char** argv) { rec = calloc(1, sizeof(struct record_t)); rec->sys.cpu = calloc(nr_cpu + 1, sizeof(struct cpu_util_t)); + rec->sys.percpu_sched_delay = calloc(nr_cpu, sizeof(int)); + pids = calloc(pidmax, sizeof(struct id_pair_t)); prev_task = calloc(pidmax, sizeof(struct task_cputime_t*)); now_task = calloc(pidmax, sizeof(struct task_cputime_t*)); @@ -762,8 +836,8 @@ int main(int argc, char** argv) { goto cleanup; } - cnt_map_fd = bpf_map__fd(skel->maps.cnt_map); fork_map_fd = bpf_map__fd(skel->maps.fork_map); + err = tasktop_bpf__attach(skel); if (err) { fprintf(stderr, "Failed to attach BPF skeleton\n"); @@ -772,7 +846,8 @@ int main(int argc, char** argv) { bool first = true; while (env.nr_iter-- && !exiting) { - look_fork(cnt_map_fd, fork_map_fd, &rec->sys); + check_delay(&rec->sys); + check_fork(fork_map_fd, &rec->sys); runnable_proc(&rec->sys); unint_proc(&rec->sys); read_stat(prev_sys, now_sys, &rec->sys); @@ -806,7 +881,8 @@ int main(int argc, char** argv) { free_records(rec, proc_num); /* update old info and free nonexist process info */ - now_to_prev(pids, proc_num, pidmax, prev_task, now_task, prev_sys, now_sys); + now_to_prev(pids, proc_num, pidmax, prev_task, now_task, prev_sys, + now_sys); if (env.nr_iter) sleep(env.delay); } diff --git a/source/tools/detect/sched/tasktop/tasktop.h b/source/tools/detect/sched/tasktop/tasktop.h index d109fb70..7ed0baaf 100644 --- a/source/tools/detect/sched/tasktop/tasktop.h +++ b/source/tools/detect/sched/tasktop/tasktop.h @@ -13,6 +13,7 @@ #define DEBUG_LOG "./log/debug.log" #define PIDMAX_PATH "/proc/sys/kernel/pid_max" #define PROC_STAT_PATH "/proc/stat" +#define SCHEDSTAT_PATH "/proc/schedstat" enum sort_type { SORT_SYSTEM, SORT_USER, SORT_CPU }; @@ -96,6 +97,8 @@ typedef struct sys_record_t { int nr_D; int nr_fork; struct proc_fork_info_t most_fork_info; + + int *percpu_sched_delay; } sys_record_t; struct record_t { -- Gitee