diff --git a/anolis-courses/Coolbpf/clcc/finish.md b/anolis-courses/Coolbpf/clcc/finish.md new file mode 100644 index 0000000000000000000000000000000000000000..f70ef0bdb742c895816a3250e68fa22b9b280be3 --- /dev/null +++ b/anolis-courses/Coolbpf/clcc/finish.md @@ -0,0 +1 @@ +恭喜您完成了使用c在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/clcc/index.yaml b/anolis-courses/Coolbpf/clcc/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6acf4d6320eec46ae3fa45170e1472619e1b410 --- /dev/null +++ b/anolis-courses/Coolbpf/clcc/index.yaml @@ -0,0 +1,13 @@ +name: Coolbpf practice with c +desc: 使用c在coolbpf框架下远程编译开发 +image: "Anolis OS 8.4 ANCK 64位" +live_time: "60 min" +details: + steps: + start: start.md + finish: finish.md + - name: 环境准备 + content: step1.md + - name: 编译开发 + content: step2.md + diff --git a/anolis-courses/Coolbpf/clcc/start.md b/anolis-courses/Coolbpf/clcc/start.md new file mode 100644 index 0000000000000000000000000000000000000000..ae4cfdd3346d85f3cd234f5fc29fed49ffde111c --- /dev/null +++ b/anolis-courses/Coolbpf/clcc/start.md @@ -0,0 +1 @@ +本场景提供一台虚拟机环境用于使用c在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/clcc/step1.md b/anolis-courses/Coolbpf/clcc/step1.md new file mode 100644 index 0000000000000000000000000000000000000000..a92627b99dab54fb6b7c4d984c4ffbd0372cc49f --- /dev/null +++ b/anolis-courses/Coolbpf/clcc/step1.md @@ -0,0 +1,19 @@ +#### 1.下载环境依赖包 + +* 安装python3包 + +[[ yum install -y python3 ]] + +* 安装coolbpf环境包 + +[[ pip3 install coolbpf ]] + +* 安装git + +[[ yum install -y git ]] + +#### 2.下载coolbpf代码 + +[[ git clone https://gitee.com/anolis/coolbpf.git ]] + +通过上述步骤,coolbpf运行环境即已安装好。 diff --git a/anolis-courses/Coolbpf/clcc/step2.md b/anolis-courses/Coolbpf/clcc/step2.md new file mode 100644 index 0000000000000000000000000000000000000000..1bf014cfff795d8b307a8682d7a4ea55ce6c4bfe --- /dev/null +++ b/anolis-courses/Coolbpf/clcc/step2.md @@ -0,0 +1,372 @@ +### 1.coolbpf 入门——clcc + + clcc 入门代码放在 coolbpf/lcc/clcc/remote/ 目录下,引导用户快速上手: + +* hello # hello world +* hash_map # 往用户态传递信息 +* config_map # 配置map参数 +* event_out # 事件输出 +* call_stack # 获取调用栈的方法 + +### 2.编译开发 + +#### 2.1 hello world 入门 + +进入hello目录,编译并执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ hello]# make +coolbpf -f hello.bpf.c +remote server compile success. +gcc -Wall -g -I../include -c hello.c -o hello.o -DSO_NAME=\"hello.so\" +gcc -lpthread -ldl -o hello hello.o +[root@iZ2zec8l5h2t7ee4suqxbmZ hello]# ./hello + +[root@iZ2zec8l5h2t7ee4suqxbmZ config_map]# cat /sys/kernel/debug/tracing/trace_pipe + <...>-1059 [000] .... 2675.933799: 0: hello lcc, parent: 6879 + <...>-6879 [000] .... 2675.940842: 0: hello lcc, parent: 6880 + <...>-6879 [000] .... 2676.138417: 0: hello lcc, parent: 6881 + <...>-6881 [000] .... 2676.246183: 0: hello lcc, parent: 6882 + <...>-6882 [000] .... 2676.248985: 0: hello lcc, parent: 6883 + <...>-6883 [000] .... 2676.251117: 0: hello lcc, parent: 6884 +``` + +#### 2.2 创建map + +进入hash_map目录,编译并执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ hash_map]# make +coolbpf -f hash_map.bpf.c +remote server compile success. +gcc -Wall -g -I../include -c hash_map.c -o hash_map.o -DSO_NAME=\"hash_map.so\" +gcc -lpthread -ldl -o hash_map hash_map.o +[root@iZ2zec8l5h2t7ee4suqxbmZ hash_map]# ./hash_map +The program starts executing and will exit after 10 seconds. +poll message: c_pid:1059, p_pid:7090 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7090, p_pid:7091 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7090, p_pid:7092 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7092, p_pid:7093 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7093, p_pid:7094 +c_comm:bash, p_comm:bash +poll message: c_pid:7094, p_pid:7095 +c_comm:grepconf.sh, p_comm:grepconf.sh +``` + +### 2.3动态修改bpf部分代码 + +进入config_map,编译并执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ config_map]# make +gcc -Wall -g -I../include -c config_map.c -o config_map.o -DSO_NAME=\"config_map.so\" +gcc -lpthread -ldl -o config_map config_map.o +[root@iZ2zec8l5h2t7ee4suqxbmZ config_map]# ./config_map +The program starts executing and will exit after 10 seconds. +user config map +``` + +#### 2.4通过perf\_event往用户态传递信息 + +进入event_out目录,编译并执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ event_out]# make +coolbpf -f event_out.bpf.c +remote server compile success. +gcc -Wall -g -I../include -c event_out.c -o event_out.o -DSO_NAME=\"event_out.so\" +gcc -lpthread -ldl -o event_out event_out.o +[root@iZ2zec8l5h2t7ee4suqxbmZ event_out]# ./event_out +poll message: c_pid:1059, p_pid:7543 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7543, p_pid:7544 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7543, p_pid:7545 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7545, p_pid:7546 +c_comm:sshd, p_comm:sshd +poll message: c_pid:7546, p_pid:7547 +``` + +#### 2.5 call stack获取 + +进入call_stack目录,编译并执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ call_stack]# make +coolbpf -f call_stack.bpf.c +remote server compile success. +gcc -Wall -g -I../include -c call_stack.c -o call_stack.o -DSO_NAME=\"call_stack.so\" +gcc -lpthread -ldl -o call_stack call_stack.o +[root@iZ2zec8l5h2t7ee4suqxbmZ call_stack]# ./call_stack +poll message: c_pid:1059, p_pid:7730, stack_id:0 +c_comm:sshd, p_comm:sshd +call stack: + 0xffffffffa00cdb01: startup_64+0xcdb01 + 0xffffffffa009fd84: _do_fork+0x124 + 0xffffffffa00040ff: do_syscall_64+0x5f + 0xffffffffa0a00085: entry_SYSCALL_64_after_hwframe+0x44 +poll message: c_pid:7730, p_pid:7731, stack_id:0 +c_comm:sshd, p_comm:sshd +call stack: + 0xffffffffa00cdb01: wake_up_new_task+0x1 + 0xffffffffa009fd84: _do_fork+0x124 + 0xffffffffa00040ff: do_syscall_64+0x5f + 0xffffffffa0a00085: entry_SYSCALL_64_after_hwframe+0x44 +poll message: c_pid:7730, p_pid:7732, stack_id:0 +c_comm:sshd, p_comm:sshd +``` + +### 3.API参考 + + 头文件clcc.h保存在 include 路径下, 实现了so加载的主要功能,主要功能如下: + +```c +/* + * function name: clcc_init + * description: load an so + * arg1: so path to load + * return: struct clcc_struct * + */ +struct clcc_struct* clcc_init(const char* so_path); + +/* + * function name: clcc_deinit + * description: release an so + * arg1: struct clcc_struct *p; mem will free this function. + * return: None + */ +struct clcc_struct* clcc_deinit(const char* so_path); + +/* + * function name: clcc_get_call_stack + * description: get call stack from table and stack id + * arg1: table id: from struct clcc_struct get_maps_id function. + * arg2: stack_id: from bpf kernel bpf_get_stackid function. + * arg3: pstack: struct clcc_call_stack, should be alloced at first, use in clcc_print_stack + * arg4: pclcc: setup from clcc_init function + * return: 0 if success. + */ +int clcc_get_call_stack(int table_id, + int stack_id, + struct clcc_call_stack *pstack, + struct clcc_struct *pclcc) + + +/* + * function name: clcc_print_stack + * description: print call stack + * arg1: pstack: struct clcc_call_stack, stack to print, setup from clcc_get_call_stack. + * arg2: pclcc: setup from clcc_init function + * return: None. + */ +void clcc_print_stack(struct clcc_call_stack *pstack, + struct clcc_struct *pclcc) +``` + + struct clcc_struct 是 clcc 最重要的结构体,封装libbpf的主要功能,结构定义如下: + +```c +struct clcc_struct{ + /* + * member: handle + * description: so file file handle pointer, it should not be modified or accessed. + */ + void* handle; + /* + * member: status + * description: reserved. + */ + int status; + /* + * member: init + * description: install libbpf programme, + * arg1: print level, 0~3. -1:do not print any thing. + * arg2: attach, 0: do not attach, !0: attach + * return: 0 if success. + */ + int (*init)(int log_level, int attach); + /* + * member: exit + * description: uninstall libbpf programme, + * return: None. + */ + void (*exit)(void); + /* + * member: get_maps_id + * description: get map id from map name which quote in LBC_XXX(). + * arg1: event: map name which quote in LBC_XXX(), eg: LBC_PERF_OUTPUT(e_out, struct data_t, 128), then arg is e_out. + * return: >=0, failed when < 0 + */ + int (*get_maps_id)(char* event); + /* + * member: set_event_cb + * description: set call back function for perf out event. + * arg1: event id, get from get_maps_id. + * arg2: callback function when event polled. + * arg3: lost callback function when event polled. + * return: 0 if success. + */ + int (*set_event_cb)(int id, + void (*cb)(void *ctx, int cpu, void *data, unsigned int size), + void (*lost)(void *ctx, int cpu, unsigned long long cnt)); + /* + * member: event_loop + * description: poll perf out put event, usually used in pairs with set_event_cb function. + * arg1: event id, get from get_maps_id. + * arg2: timeout, unit seconds. -1 nevet timeout. + * return: 0 if success. + */ + int (*event_loop)(int id, int timeout); + /* + * member: map_lookup_elem + * description: lookup element by key. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * arg3: value point. + * return: 0 if success. + */ + int (*map_lookup_elem)(int id, const void *key, void *value); + /* + * member: map_lookup_elem_flags + * description: lookup element by key. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * arg3: value point. + * return: 0 if success. + */ + int (*map_lookup_elem_flags)(int id, const void *key, void *value, unsigned long int); + /* + * member: map_lookup_and_delete_elem + * description: lookup element by key then delete key. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * arg3: value point. + * return: 0 if success. + */ + int (*map_lookup_and_delete_elem)(int id, const void *key, void *value); + /* + * member: map_delete_elem + * description: lookup element by key then delete key. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * return: 0 if success. + */ + int (*map_delete_elem)(int id, const void *key); + /* + * member: map_update_elem + * description: update element by key. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * arg3: value point. + * return: 0 if success. + */ + int (*map_update_elem)(int id, const void *key, void *value); + /* + * member: map_get_next_key + * description: walk keys from maps. + * arg1: event id, get from get_maps_id. + * arg2: key point. + * arg3: next key point. + * return: 0 if success. + */ + int (*map_get_next_key)(int id, const void *key, void *next_key); + /* + * member: attach_perf_event + * description: attach perf event. + * arg1: function name in bpf.c. + * arg2: perf event id. + * return: 0 if success. + */ + int (*attach_perf_event)(const char* func, int pfd); + /* + * member: attach_kprobe + * description: attach kprobe. + * arg1: function name in bpf.c. + * arg2: kprobe symbol. + * return: 0 if success. + */ + int (*attach_kprobe)(const char* func, const char* sym); + /* + * member: attach_kretprobe + * description: attach kprobe. + * arg1: function name in bpf.c. + * arg2: kprobe symbol. + * return: 0 if success. + */ + int (*attach_kretprobe)(const char* func, const char* sym); + /* + * member: attach_uprobe + * description: attach uprobe. + * arg1: function name in bpf.c. + * arg2: task pid + * arg3: binary_path. + * arg4: offset. + * return: 0 if success. + */ + int (*attach_uprobe)(const char* func, int pid, const char *binary_path, unsigned long func_offset); + /* + * member: attach_uretprobe + * description: attach uretprobe. + * arg1: function name in bpf.c. + * arg2: task pid + * arg3: binary_path. + * arg4: offset. + * return: 0 if success. + */ + int (*attach_uretprobe)(const char* func, int pid, const char *binary_path, unsigned long func_offset); + /* + * member: attach_tracepoint + * description: attach kprobe. + * arg1: function name in bpf.c. + * arg2: tp_category. + * arg3: tp_name. + * return: 0 if success. + */ + int (*attach_tracepoint)(const char* func, const char *tp_category, const char *tp_name); + /* + * member: attach_raw_tracepoint + * description: attach kprobe. + * arg1: function name in bpf.c. + * arg2: tp_name. + * return: 0 if success. + */ + int (*attach_raw_tracepoint)(const char* func, const char *tp_name); + /* + * member: attach_cgroup + * description: attach cgroup. + * arg1: function name in bpf.c. + * arg2: cgroup_fd. + * return: 0 if success. + */ + int (*attach_cgroup)(const char* func, int cgroup_fd); + /* + * member: attach_netns + * description: attach netns. + * arg1: function name in bpf.c. + * arg2: netns. + * return: 0 if success. + */ + int (*attach_netns)(const char* func, int netns); + /* + * member: attach_xdp + * description: attach xdp. + * arg1: function name in bpf.c. + * arg2: ifindex. + * return: 0 if success. + */ + int (*attach_xdp)(const char* func, int ifindex); + const char* (*get_map_types)(void); + /* + * member: ksym_search + * description: get symbol from kernel addr. + * arg1: kernnel addr. + * return: symbol name and address information. + */ + struct ksym* (*ksym_search)(unsigned long addr); +}; +``` diff --git a/anolis-courses/Coolbpf/glcc/finish.md b/anolis-courses/Coolbpf/glcc/finish.md new file mode 100644 index 0000000000000000000000000000000000000000..d4f9b6786950863060b03f9f24730cac9b5d91bb --- /dev/null +++ b/anolis-courses/Coolbpf/glcc/finish.md @@ -0,0 +1 @@ +恭喜您完成了coolbpf在低版本内核下ebpf程序的开发体验 diff --git a/anolis-courses/Coolbpf/glcc/index.yaml b/anolis-courses/Coolbpf/glcc/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8a824d3bef8e0573f7d1e8ec724356c30af56ec --- /dev/null +++ b/anolis-courses/Coolbpf/glcc/index.yaml @@ -0,0 +1,13 @@ +name: Coolbpf practice with low kernel +desc: coolbpf支持低版本内核执行bpf程序 +image: "Anolis OS 7.7 ANCK 64位" +live_time: "60 min" +details: + steps: + start: start.md + finish: finish.md + - name: 环境准备 + content: step1.md + - name: 编译开发 + content: step2.md + diff --git a/anolis-courses/Coolbpf/glcc/start.md b/anolis-courses/Coolbpf/glcc/start.md new file mode 100644 index 0000000000000000000000000000000000000000..6244fbfd4e551997b1d8d885185202c29c0ab6f8 --- /dev/null +++ b/anolis-courses/Coolbpf/glcc/start.md @@ -0,0 +1,2 @@ +本场景提供一台虚拟机环境用于coolbpf在低版本内核下ebpf程序的开发体验 + diff --git a/anolis-courses/Coolbpf/glcc/step1.md b/anolis-courses/Coolbpf/glcc/step1.md new file mode 100644 index 0000000000000000000000000000000000000000..d9a4eb7f480ab6f072a749572cf3c32372885acf --- /dev/null +++ b/anolis-courses/Coolbpf/glcc/step1.md @@ -0,0 +1,34 @@ +#### 1.下载环境依赖包 + +* 安装python3 + +[[ yum install -y python3 ]] + +* 安装coolbpf + +[[ pip3 install coolbpf ]] + +* 安装cmake及make + +[[ yum install -y cmake ]] +[[ yum install -y make ]] + +* 安装git + +[[ yum install -y git ]] + +* 安装perl + +[[ yum install -y perl ]] + +* 安装elf库 + +[[ yum install -y elfutils-devel.x86_64 ]] +[[ yum install -y elfutils-libelf.x86_64 ]] + +#### 2.下载coolbpf代码 + +[[ git clone https://gitee.com/anolis/coolbpf.git ]] + +通过上述步骤,coolbpf运行环境即已安装好。 + diff --git a/anolis-courses/Coolbpf/glcc/step2.md b/anolis-courses/Coolbpf/glcc/step2.md new file mode 100644 index 0000000000000000000000000000000000000000..ca00ab73cb4c4295f8e64b53ebc1d859540dce9d --- /dev/null +++ b/anolis-courses/Coolbpf/glcc/step2.md @@ -0,0 +1,90 @@ +### 1.coolbpf 入门——低版本内核开发 + +低版本内核支持代码放在 coolbpf/lcc/glcc/ 目录下,引导用户快速上手: + +* ebpf 支持ebpf的相关驱动代码 +* hook hook库相关的代码 + +#### 2.1 编译开发 + +##### 2.1.1 实现原理 + +gLCC 为了保证 eBPF 程序能够运行在低版本内核,提供了 hook 库和 eBPF 驱动。其中 hook 是一个动态库,用于截获 eBPF 程序运行时的系统调用,将其转换成向 eBPF 驱动发起的 ioctl 请求。eBPF 驱动则根据 ioctl 请求的具体信息执行相应的动作,如创建 map,prog 的安全检查、JIT 等。需要注意的是目前 prog 类型只支持 k(ret)probe 和 tracepoint 两种。 + +##### 2.1.2 hook库简介 + +由于低版本内核不支持 eBPF 的系统调用,原来在用户态创建 map、创建 prog 以及很多 helper 函数(如 `bpf_map_update_elem` 等)将不能直接在低版本内核上运行。为此 hook 截获系统调用并将这些系统调用转成发给 eBPF 驱动的 ioctl 请求。 + +##### 2.1.3 eBPF驱动简介 + +前面提到 eBPF 程序发起的系统调用会被 hook 库截获,并将其转换成相应的 ioctl 请求。当 eBPF 驱动收到该 Ioctl 命令时,会根据不同的请求来进行相应的操作,如: + +1. `IOCTL_BPF_MAP_CREATE`:创建 map; +2. `IOCTL_BPF_PROG_LOAD`:加载 eBPF 程序,将该程序的 eBPF 字节码进行安全验证、JIT 等操作; +3. `IOCTL_BPF_PROG_ATTACH`:将该 eBPF 程序 attach 到指定的内核函数,利用 `register_kprobe` 和 `tracepoint_probe_register` 功能完成 eBPF 程序的 attach。 + +#### 2.1.2编译执行 + +本节介绍在低版本内核上运行 eBPF 程序流程,主要分成两个大步骤,一个是编译生成 hook 库和 eBPF 驱动流程,另外一个是在低版本内核上运行 eBPF 程序流程。 + +* 下载低版本内核开发包 + +```bash +wget https://buildlogs.centos.org/c7.1611.u/kernel/20170704132018/3.10.0-514.26.2.el7.x86_64/kernel-devel-3.10.0-514.26.2.el7.x86_64.rpm --no-check-certificate +``` + +* 安装开发包 + +```bash +rpm -vhi kernel-devel-3.10.0-514.26.2.el7.x86_64.rpm --force +``` + +* 进入 coolbpf 项目根目录,创建 build 编译目录,利用cmake 生成编译所需的 Makefile 文件 + + ``` + cd coolbpf && mkdir build && cd build && cmake .. + ``` + +* 编译ebpf驱动 + +make ebpfdrv KERNEL_VERSION=3.10.0-514.26.2.el7.x86_64:生成 eBPF 驱动; + +驱动生成路径:./lcc/glcc/lib/ebpf/ebpfdrv.ko + +``` +[root@iZ2ze830e6ctgnu5dh1655Z build]# make ebpfdrv KERNEL_VERSION=3.10.0-514.26.2.el7.x86_64 +Scanning dependencies of target ebpfdrv +[ 50%] Copy ebpf driver code +[100%] Compile ebpf driver: Generate ebpfdrv.ko + +[root@iZ2ze830e6ctgnu5dh1655Z build]# find . -name ebpfdrv.ko +./lcc/glcc/lib/ebpf/ebpfdrv.ko +``` + +* 执行代码 + + 由于龙栖没有低版本的内核,因此以上程序我们在centos7.2上执行,以上代码已经放在了centos 的/root/longxi下,机器ip:39.96.202.191 root: + + * 插入ebpf驱动 + + insmod ebpfdrv.ko + + * 导出环境变量 + + export ENABLE_BPF_DRV=1 + + * 执行ebpf程序 + + ```bash + /root/longxi/./event_out + c_comm:event_out, p_comm: + poll message: c_pid:7294, p_pid:-1 + c_comm:kworker/0:2, p_comm: + poll message: c_pid:7116, p_pid:0 + c_comm:sshd, p_comm: + poll message: c_pid:7704, p_pid:0 + c_comm:event_out, p_comm: + poll message: c_pid:7294, p_pid:-1 + ``` + + diff --git a/anolis-courses/Coolbpf/golcc/finish.md b/anolis-courses/Coolbpf/golcc/finish.md new file mode 100644 index 0000000000000000000000000000000000000000..b3aba68c6f6e7c4ee97b7b52a58a493c7859b0d0 --- /dev/null +++ b/anolis-courses/Coolbpf/golcc/finish.md @@ -0,0 +1 @@ +恭喜您完成了使用go在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/golcc/index.yaml b/anolis-courses/Coolbpf/golcc/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dea217d5f1f33f4e663945bce02b574e26f56777 --- /dev/null +++ b/anolis-courses/Coolbpf/golcc/index.yaml @@ -0,0 +1,13 @@ +name: Coolbpf practice with go +desc: 使用go在coolbpf框架下远程编译开发 +image: "Anolis OS 8.4 ANCK 64位" +live_time: "60 min" +details: + steps: + start: start.md + finish: finish.md + - name: 环境准备 + content: step1.md + - name: 编译开发 + content: step2.md + diff --git a/anolis-courses/Coolbpf/golcc/start.md b/anolis-courses/Coolbpf/golcc/start.md new file mode 100644 index 0000000000000000000000000000000000000000..cc1507836cfea3ee4ad0f6d6ec3641b56809fb09 --- /dev/null +++ b/anolis-courses/Coolbpf/golcc/start.md @@ -0,0 +1 @@ +本场景提供一台虚拟机环境用于使用go在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/golcc/step1.md b/anolis-courses/Coolbpf/golcc/step1.md new file mode 100644 index 0000000000000000000000000000000000000000..3f8cac0464da38592c443d829d4b79b1afb713bb --- /dev/null +++ b/anolis-courses/Coolbpf/golcc/step1.md @@ -0,0 +1,28 @@ +#### 1.下载环境依赖包 + +* 安装python3包 + +[[ yum install -y python3 ]] + +* 安装coolbpf环境包 + +[[ pip3 install coolbpf ]] + +* 安装golang + +[[ yum install -y go ]] + +* 配置golang代理 + +[[ go env -w GOPROXY="https://proxy.golang.com.cn,direct" ]] + +* 安装git + +[[ yum install -y git ]] + +#### 2.下载coolbpf代码 + +[[ git clone https://gitee.com/anolis/coolbpf.git ]] + +通过上述步骤,coolbpf运行环境即已安装好。 + diff --git a/anolis-courses/Coolbpf/golcc/step2.md b/anolis-courses/Coolbpf/golcc/step2.md new file mode 100644 index 0000000000000000000000000000000000000000..a8ff1b161b9ff19d2031fba863dda98eca308af3 --- /dev/null +++ b/anolis-courses/Coolbpf/golcc/step2.md @@ -0,0 +1,250 @@ +### 1.coolbpf 入门——golcc + +golcc入门代码放在coolbpf/lcc/golcc目录下 + +* include + + libbpf 中的头文件,go编译的时候需要使用 + +* lib + + libbpf的静态库 + +* Rexample + + 样例代码,一个bpf的go程序主要包含三部分: + + * bpf/*.bpf.c:注入内核的bpf字节码部分的代码 + + * *.go:go语言加载bpf字节码的用户态运行代码 + + * Makefile:编译程序的makefile,只要在目录下执行make即可生成二进制程序 + +第三方库:Libbpfgo 主要包含两个文件: + +libbpf_cb.go : 利用cgo将libbpf中的原生接口及数据结构转成golang形式 + +libbpfgo.go:加载bpf程序的golang用户态接口 + +### 2.编译开发 + +#### 2.1 代码解读 + +##### go 部分代码 + +```go +package main + +import "C" + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/aquasecurity/libbpfgo" + "golang.org/x/sys/unix" +) + +var ( + h bool + path string +) + +func init() { + var name unix.Utsname + unix.Uname(&name) + defaultPath := fmt.Sprintf("/boot/vmlinux-%s", name.Release) + defaultPath = strings.Trim(defaultPath, "\x00") + + flag.BoolVar(&h, "h", false, "this help") + flag.StringVar(&path, "p", defaultPath, "set BTF custom path") +} + +func main() { + + flag.Parse() + fmt.Println("btf path:", path) + bpfModule, err := libbpfgo.NewModuleFromFileArgs(libbpfgo.NewModuleArgs{ + BPFObjPath: "hello.bpf.o", + BTFObjPath: path, + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(-1) + } + defer bpfModule.Close() + err = bpfModule.BPFLoadObject() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(-1) + } + + prog1, err := bpfModule.GetProgram("j_wake_up_new_task") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(-1) + } + link1, err := prog1.AttachGeneric() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(-1) + } + if link1.GetFd() == 0 { + os.Exit(-1) + } + + fmt.Println(prog1.GetType().String()) + time.Sleep(10000 * time.Second) +} +``` + +gobpf是对libbpf的封装,常见接口如下: + +* 加载bpf代码 + +```go +func NewModuleFromFile(bpfObjPath string)(*Module, error) +如果需要指定btf文件路径则可以使用如下接口 +func NewModuleFromFile(bpfObjPath string)(*Module, error) +``` + +* 加载bpf代码到内核 + + ```go + func (m *Module) BPFLoadObject() error + ``` + +* 获取内核bpf程序中的执行函数并attach上执行 + + ```go + func (m *Module) GetProgram(progName string) (*BPFProg, error) + func (p *BPFProg) AttachGeneric() (*BPFLink, error) + ``` + +* 创建、获取、设置map的接口 + + ```go + // See usage of `bpf_map_create()` in kernel selftests for more info + func CreateMap(mapType MapType, mapName string, keySize, valueSize, maxEntries int, opts *BPFMapCreateOpts) (*BPFMap, error) + func (m *Module) GetMap(mapName string) (*BPFMap, error) + ``` + +* 利用perf event获取数据的接口 + + ```go + 初始化 + func (m *Module) InitPerfBuf(mapName string, eventsChan chan []byte, lostChan chan uint64, pageCnt int) (*PerfBuffer, error) + poll事件 + func (pb *PerfBuffer) poll() error + + ``` + +更多接口可以参考下libbpfgo的libbpfgo.go文件 + +##### bpf部分代码 + +*.bpf.c的编写与c版本,python版本等无异 + +```c +#include "lbc.h" + +SEC("kprobe/wake_up_new_task") +int j_wake_up_new_task(struct pt_regs *ctx) +{ + struct task_struct *parent = (struct task_struct *)PT_REGS_PARM1(ctx); + + bpf_printk("hello golcc, parent:%x\n", parent); + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; +``` + +#### 2.2 编译执行 + +已编译hello项目为例,在helllo目录下执行make完成编译工作,此处bpf字节码部分的编译是借助远程编译完成的,golang部分代码借助本地的golang环境,首次编译会下载libbpfgo软件包。 + +``` +[root@iZ2zec8l5h2t7ee4suqxbmZ hello]# make +CGO_CFLAGS="-I /root/coolbpf/lcc/golcc/rexample/hello/../../include" CGO_LDFLAGS="/root/coolbpf/lcc/golcc/rexample/hello/../../lib/libbpf.a" go build -o hello +go: downloading github.com/aquasecurity/libbpfgo v0.3.0-libbpf-0.8.0 +go: downloading golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 +[root@iZ2zec8l5h2t7ee4suqxbmZ hello]# ls +bpf go.mod go.sum hello hello.bpf.o hello.go Makefile +``` + +执行hello程序 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ hello]# ./hello +btf path: /boot/vmlinux-4.19.91-26.1.an8.x86_64 +BPF_PROG_TYPE_KPROBE + +[root@iZ2zec8l5h2t7ee4suqxbmZ ~]# cat /sys/kernel/debug/tracing/trace_pipe + sshd-1059 [000] .... 78264.436460: 0: hello golcc, parent:3a6ea080 + <...>-37462 [000] .... 78264.444749: 0: hello golcc, parent:362a0000 + <...>-37462 [000] .... 78264.687852: 0: hello golcc, parent:362a0000 + <...>-37464 [000] .... 78264.815698: 0: hello golcc, parent:35760000 + <...>-37465 [000] .... 78264.818222: 0: hello golcc, parent:37968000 + <...>-37466 [000] .... 78264.820211: 0: hello golcc, parent:3c65c100 + <...>-37465 [000] .... 78264.822718: 0: hello golcc, parent:3a97a080 +``` + +### 2.3 常见错误 + +#### 2.3.1 libbpfgo 与libbpf版本不一致 + +默认下载的libbpfgo是最新版本,lib路径下的libbpf.a二进制有可能版本过老,造成版本不一致导致运行失败 + +``` +[root@wsgy3 hello]# ./hello +btf path: /boot/vmlinux-5.10.84-10.4.al8.x86_64 +fatal error: unexpected signal during runtime execution +[signal SIGSEGV: segmentation violation code=0x1 addr=0xffffffc0 pc=0x7fc1e79b0b0b] +。。。 +goroutine 1 [syscall]: +runtime.cgocall(0x4a8566, 0xc000045e30) + /usr/lib/golang/src/runtime/cgocall.go:156 +0x5c fp=0xc000045e08 sp=0xc000045dd0 pc=0x40ea1c +github.com/aquasecurity/libbpfgo._Cfunc_bpf_object__load(0xc35750) + _cgo_gotypes.go:923 +0x4c fp=0xc000045e30 sp=0xc000045e08 pc=0x4a34ec +``` + +* 解决方法:更新libbpf版本,重新编译 + +从github仓库下载最新版本,更新.h头文件和libbpf.a库 [https://github.com/libbpf/libbpf.git](https://gitee.com/link?target=https%3A%2F%2Fgithub.com%2Flibbpf%2Flibbpf.git) + +``` +$ cd libbpf/src +$ make install +$ cp /usr/include/bpf/* golcc/include/bpf/ +$ cp /usr/lib64/libbpf.a golcc/lib/ +``` + +* 或者通过go.mod指定libbpfgo的版本 + +``` +$cat lcc/golcc/rexample/hello/go.mod +module hello +go 1.17 +require github.com/aquasecurity/libbpfgo v0.3.0-libbpf-0.8.0 +require golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect +``` + +* 版本号说明:v0.3.0 是libbpfgo的版本号,0.8.0 是对应libbpf的版本号,两个需要配套,否则可能会出现兼容性问题 + +#### 2.3.2 运行内核版本中没有btf文件 + +python版本会执行时自动下载btf文件到/boot,建议先执行python版本 + +``` +[5] INT int size=4 bits_offset=0 nr_bits=32 encoding=SIGNED +[6] TYPEDEF j_wake_up_new_task type_id=4 +[7] Invalid btf_info:840000db +-- END BTF LOAD LOG -- +libbpf: Error loading .BTF into kernel: -22. BTF is optional, ignoring. +BPF_PROG_TYPE_KPROBE +``` + diff --git a/anolis-courses/Coolbpf/index.yaml b/anolis-courses/Coolbpf/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b9669f4a2e727b8f21e0a5d214c26715c017349 --- /dev/null +++ b/anolis-courses/Coolbpf/index.yaml @@ -0,0 +1,28 @@ +name: Coolbpf +desc: Coolbpf,以CO-RE(Compile Once-Run Everywhere)为基础实现,保留了资源占用低、可移植性强等优点,还融合了BCC动态编译的特性,适合在生产环境批量部署所开发的应用。coolbpf开创了一个新的思路,利用远程编译的思想,把用户的BPF程序推送 +到远端的服务器并返回给用户.o或.so,提供高级语言如python/rust/go/c等进行加载,然后在全量内核版本安全运行。用户只需专注自己的功能开发,不用关心底层库(如LLVM、python等)安装、环境搭建,给广大BPF爱好者提供一种新的探索和实践。 + 本教程介绍coolbpf的四种高级语言如何进行远程编译,以及如何在低版本内核上跑高版本上运行成功的eBPF程序. +type: course +total_time: "300 min" +level: "beginner" +chapters: + - name: "python hello world example" + desc: "使用python去做远程编译" + content: "pylcc" + live_time: "60 min" + - name: "c hello world example" + desc: "学习如何使用c程序去做eBPF的远程编译" + content: "clcc" + live_time: "60 min" + - name: "go hello world example" + desc: "学习如何使用go语言去做eBPF远程编译" + content: "golcc" + live_time: "60 min" + - name: "rust hello world example" + desc: "学习如何使用rust语言去做eBPF远程编译" + content: "rlcc" + live_time: "60 min" + - name: "learn how to run eBPF program on low Linux version" + desc: "学习如何在无eBPF特性的内核版本上跑高版本编译通过的eBPF程序" + content: "glcc" + live_time: "60 min" \ No newline at end of file diff --git a/anolis-courses/Coolbpf/pylcc/finish.md b/anolis-courses/Coolbpf/pylcc/finish.md new file mode 100644 index 0000000000000000000000000000000000000000..ece7f50ed538f4c4e7f97e3ae85169d244ed1ce7 --- /dev/null +++ b/anolis-courses/Coolbpf/pylcc/finish.md @@ -0,0 +1 @@ +恭喜您完成了使用python在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/pylcc/index.yaml b/anolis-courses/Coolbpf/pylcc/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e62aa9b4703d2ba85257e897c3a6b6995ff1049 --- /dev/null +++ b/anolis-courses/Coolbpf/pylcc/index.yaml @@ -0,0 +1,13 @@ +name: Coolbpf practice with python +desc: 使用python在coolbpf框架下远程编译开发 +image: "Anolis OS 8.4 ANCK 64位" +live_time: "60 min" +details: + steps: + start: start.md + finish: finish.md + - name: 环境准备 + content: step1.md + - name: 编译开发 + content: step2.md + diff --git a/anolis-courses/Coolbpf/pylcc/start.md b/anolis-courses/Coolbpf/pylcc/start.md new file mode 100644 index 0000000000000000000000000000000000000000..ff39b4a06be75a669ac0f87ead44fee82b7e620f --- /dev/null +++ b/anolis-courses/Coolbpf/pylcc/start.md @@ -0,0 +1 @@ +本场景提供一台虚拟机环境用于使用python在coolbpf框架下远程编译开发 \ No newline at end of file diff --git a/anolis-courses/Coolbpf/pylcc/step1.md b/anolis-courses/Coolbpf/pylcc/step1.md new file mode 100644 index 0000000000000000000000000000000000000000..d669c6055929dfe985ee2a62b18649514e3d8f7b --- /dev/null +++ b/anolis-courses/Coolbpf/pylcc/step1.md @@ -0,0 +1,19 @@ +#### 1.下载环境依赖包 + +* 安装python3包 + +[[ yum install -y python3 ]] + +* 安装coolbpf环境包 + +[[ pip3 install coolbpf ]] + +* 安装git + +[[ yum install -y git ]] + +#### 2.下载coolbpf代码 + +[[ git clone https://gitee.com/anolis/coolbpf.git ]] + +通过上述步骤,coolbpf运行环境即已安装好。 diff --git a/anolis-courses/Coolbpf/pylcc/step2.md b/anolis-courses/Coolbpf/pylcc/step2.md new file mode 100644 index 0000000000000000000000000000000000000000..70f6565219a2297714a75cbbf69223c466a31a2e --- /dev/null +++ b/anolis-courses/Coolbpf/pylcc/step2.md @@ -0,0 +1,338 @@ +### 1.coolbpf 入门——pylcc + + pylcc 入门代码放在 coolbpf/lcc/pylcc/guide/ 目录下,引导用户快速上手: + +* hello.py # hello world +* eventOut.py # 往用户态传递信息 +* dynamicVar.py # 动态修改代码 +* hashMap.py # hash map应用 +* callStack.py # 获取调用栈的方法 +* codeSeparate.py/independ.bpf.c # 独立bpf 文件实现 + +### 2.编译开发 + +#### 2.1 hello world 入门 + +Hello.py 代码如下 + +```python +import time +from pylcc.lbcBase import ClbcBase + +bpfPog = r""" +#include "lbc.h" + +SEC("kprobe/wake_up_new_task") +int j_wake_up_new_task(struct pt_regs *ctx) +{ + struct task_struct* parent = (struct task_struct *)PT_REGS_PARM1(ctx); + + bpf_printk("hello lcc, parent: %d\n", _(parent->tgid)); + return 0; +} + +char _license[] SEC("license") = "GPL"; +""" + +class Chello(ClbcBase): + def __init__(self): + super(Chello, self).__init__("hello", bpf_str=bpfPog) + while True: + time.sleep(1) + +if __name__ == "__main__": + hello = Chello() + pass +``` + +##### 2.1.1代码解读 + +##### bpf部分代码 + +* bpf代码包含 lbc.h 头文件即可,该头文件会包含以下头文件,并且会加上我们常见的宏定义和数据类型: + +```c +#include "vmlinux.h" +#include +#include +#include +#include +``` + +* SEC的定义和函数内部实现与libbpf应用方法保持一致; +* 访问结构体成员使用了_宏,该方法访问方式相对固定,下一节会提供core的获取方法; +* 末尾不要遗漏 _license声明 + +##### python代码 + + python 部分代码从ClbcBase 类继承,__init__函数中,第一入参必须要指定,用于指定生成so的文件名。在执行完__init__函数后,bfp模块就已经注入到内核当中执行。 + +##### 2.1.2编译执行 + +执行 python3 hello.py 运行,并在新的窗口查看调试信息输出: + +``` +#cat /sys/kernel/debug/tracing/trace_pipe + <...>-1091294 [005] d... 17658161.425644: : hello lcc, parent: 106880 + <...>-4142485 [003] d... 17658161.428568: : hello lcc, parent: 4142485 + <...>-4142486 [002] d... 17658161.430972: : hello lcc, parent: 4142486 + <...>-4142486 [002] d... 17658161.431228: : hello lcc, parent: 4142486 + <...>-4142486 [002] d... 17658161.431557: : hello lcc, parent: 4142486 + <...>-4142485 [003] d... 17658161.435385: : hello lcc, parent: 4142485 + <...>-4142490 [000] d... 17658161.437562: : hello lcc, parent: 4142490 +``` + +#### 2.2通过perf\_event往用户态传递信息 + +eventOut.py代码如下: + +```python +from pylcc.lbcBase import ClbcBase + +bpfPog = r""" +#include "lbc.h" +#define TASK_COMM_LEN 16 +struct data_t { + u32 c_pid; + u32 p_pid; + char c_comm[TASK_COMM_LEN]; + char p_comm[TASK_COMM_LEN]; +}; + +LBC_PERF_OUTPUT(e_out, struct data_t, 128); +SEC("kprobe/wake_up_new_task") +int j_wake_up_new_task(struct pt_regs *ctx) +{ + struct task_struct* parent = (struct task_struct *)PT_REGS_PARM1(ctx); + struct data_t data = {}; + + data.c_pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&data.c_comm, TASK_COMM_LEN); + data.p_pid = BPF_CORE_READ(parent, pid); + bpf_core_read(&data.p_comm[0], TASK_COMM_LEN, &parent->comm[0]); + + bpf_perf_event_output(ctx, &e_out, BPF_F_CURRENT_CPU, &data, sizeof(data)); + return 0; +} + +char _license[] SEC("license") = "GPL"; +""" + + +class CeventOut(ClbcBase): + def __init__(self): + super(CeventOut, self).__init__("eventOut", bpf_str=bpfPog) + + def _cb(self, cpu, data, size): + e = self.getMap('e_out', data, size) + print("current pid:%d, comm:%s. wake_up_new_task pid: %d, comm: %s" % ( + e.c_pid, e.c_comm, e.p_pid, e.p_comm + )) + + def loop(self): + self.maps['e_out'].open_perf_buffer(self._cb) + try: + self.maps['e_out'].perf_buffer_poll() + except KeyboardInterrupt: + print("key interrupt.") + exit() + + +if __name__ == "__main__": + e = CeventOut() + e.loop() +``` + +##### 2.2.1代码解读 + +##### libbpf部分代码 + +* LBC\_PERF\_OUTPUT宏不能用原有的bpf\_map\_def ……BPF\_MAP\_TYPE\_PERF\_EVENT\_ARRAY…… 替代,虽然是同样申明一个 perf maps,但如果用原始的声明方式,python在加载的时候将无法识别出对应的内核数据类型。 +* 可以使用 bpf\_get\_current\_pid\_tgid 等libbpf helper函数; +* 可以使用 bpf\_core\_read 等方法,达到一次编译,处处运行; +* 不可使用 bcc 独有的方法,如直接指针访问变量等; + +##### python部分代码 + + loop函数为入口: + +* self.maps['e\_out'].open\_perf\_buffer(self.\_cb)函数是为 e\_out事件注册回调钩子函数,其中e\_out命名与bpfProg中LBC\_PERF\_OUTPUT(e\_out, struct data\_t, 128) 对应; +* self.maps['e\_out'].perf\_buffer\_poll() 即poll 对应的event事件,与bpfProg中 bpf\_perf\_event\_output(ctx, &e\_out……对应; + + 接下来看\_cb 回调函数: + +* e = self.getMap('e\_out', data, size) 将数据流生成对应的数据对象; +* 生成了数据对象后,就可以通过成员的方式来访问数据对象,该对象成员与bpfProg中 struct data\_t 定义保持一致 + +##### 2.2.2编译执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ guide]# python3 eventOut.py +remote server compile success. +current pid:3944, comm:sshd. wake_up_new_task pid: 3945, comm: sshd +current pid:3945, comm:bash. wake_up_new_task pid: 3946, comm: bash +current pid:3946, comm:grepconf.sh. wake_up_new_task pid: 3947, comm: grepconf.sh +current pid:3945, comm:bash. wake_up_new_task pid: 3948, comm: bash +current pid:3948, comm:grepconf.sh. wake_up_new_task pid: 3949, comm: grepconf.sh +current pid:3945, comm:bash. wake_up_new_task pid: 3950, comm: bash +current pid:3950, comm:grepconf.sh. wake_up_new_task pid: 3951, comm: grepconf.sh +current pid:3945, comm:bash. wake_up_new_task pid: 3952, comm: bash +current pid:3952, comm:bash. wake_up_new_task pid: 3953, comm: bash +``` + +### 2.3动态修改bpf部分代码 + +##### 2.3.1代码解读 + + 动态适配是诸如python、lua等动态语言的灵魂特性。该示例主要展现了coolbpf的动态特性,大部分代码与eventOut.py一致。主要差异在bpfProg代码添加了过滤动作: + +```c + u32 pid = BPF_CORE_READ(parent, pid); + if (pid != FILTER_PID) { + return 0; + } +``` + +在main入口处进行了pid替换: + +```python +if __name__ == "__main__": + bpfPog = bpfPog.replace("FILTER_PID", sys.argv[1]) + e = CdynamicVar() + e.loop +``` + +##### 2.3.2编译执行 + +将要过滤的pid作为参数传入,执行效果 + +```bash +python3 dynamicVar.py 241871 +current pid:241808, comm:python. wake_up_new_task pid: 241871, comm: python +current pid:241808, comm:python. wake_up_new_task pid: 241871, comm: python +current pid:241808, comm:python. wake_up_new_task pid: 241871, comm: python +``` + +#### 2.4hash map应用 + + 与perf_event需要用户态不断同步轮询的方法不同,maps作为libbpf内置的数据类型,内核态可以直接访问和操作maps数据,用户态进行异步查询即可。 + +##### 2.4.1代码解读 + +##### bpf部分代码 + + 示例代码中定位hash map的方法: + +```C +LBC_HASH(pid_cnt, u32, u32, 1024); +``` + +使用方法与libbpf 一致: + +```C +u32 *pcnt, cnt; + + pcnt = bpf_map_lookup_elem(&pid_cnt, &pid); + cnt = pcnt ? *pcnt + 1 : 1; // 为了确保原子性,推荐 __sync_fetch_and_add 方法 + bpf_map_update_elem(&pid_cnt, &pid, &cnt, BPF_ANY); +``` + +##### python部分代码 + + 查询maps的位置在exit退出之前打印所有信息 + +```python +…… + dMap = self.maps['pid_cnt'] + print(dMap.get()) + exit() +``` + + 哈希表对象可以直接由 self.maps['pid\_cnt'] 方法获取到,可以调用get函数,获取到dict对象。 +  除了BPF\_MAP\_TYPE\_HASH,lcc当前还支持BPF\_MAP\_TYPE\_LRU\_HASH、BPF\_MAP\_TYPE\_PERCPU\_HASH、 BPF\_MAP\_TYPE\_LRU\_PERCPU\_HASH等类型,更多类型支持在完善中,敬请期待。 + +##### 2.4.2编译执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ guide]# python3 hashMap.py +remote server compile success. +current pid:4243, comm:sshd. wake_up_new_task pid: 4245, comm: sshd +current pid:4245, comm:sshd. wake_up_new_task pid: 4246, comm: sshd +current pid:4246, comm:bash. wake_up_new_task pid: 4247, comm: bash +current pid:4247, comm:grepconf.sh. wake_up_new_task pid: 4248, comm: grepconf.sh +current pid:4246, comm:bash. wake_up_new_task pid: 4249, comm: bash +current pid:4249, comm:grepconf.sh. wake_up_new_task pid: 4250, comm: grepconf.sh +current pid:4246, comm:bash. wake_up_new_task pid: 4251, comm: bash +current pid:4251, comm:grepconf.sh. wake_up_new_task pid: 4252, comm: grepconf.sh +current pid:4246, comm:bash. wake_up_new_task pid: 4253, comm: bash +current pid:4253, comm:bash. wake_up_new_task pid: 4254, comm: bash +current pid:4246, comm:bash. wake_up_new_task pid: 4255, comm: bash +current pid:4255, comm:bash. wake_up_new_task pid: 4256, comm: bash +``` + +#### 2.5 call stack获取 + + 获取内核调用栈是bpf一项非常重要的调试功能,参考 callStack.py,大部分代码与eventOut.py一致。 + +##### 2.5.1 代码解读 + +##### bpf部分代码 + + 吐出的数据结构体中增加stack\_id成员: + +```python +struct data_t { + u32 c_pid; + u32 p_pid; + char c_comm[TASK_COMM_LEN]; + char p_comm[TASK_COMM_LEN]; + u32 stack_id; +}; + +LBC_PERF_OUTPUT(e_out, struct data_t, 128); +LBC_STACK(call_stack,32); +``` + + 在处理函数中记录call stack + +``` +data.stack_id = bpf_get_stackid(ctx, &call_stack, KERN_STACKID_FLAGS); +``` + +##### python部分代码 + + 通过getStacks传入的stack\_id,即可获取调用栈符号数组,然后列出来即可 + +```python + stacks = self.maps['call_stack'].getStacks(e.stack_id) + print("call trace:") + for s in stacks: + print(s) +``` + +##### 2.5.2编译执行 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ guide]# python3 callStack.py +remote server compile success. +current pid:4374, comm:sshd. wake_up_new_task pid: 4375, comm: sshd +call trace: +startup_64 +_do_fork +do_syscall_64 +entry_SYSCALL_64_after_hwframe +current pid:4375, comm:bash. wake_up_new_task pid: 4376, comm: bash +call trace: +wake_up_new_task +_do_fork +do_syscall_64 +entry_SYSCALL_64_after_hwframe +current pid:4376, comm:grepconf.sh. wake_up_new_task pid: 4377, comm: grepconf.sh +call trace: +wake_up_new_task +_do_fork +do_syscall_64 +entry_SYSCALL_64_after_hwframe +current pid:4375, comm:bash. wake_up_new_task pid: 4378, comm: bash +``` + diff --git a/anolis-courses/Coolbpf/rlcc/finish.md b/anolis-courses/Coolbpf/rlcc/finish.md new file mode 100644 index 0000000000000000000000000000000000000000..54d3fb9258fc3f2caf25ab9a675d53ed43dd589e --- /dev/null +++ b/anolis-courses/Coolbpf/rlcc/finish.md @@ -0,0 +1 @@ +恭喜您完成了使用rust在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/rlcc/index.yaml b/anolis-courses/Coolbpf/rlcc/index.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1abab157f349a9600fad635eefc7e1de7b248048 --- /dev/null +++ b/anolis-courses/Coolbpf/rlcc/index.yaml @@ -0,0 +1,13 @@ +name: Coolbpf practice with rust +desc: 使用rust在coolbpf框架下远程编译开发 +image: "Anolis OS 8.4 ANCK 64位" +live_time: "60 min" +details: + steps: + start: start.md + finish: finish.md + - name: 环境准备 + content: step1.md + - name: 编译开发 + content: step2.md + diff --git a/anolis-courses/Coolbpf/rlcc/start.md b/anolis-courses/Coolbpf/rlcc/start.md new file mode 100644 index 0000000000000000000000000000000000000000..28ed759a80c688f28099112c42985052b9c3faff --- /dev/null +++ b/anolis-courses/Coolbpf/rlcc/start.md @@ -0,0 +1 @@ +本场景提供一台虚拟机环境用于使用rust在coolbpf框架下远程编译开发 diff --git a/anolis-courses/Coolbpf/rlcc/step1.md b/anolis-courses/Coolbpf/rlcc/step1.md new file mode 100644 index 0000000000000000000000000000000000000000..ddac70996b620db6580a188f9d7758b2326ef00e --- /dev/null +++ b/anolis-courses/Coolbpf/rlcc/step1.md @@ -0,0 +1,45 @@ +#### 1.下载环境依赖包 + +* 安装python3包 + +[[ yum install -y python3 ]] + +* 安装coolbpf环境包 + +[[ pip3 install coolbpf ]] + +* 安装git + +[[ yum install -y git ]] + +* 安装elf三方库 + +[[ yum install -y elfutils-libelf-devel.x86_64 ]] + +* 安装clang + +[[ yum install -y clang ]] + +* 安装cmake + +[[ yum install -y cmake ]] + +* 安装rust + +[[ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ]] + +``` +stable-x86_64-unknown-linux-gnu installed - rustc 1.63.0 (4b91a6ea7 2022-08-08) +Rust is installed now. Great! +``` + +安装完成后source 环境变量 + +[[ source "$HOME/.cargo/env" ]] + +#### 2.下载coolbpf代码 + +[[ git clone https://gitee.com/anolis/coolbpf.git ]] + +通过上述步骤,coolbpf运行环境即已安装好。 + diff --git a/anolis-courses/Coolbpf/rlcc/step2.md b/anolis-courses/Coolbpf/rlcc/step2.md new file mode 100644 index 0000000000000000000000000000000000000000..1f1169aae6f127f6a8dab9db7414621044339976 --- /dev/null +++ b/anolis-courses/Coolbpf/rlcc/step2.md @@ -0,0 +1,182 @@ +### 1.coolbpf 入门——rlcc + +rlcc rexample主要包含几个部分: + +1. `coolbpf/lcc/rlcc/rexample/src/bpf`: 该目录下存放了bpf代码 +2. `coolbpf/lcc/rlcc/rexample/src/main.rs`:该文件用于加载、attach bpf程序等交互操作 + +### 2.编译开发 + +#### 2.1 代码简介 + +main.rs是eBPF用户态处理程序,负责加载、attach eBPF程序等操作。下面是rexample的代码实例: + +```rust +fn main() -> Result<()>{ + let opts = Command::from_args(); + let mut skel_builder = ExampleSkelBuilder::default(); + if opts.verbose { + skel_builder.obj_builder.debug(true); + } + + bump_memlock_rlimit()?; + let mut open_skel = skel_builder.open()?; + // 加载eBPF程序 + let mut skel = open_skel.load()?; + // attach eBPF程序 + skel.attach()?; + // 配置perf事件的处理函数 + let perf = PerfBufferBuilder::new(skel.maps_mut().events()) + .sample_cb(handle_event) + .lost_cb(handle_lost_events) + .build()?; + + // 接收perf事件 + loop { + perf.poll(Duration::from_millis(100))?; + } +} +``` + +bpf程序, "lbc.h"为coolbpf的头文件其中封装了各个api + +```c +#include "lbc.h" +#include "example.h" + +struct +{ + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u32)); +} events SEC(".maps"); + +SEC("kprobe/tcp_connect") +int BPF_KPROBE(tcp_connect, struct sock *sk) +{ + struct example e; + struct inet_sock *inet = (struct inet_sock *)sk; + + e.pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&e.comm, TASK_COMM_LEN); + BPF_CORE_READ_INTO(&e.daddr, sk, __sk_common.skc_daddr); + BPF_CORE_READ_INTO(&e.dport, sk, __sk_common.skc_dport); + BPF_CORE_READ_INTO(&e.saddr, sk, __sk_common.skc_rcv_saddr); + BPF_CORE_READ_INTO(&e.sport, inet, inet_sport); + + bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &e, sizeof(e)); + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; +``` + + + +#### 2.2编译执行 + +1. 在coolbpf根目录下新建编译目录,/root/coolbpf + + ```bash + mkdir build && cd build + + [root@iZ2zec8l5h2t7ee4suqxbmZ coolbpf]# mkdir build && cd build + [root@iZ2zec8l5h2t7ee4suqxbmZ build]# pwd + /root/coolbpf/build + ``` + +2. 生成Makefile + + ```bash + cmake .. + + -- The C compiler identification is GNU 8.5.0 + -- Detecting C compiler ABI info + -- Detecting C compiler ABI info - done + -- Check for working C compiler: /usr/bin/cc - skipped + -- Detecting C compile features + -- Detecting C compile features - done + Found libelf library: /usr/lib64/libelf.so + -- Configuring done + -- Generating done + -- Build files have been written to: /root/coolbpf/build + ``` + +3. 执行编译 + + ```bash + make rexample + + = note: `#[warn(dead_code)]` on by default + + warning: `rexample` (bin "rexample") generated 3 warnings + Finished release [optimized] target(s) in 6.26s + [100%] Built target rexample + ``` + +4. 运行程序 + + ```bash + ../lcc/rlcc/rexample/target/release/rexample + ``` + +BTF信息可以忽略,下面的IP信息追踪即为trace结果 + +```bash +[root@iZ2zec8l5h2t7ee4suqxbmZ build]# ../lcc/rlcc/rexample/target/release/rexample +libbpf: Error loading BTF: Invalid argument(22) +libbpf: magic: 0xeb9f +version: 1 +flags: 0x0 +hdr_len: 24 +type_off: 0 +type_len: 13660 +str_off: 13660 +str_len: 9340 +btf_total_size: 23024 +[1] PTR (anon) type_id=3 +[2] INT int size=4 bits_offset=0 nr_bits=32 encoding=SIGNED +[3] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=4 +[4] INT __ARRAY_SIZE_TYPE__ size=4 bits_offset=0 nr_bits=32 encoding=(none) +[5] STRUCT (anon) size=24 vlen=3 + type type_id=1 bits_offset=0 + key_size type_id=1 bits_offset=64 + value_size type_id=1 bits_offset=128 +[6] INT events size=1 bits_offset=0 nr_bits=8 encoding=(none) +[7] PTR (anon) type_id=8 +[8] STRUCT pt_regs size=168 vlen=21 + r15 type_id=9 bits_offset=0 + r14 type_id=9 bits_offset=64 + r13 type_id=9 bits_offset=128 + r12 type_id=9 bits_offset=192 + bp type_id=9 bits_offset=256 + bx type_id=9 bits_offset=320 + r11 type_id=9 bits_offset=384 + r10 type_id=9 bits_offset=448 + r9 type_id=9 bits_offset=512 + r8 type_id=9 bits_offset=576 + ax type_id=9 bits_offset=640 + cx type_id=9 bits_offset=704 + dx type_id=9 bits_offset=768 + si type_id=9 bits_offset=832 + di type_id=9 bits_offset=896 + orig_ax type_id=9 bits_offset=960 + ip type_id=9 bits_offset=1024 + cs type_id=9 bits_offset=1088 + flags type_id=9 bits_offset=1152 + sp type_id=9 bits_offset=1216 + ss type_id=9 bits_offset=1280 +[9] INT long unsigned int size=8 bits_offset=0 nr_bits=64 encoding=(none) +[10] ENUM (anon) size=4 vlen=1 + ctx val=7 +[11] TYPEDEF tcp_connect type_id=10 +[12] Invalid btf_info:8400005a +libbpf: Error loading .BTF into kernel: -22. BTF is optional, ignoring. +(PID)548:exe (LOCAL)192.168.20.178:662 -> (REMOTE)100.100.105.70:20480 +(PID)1335:aliyun-service (LOCAL)192.168.20.178:6892 -> (REMOTE)100.100.18.120:47873 +(PID)548:exe (LOCAL)192.168.20.178:1686 -> (REMOTE)100.100.105.70:20480 +(PID)548:exe (LOCAL)192.168.20.178:63183 -> (REMOTE)100.100.27.15:14348 +(PID)548:exe (LOCAL)192.168.20.178:2710 -> (REMOTE)100.100.105.70:20480 +(PID)548:exe (LOCAL)192.168.20.178:3222 -> (REMOTE)100.100.105.70:20480 +``` +