From d909a69ac04c8e185294890b6cdfa44db7aec31c Mon Sep 17 00:00:00 2001 From: wynyibo <2548712106@qq.com> Date: Sat, 10 Aug 2024 17:57:21 +0800 Subject: [PATCH] add drop information and update makefile --- agent/plugin/Makefile | 62 ++++++++--- agent/plugin/common.bpf.h | 121 +++++++++++++++++--- agent/plugin/probe.bpf.c | 33 ++++-- agent/plugin/probe.c | 222 +++++++++++++++++++++++++++++++++---- agent/plugin/probe.h | 86 +++++++++++--- agent/plugin/traffic.bpf.h | 118 +++++++++++++++++++- 6 files changed, 568 insertions(+), 74 deletions(-) diff --git a/agent/plugin/Makefile b/agent/plugin/Makefile index 6d79d5c..d528f0d 100644 --- a/agent/plugin/Makefile +++ b/agent/plugin/Makefile @@ -8,6 +8,7 @@ LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a) BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool) BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool +# Determine architecture based on system's machine type ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/arm.*/arm/' \ | sed 's/aarch64/arm64/' \ @@ -17,27 +18,27 @@ ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \ | sed 's/loongarch64/loongarch/') VMLINUX := ./vmlinux.h - -# Use our own libbpf API headers and Linux UAPI headers distributed with -# libbpf to avoid dependency on system-wide headers, which could be missing or -# outdated INCLUDES := -I$(OUTPUT) -I../../libbpf/include/uapi -I$(dir $(VMLINUX)) CFLAGS := -g -Wall -Wno-unknown-pragmas ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS) +#kernel version +VERSION_INFO := $(shell uname -r | cut -d'-' -f1) +VERSION_MAJOR := $(shell echo $(VERSION_INFO) | cut -d'.' -f1) +VERSION_MINOR := $(shell echo $(VERSION_INFO) | cut -d'.' -f2) +VERSION_PATCH := $(shell echo $(VERSION_INFO) | cut -d'.' -f3) +export VERSION_INFO +export VERSION_MAJOR +export VERSION_MINOR +export VERSION_PATCH + +# User-space applications APPS = probe -# Get Clang's default includes on this system. We'll explicitly add these dirs -# to the includes list when compiling with `-target bpf` because otherwise some -# architecture-specific dirs will be "missing" on some architectures/distros - -# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h, -# sys/cdefs.h etc. might be missing. -# -# Use '-idirafter': Don't interfere with include mechanics except where the -# build would have failed anyways. +# Get system include paths for clang CLANG_BPF_SYS_INCLUDES = $(shell $(CLANG) -v -E - &1 \ | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') - +# Output message settings for make ifeq ($(V),1) Q = msg = @@ -59,13 +60,45 @@ endef $(call allow-override,CC,$(CROSS_COMPILE)cc) $(call allow-override,LD,$(CROSS_COMPILE)ld) +# Check if bpftool is installed, otherwise install it .PHONY: all -all: $(APPS) +all: dependence $(APPS) +.PHONY: dependence +dependence:check_bpftool check_clang + @echo "Kernel version is $(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_PATCH)" + bpftool btf dump file /sys/kernel/btf/vmlinux format c > $(VMLINUX) + +.PHONY: check_bpftool +check_bpftool: + @if ! command -v bpftool &> /dev/null; then \ + echo "bpftool Not installed, installing..."; \ + sudo yum update; \ + sudo yum install -y linux-tools-$(shell uname -r); \ + else \ + echo "bpftool Have been installed"; \ + fi +.PHONY: check_clang + +check_clang: + @if ! command -v clang &> /dev/null; then \ + echo "clang Not installed, installing..."; \ + sudo yum update; \ + sudo yum install -y clang; \ + else \ + echo "clang Have been installed"; \ + fi .PHONY: clean clean: $(call msg,CLEAN) $(Q)rm -rf $(OUTPUT) $(APPS) + +.PHONY: clean2 +clean2: + $(call msg,CLEAN) + rm -f $(APPS) + rm -f $(OUTPUT)/*.skel.h + rm -f $(OUTPUT)/*.o $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT): $(call msg,MKDIR,$@) @@ -83,6 +116,7 @@ $(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPU $(BPFTOOL): | $(BPFTOOL_OUTPUT) $(call msg,BPFTOOL,$@) $(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap + bpftool btf dump file /sys/kernel/btf/vmlinux format c > $(VMLINUX) # Build BPF code $(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL) diff --git a/agent/plugin/common.bpf.h b/agent/plugin/common.bpf.h index 21075ac..a9840b0 100644 --- a/agent/plugin/common.bpf.h +++ b/agent/plugin/common.bpf.h @@ -16,24 +16,31 @@ struct __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } udp_rb SEC(".maps"); - struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } tcp_rb SEC(".maps"); - struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); } tcp_output_rb SEC(".maps"); - struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 256 * 1024); -} port_events SEC(".maps"); +} port_events_rb SEC(".maps"); +struct +{ + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 256 * 1024); +} trace_all_drop SEC(".maps"); +struct +{ + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1024); +} perf_map SEC(".maps"); /*map helper*/ struct @@ -59,6 +66,7 @@ struct __type(key, struct sock *); __type(value, struct sock_stats_s); } tcp_link_map SEC(".maps"); + // packets struct { @@ -68,7 +76,21 @@ struct __type(value, struct packet_count); } proto_stats SEC(".maps"); -int udp_info = 0, tcp_status_info = 0, tcp_output_info = 0; +struct +{ + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 65536); + __type(key, u16); + __type(value, struct packet_info); +} port_count SEC(".maps"); + +struct +{ + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 65536); + __type(key, u32); + __type(value, struct tid_map_value); +} inner_tid_map SEC(".maps"); /*funcation hepler*/ static __always_inline int get_current_tgid() @@ -101,11 +123,19 @@ static __always_inline void get_udp_pkt_tuple(struct event *pkt_tuple, pkt_tuple->server_ip = _R(ip, daddr); pkt_tuple->client_port = __bpf_ntohs(_R(udp, source)); pkt_tuple->server_port = __bpf_ntohs(_R(udp, dest)); - pkt_tuple->seq = 0; - pkt_tuple->ack = 0; pkt_tuple->tran_flag = UDP; } +static void get_tcp_pkt_tuple(struct event *pkt_tuple, struct iphdr *ip, struct tcphdr *tcp) +{ + pkt_tuple->client_ip = _R(ip, saddr); + pkt_tuple->server_ip = _R(ip, daddr); + pkt_tuple->client_port = __bpf_ntohs(_R(tcp, source)); + pkt_tuple->server_port = __bpf_ntohs(_R(tcp, dest)); + pkt_tuple->seq = __bpf_ntohl(_R(tcp, seq)); + pkt_tuple->ack = __bpf_ntohl(_R(tcp, ack_seq)); +} + static __always_inline void *bmloti(void *map, const void *key, const void *init) { void *val; @@ -214,24 +244,89 @@ static __always_inline struct packet_count *count_packet(__u32 proto, bool is_tx initial_count.rx_count = 0; if (bpf_map_update_elem(&proto_stats, &proto, &initial_count, BPF_ANY)) { - // bpf_printk("proto:%u failed to initialize count\n", proto); - return NULL; + return 0; } count = bpf_map_lookup_elem(&proto_stats, &proto); if (!count) { - // bpf_printk("proto:%u count is NULL after initialization\n", proto); - return NULL; + return 0; } } - if (is_tx) __sync_fetch_and_add(&count->tx_count, 1); else __sync_fetch_and_add(&count->rx_count, 1); - // bpf_printk("proto:%u count_tx:%llu count_rx:%llu\n", proto, count->tx_count, count->rx_count); return count; } +static __always_inline u64 bpf_core_xt_table_name(void *ptr) +{ + struct xt_table *table = ptr; + if (bpf_core_field_exists(table->name)) + return (u64)(&table->name[0]); + return 0; +} +static __always_inline int fill_sk_skb(struct drop_event *event, struct sock *sk, struct sk_buff *skb) +{ + struct net *net = NULL; + struct iphdr ih = {}; + struct tcphdr th = {}; + struct udphdr uh = {}; + u16 protocol = 0; + bool has_netheader = false; + u16 network_header, transport_header; + char *head; + event->has_sk = false; + if (sk) + { + event->has_sk = true; + bpf_probe_read(&event->skbap.daddr, sizeof(event->skbap.daddr), &sk->__sk_common.skc_daddr); + bpf_probe_read(&event->skbap.dport, sizeof(event->skbap.dport), &sk->__sk_common.skc_dport); + bpf_probe_read(&event->skbap.saddr, sizeof(event->skbap.saddr), &sk->__sk_common.skc_rcv_saddr); + bpf_probe_read(&event->skbap.sport, sizeof(event->skbap.sport), &sk->__sk_common.skc_num); + event->skbap.dport = bpf_ntohs(event->skbap.dport); + protocol = _R(sk, __sk_common.skc_family); + bpf_probe_read(&event->sk_state, sizeof(event->sk_state), (const void *)&sk->__sk_common.skc_state); + // bpf_printk(" IP:%U Dip:%u", event->skbap.saddr, event->skbap.daddr); + } + // 从 sk_buff 中提取头部和网络层偏移信息 + bpf_probe_read(&head, sizeof(head), &skb->head); + bpf_probe_read(&network_header, sizeof(network_header), &skb->network_header); + if (network_header != 0) + { + bpf_probe_read(&ih, sizeof(ih), head + network_header); + has_netheader = true; + event->skbap.saddr = ih.saddr; + event->skbap.daddr = ih.daddr; + event->skb_protocol = ih.protocol; + transport_header = network_header + (ih.ihl << 2); + } + else + { + bpf_probe_read(&transport_header, sizeof(transport_header), &skb->transport_header); + } + switch (event->skb_protocol) + { + case IPPROTO_TCP: + bpf_probe_read(&th, sizeof(th), head + transport_header); + event->skbap.sport = bpf_ntohs(th.source); + event->skbap.dport = bpf_ntohs(th.dest); + break; + case IPPROTO_UDP: + if (transport_header != 0 && transport_header != 0xffff) + { + bpf_probe_read(&uh, sizeof(uh), head + transport_header); + event->skbap.sport = bpf_ntohs(uh.source); + event->skbap.dport = bpf_ntohs(uh.dest); + } + break; + case IPPROTO_ICMP: + break; + default: + return -1; + break; + } + return 0; +} #endif // __COMMON_BPF_H \ No newline at end of file diff --git a/agent/plugin/probe.bpf.c b/agent/plugin/probe.bpf.c index 6320607..d78af67 100644 --- a/agent/plugin/probe.bpf.c +++ b/agent/plugin/probe.bpf.c @@ -49,18 +49,35 @@ int __handle_tcp_cleanup_rbuf(struct pt_regs *ctx) return __tcp_cleanup_rbuf(ctx); } -//count the usage of protocol ports -//receive +// count the usage of protocol ports +// receive SEC("kprobe/eth_type_trans") -int BPF_KPROBE(eth_type_trans,struct sk_buff *skb) +int BPF_KPROBE(eth_type_trans, struct sk_buff *skb) { - bpf_printk("eth_type_trans"); return __eth_type_trans(skb); } -//send +// send SEC("kprobe/dev_hard_start_xmit") -int BPF_KPROBE(dev_hard_start_xmit,struct sk_buff *skb) +int BPF_KPROBE(dev_hard_start_xmit, struct sk_buff *skb) { - bpf_printk("dev_hard_start_xmit"); return __dev_hard_start_xmit(skb); -} \ No newline at end of file +} +// iptables drop +SEC("kprobe/ipt_do_table") +int BPF_KPROBE(ipt_do_table, struct sk_buff *skb, u32 hook, struct nf_hook_state *state) +{ + struct xt_table *table = (struct xt_table *)PT_REGS_PARM4(ctx); + return __ipt_do_table_start(ctx); +} +SEC("kretprobe/ipt_do_table") +int BPF_KRETPROBE(ipt_do_table_ret) +{ + int ret = PT_REGS_RC(ctx); + return __ipt_do_table_ret(ctx, ret); +} +SEC("tracepoint/skb/kfree_skb") +int handle_kfree_skb(struct trace_event_raw_kfree_skb *ctx) +{ + return __kfree_skb(ctx); +} + diff --git a/agent/plugin/probe.c b/agent/plugin/probe.c index 7f40e4e..7f9ba53 100755 --- a/agent/plugin/probe.c +++ b/agent/plugin/probe.c @@ -9,16 +9,18 @@ #include #include #include +#include #include #include "probe.h" #include "probe.skel.h" #include "probe.h" - +int stack_map_fd; // 在全局范围内声明 static volatile bool exiting = false; -int udp_info = 0, tcp_status_info = 0, tcp_output_info = 0, protocol_info = 0; +static int udp_info = 0, tcp_status_info = 0, tcp_output_info = 0, protocol_info = 0, port_distribution = 0, drop_info = 0, drop_skb = 0; struct protocol_stats proto_stats[256] = {0}; -time_t start_time; -int interval = 5; // 每5 秒计算一次 +static int interval = 5, entry_count = 0; // 每5 秒计算一次 +struct packet_info entries[MAX_ENTRIES]; +time_t start_time = 0; const char argp_program_doc[] = "Trace time delay in network subsystem \n"; @@ -26,7 +28,10 @@ static const struct argp_option opts[] = { {"udp", 'u', 0, 0, "trace the udp message"}, {"tcp_status_info", 't', 0, 0, "trace the tcp states"}, {"tcp_output_info", 'o', 0, 0, "trace the tcp flow"}, - {"protocol_info", 'p', 0, 0, "trace the tcp flow"}, + {"protocol_info", 'p', 0, 0, "statistics on the use of different protocols"}, + {"port_distribution_info", 'P', 0, 0, "statistical use of top10 destination ports"}, + {"drop_info", 'i', 0, 0, "trace the iptables drop"}, + {"drop_skb", 'd', 0, 0, "trace the all skb drop"}, {}, }; @@ -46,6 +51,15 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case 'p': protocol_info = 1; break; + case 'P': + port_distribution = 1; + break; + case 'i': + drop_info = 1; + break; + case 'd': + drop_skb = 1; + break; default: return ARGP_ERR_UNKNOWN; } @@ -171,16 +185,22 @@ void calculate_protocol_usage(struct protocol_stats proto_stats[], int num_proto for (int i = 0; i < num_protocols; i++) { - if (proto_stats[i].rx_count >= last_rx[i]) { + if (proto_stats[i].rx_count >= last_rx[i]) + { delta_rx[i] = proto_stats[i].rx_count - last_rx[i]; - } else { - delta_rx[i] = proto_stats[i].rx_count; + } + else + { + delta_rx[i] = proto_stats[i].rx_count; } - if (proto_stats[i].tx_count >= last_tx[i]) { + if (proto_stats[i].tx_count >= last_tx[i]) + { delta_tx[i] = proto_stats[i].tx_count - last_tx[i]; - } else { - delta_tx[i] = proto_stats[i].tx_count; + } + else + { + delta_tx[i] = proto_stats[i].tx_count; } current_rx += delta_rx[i]; @@ -223,25 +243,149 @@ void calculate_protocol_usage(struct protocol_stats proto_stats[], int num_proto memset(proto_stats, 0, num_protocols * sizeof(struct protocol_stats)); } +int compare_by_pps(const void *a, const void *b) +{ + return ((struct packet_info *)b)->packet_count - ((struct packet_info *)a)->packet_count; +} + +void init_start_time() +{ + start_time = time(NULL); +} + +int find_port_entry(int dst_port, int proto) +{ + for (int i = 0; i < entry_count; i++) + { + if (entries[i].dst_port == dst_port && entries[i].proto == proto) + { + return i; + } + } + return -1; +} + +static int print_drop(void *ctx, void *packet_info, size_t data_sz) +{ + if (!drop_info) + { + return 0; + } + const struct drop_event *event = (const struct drop_event *)packet_info; + + char s_str[INET_ADDRSTRLEN]; + char d_str[INET_ADDRSTRLEN]; + + if (!drop_info) + { + return 0; + } + + format_ip_address(event->skbap.saddr, s_str, sizeof(s_str)); + format_ip_address(event->skbap.daddr, d_str, sizeof(d_str)); + const char *type_str = event->type >= 0 && event->type < 4 ? drop_type_str[event->type] : "UNKNOWN"; + // protocol string + const char *proto_str = (event->skb_protocol >= 0 && event->skb_protocol < sizeof(protocol_names) / sizeof(char *) && + protocol_names[event->skb_protocol]) + ? protocol_names[event->skb_protocol] + : "UNKNOWN"; + printf("%-20d %-20s %-20s %-20d %-20d %-20s %-20s\n", + event->pid, s_str, d_str, event->skbap.sport, event->skbap.dport, + proto_str, type_str); + return 0; +} +static int print_drop_skb(void *ctx, void *packet_info, size_t data_sz) +{ + if (!drop_skb) + { + return 0; + } + const struct event *event = (const struct event *)packet_info; + char s_str[INET_ADDRSTRLEN]; + char d_str[INET_ADDRSTRLEN]; + char protol[6]; + if (event->client_ip == 0 && event->server_ip == 0) + { + return 0; + } + format_ip_address(event->client_ip, s_str, sizeof(s_str)); + format_ip_address(event->server_ip, d_str, sizeof(d_str)); + if (event->protocol == IPV4) + { + strcpy(protol, "ipv4"); + } + else if (event->protocol == IPV6) + { + strcpy(protol, "ipv6"); + } + else + { + strcpy(protol, "other"); + } + printf("%-20d %-20s %-20s %-20d %-20d %-20s %-34lx \n", + event->pid, s_str, d_str, event->client_port, event->server_port, protol, event->location); + return 0; +} static int print_count_protocol_use(void *ctx, void *packet_info, size_t data_sz) { const struct packet_info *pack_protocol_info = (const struct packet_info *)packet_info; + if (protocol_info) { - proto_stats[pack_protocol_info->proto].rx_count += pack_protocol_info->count.rx_count; - proto_stats[pack_protocol_info->proto].tx_count += pack_protocol_info->count.tx_count; + proto_stats[pack_protocol_info->proto].rx_count = pack_protocol_info->count.rx_count; + proto_stats[pack_protocol_info->proto].tx_count = pack_protocol_info->count.tx_count; + } + if (port_distribution) + { + // 查找当前端口号和协议号是否已经存在于 entries 数组中 + int index = find_port_entry(pack_protocol_info->dst_port, pack_protocol_info->proto); + if (index != -1) + { + entries[index].packet_count++; + } + else + { + if (entry_count >= MAX_ENTRIES) + { + printf("entry_count big"); + return 0; + } + entries[entry_count].dst_port = pack_protocol_info->dst_port; + entries[entry_count].proto = pack_protocol_info->proto; + entries[entry_count].packet_count = 1; + entry_count++; + } } return 0; } +static int print_top_5_keys() +{ + printf("Entry count: %d\n", entry_count); + // 使用 qsort 对 PPS 进行排序 + qsort(entries, entry_count, sizeof(struct packet_info), compare_by_pps); + // 输出前10个最频繁使用的端口号及其 PPS 值和协议号 + printf("==========Top %d Ports by PPS:\n", TOP_N); + for (int i = 0; i < TOP_N && i < entry_count; i++) + { + const char *proto_str = (entries[i].proto >= 0 && entries[i].proto <= 3) ? protocol[entries[i].proto] : "UNKNOWN"; + printf("Port: %d, PPS: %d, Protocol: %s\n", entries[i].dst_port, entries[i].packet_count, proto_str); + } + memset(entries, 0, entry_count * sizeof(struct packet_info)); + entry_count = 0; + return 0; +} int main(int argc, char **argv) { + struct probe_bpf *skel; int err = 0; struct ring_buffer *udp_rb = NULL; struct ring_buffer *tcp_rb = NULL; struct ring_buffer *tcp_output_rb = NULL; - struct ring_buffer *port_events = NULL; + struct ring_buffer *port_events_rb = NULL; + struct ring_buffer *perf_map = NULL; + struct ring_buffer *trace_all_drop = NULL; /* Parse command line arguments */ err = argp_parse(&argp, argc, argv, 0, NULL, NULL); if (err) @@ -270,7 +414,6 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to load and verify BPF skeleton\n"); goto cleanup; } - /* Attach tracepoints */ err = probe_bpf__attach(skel); if (err) @@ -301,13 +444,28 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to create ring buffer\n"); goto cleanup; } - port_events = ring_buffer__new(bpf_map__fd(skel->maps.port_events), print_count_protocol_use, NULL, NULL); - if (!port_events) + port_events_rb = ring_buffer__new(bpf_map__fd(skel->maps.port_events_rb), print_count_protocol_use, NULL, NULL); + if (!port_events_rb) + { + err = -1; + fprintf(stderr, "Failed to create ring buffer\n"); + goto cleanup; + } + perf_map = ring_buffer__new(bpf_map__fd(skel->maps.perf_map), print_drop, NULL, NULL); + if (!perf_map) + { + err = -1; + fprintf(stderr, "Failed to create ring buffer\n"); + goto cleanup; + } + trace_all_drop = ring_buffer__new(bpf_map__fd(skel->maps.trace_all_drop), print_drop_skb, NULL, NULL); + if (!trace_all_drop) { err = -1; fprintf(stderr, "Failed to create ring buffer\n"); goto cleanup; } + /* Process events */ if (udp_info) { @@ -321,18 +479,33 @@ int main(int argc, char **argv) { printf("%-20s %-20s %-20s %-20s %-20s %-20s %-20s %-20s %-20s %-20s\n", "Pid", "Client_ip", "Server_ip", "Client_port", "Server_port", "Send/bytes", "receive/bytes", "segs_in", "segs_out", "Direction"); } + if (drop_info) + { + printf("%-20s %-20s %-20s %-20s %-20s %-20s %-20s \n", "Pid", "Client_ip", "Server_ip", "Client_port", "Server_port", "Protocol", "Drop_type"); + } + if (drop_skb) + { + printf("%-20s %-20s %-20s %-20s %-20s %-20s %-20s \n", "Pid", "Client_ip", "Server_ip", "Client_port", "Server_port", "Protocol", "DROP_addr"); + } if (protocol_info) { printf("==========Proportion of each agreement==========\n"); } + if (port_distribution) + { + printf("==========port_distribution==========\n"); + } start_time = time(NULL); + struct timeval start, end; + gettimeofday(&start, NULL); while (!exiting) { err = ring_buffer__poll(udp_rb, 100 /* timeout, ms */); err = ring_buffer__poll(tcp_rb, 100 /* timeout, ms */); err = ring_buffer__poll(tcp_output_rb, 100 /* timeout, ms */); - err = ring_buffer__poll(port_events, 100 /* timeout, ms */); - + err = ring_buffer__poll(port_events_rb, 100 /* timeout, ms */); + err = ring_buffer__poll(perf_map, 100 /* timeout, ms */); + err = ring_buffer__poll(trace_all_drop, 100 /* timeout, ms */); /* Ctrl-C will cause -EINTR */ // Regularly calculate and print the proportion of agreements if (protocol_info) @@ -354,6 +527,13 @@ int main(int argc, char **argv) printf("Error polling perf buffer: %d\n", err); break; } + gettimeofday(&end, NULL); + if ((end.tv_sec - start.tv_sec) >= 5) + { + if (port_distribution) + print_top_5_keys(); + gettimeofday(&start, NULL); + } } cleanup: @@ -361,7 +541,9 @@ cleanup: ring_buffer__free(udp_rb); ring_buffer__free(tcp_rb); ring_buffer__free(tcp_output_rb); - ring_buffer__free(port_events); + ring_buffer__free(port_events_rb); + ring_buffer__free(perf_map); + ring_buffer__free(trace_all_drop); probe_bpf__destroy(skel); return err < 0 ? -err : 0; diff --git a/agent/plugin/probe.h b/agent/plugin/probe.h index b469714..15d02d7 100644 --- a/agent/plugin/probe.h +++ b/agent/plugin/probe.h @@ -12,19 +12,22 @@ typedef unsigned long long u64; #define IPV6_LEN 16 #define MAX_COMM 16 #define MAX_PACKET 1000 -#define AF_INET 2 #define BPF_MAP_TYPE_PERCPU_COUNTER 10 #define PID 32 #define MAX 256 #define MAX_SUM 1024 #define TASK_COMM_LEN 16 #define ETH_P_IP 0X0800 -#define IPPROTO_TCP 6 -#define IPPROTO_UDP 17 #define TCP 1 #define UDP 2 #define TIMEOUT_NS 5000ULL - +#define TOP_N 5 +#define MAX_ENTRIES 1000 +#define NF_DROP 0 +#define MAX_STACK_DEPTH 127 +#define XT_TABLE_MAXNAMELEN 100 +#define IPV4 2048 +#define IPV6 34525 #define TCP_TX_DATA(data, delta) __sync_fetch_and_add(&((data).tx), (__u64)(delta)) #define TCP_RX_DATA(data, delta) __sync_fetch_and_add(&((data).rx), (__u64)(delta)) #define TCP_PROBE_TXRX (u32)(1 << 3) @@ -59,6 +62,7 @@ struct event int oldstate; int newstate; u8 type; + long location; }; struct tcp_tx_rx @@ -100,18 +104,19 @@ enum struct packet_count { - u64 rx_count; - u64 tx_count; + u64 rx_count; + u64 tx_count; }; struct packet_info { - __u32 src_ip; - __u32 dst_ip; - __u16 src_port; - __u16 dst_port; - __u32 proto; - struct packet_count count; + u32 src_ip; + u32 dst_ip; + u16 src_port; + u16 dst_port; + u32 proto; + int packet_count; + struct packet_count count; }; struct protocol_stats @@ -120,6 +125,13 @@ struct protocol_stats uint64_t tx_count; }; +struct addr_pair +{ + u32 saddr; + u32 daddr; + u16 sport; + u16 dport; +}; static const char *tcp_states[] = { [1] = "ESTABLISHED", [2] = "SYN_SENT", @@ -135,11 +147,59 @@ static const char *tcp_states[] = { [12] = "NEW_SYN_RECV", [13] = "UNKNOWN", }; - static const char *protocol[] = { [0] = "TCP", [1] = "UDP", [2] = "ICMP", [3] = "UNKNOWN", }; + +struct tid_map_value +{ + struct sk_buff *skb; + struct nf_hook_state *state; + struct xt_table *table; + u32 hook; + void *ctx; +}; + +typedef u64 stack_trace_t[MAX_STACK_DEPTH]; +struct drop_event +{ + u8 type; + u8 name[32]; + u32 hook; + u32 pid; + u8 comm[16]; + u8 has_sk; + u8 skb_protocol; + u8 sk_state; + u8 sk_protocol; + struct addr_pair skbap; + signed int kstack_sz; + stack_trace_t kstack; +}; + +enum +{ + DROP_KFREE_SKB = 0, + DROP_TCP_DROP, + DROP_IPTABLES_DROP, + DROP_NFCONNTRACK_DROP, + UNK, +}; + +static const char *drop_type_str[] = { + "DROP_KFREE_SKB", + "DROP_TCP_DROP", + "DROP_IPTABLES_DROP", + "DROP_NFCONNTRACK_DROP", + "UNKNOWN"}; + +static const char *protocol_names[] = { + [IPPROTO_ICMP] = "ICMP", // 1 + [IPPROTO_TCP] = "TCP", // 6 + [IPPROTO_UDP] = "UDP", // 17 +}; + #endif diff --git a/agent/plugin/traffic.bpf.h b/agent/plugin/traffic.bpf.h index d8907ab..01e8652 100644 --- a/agent/plugin/traffic.bpf.h +++ b/agent/plugin/traffic.bpf.h @@ -226,7 +226,7 @@ static __always_inline int process_packet(struct sk_buff *skb, bool is_tx) const struct ethhdr *eth = (struct ethhdr *)_R(skb, data); u16 protocol = _R(eth, h_proto); - struct packet_info *pkt = bpf_ringbuf_reserve(&port_events, sizeof(*pkt), 0); + struct packet_info *pkt = bpf_ringbuf_reserve(&port_events_rb, sizeof(*pkt), 0); if (!pkt) { return 0; @@ -275,7 +275,7 @@ static __always_inline int process_packet(struct sk_buff *skb, bool is_tx) pkt->proto = PROTO_UNKNOWN; // bpf_printk("proto=%u\n", pkt->proto); } - // bpf_printk("proto=%u\n", pkt->proto); + struct packet_count *count = count_packet(pkt->proto, is_tx); if (count) { @@ -288,10 +288,6 @@ static __always_inline int process_packet(struct sk_buff *skb, bool is_tx) pkt->count.rx_count = 0; } - // bpf_printk("pkt: src_ip=%u, dst_ip=%u, proto=%u\n", pkt->src_ip, pkt->dst_ip, pkt->proto); - // bpf_printk("src_port=%d, dst_port=%d\n", pkt->src_port, pkt->dst_port); - // bpf_printk("count_tx=%llu, count_rx=%llu\n", pkt->count.tx_count, pkt->count.rx_count); - bpf_ringbuf_submit(pkt, 0); return 0; @@ -306,3 +302,113 @@ static __always_inline int __dev_hard_start_xmit(struct sk_buff *skb) { return process_packet(skb, true); // send } + +static __always_inline int __ipt_do_table_start(struct pt_regs *ctx) +{ + + u32 tid = bpf_get_current_pid_tgid(); + struct tid_map_value value = {}; + + struct sk_buff *skb = (struct sk_buff *)PT_REGS_PARM1(ctx); + struct nf_hook_state *state = (struct nf_hook_state *)PT_REGS_PARM3(ctx); + struct xt_table *table = (struct xt_table *)PT_REGS_PARM4(ctx); + u32 hook = (u32)PT_REGS_PARM2(ctx); + + value.skb = skb; + value.state = state; + value.hook = hook; + value.table = table; + + bpf_map_update_elem(&inner_tid_map, &tid, &value, BPF_ANY); + return 0; +} + +static __always_inline int submit_event(struct pt_regs *ctx, struct tid_map_value *value, u32 drop_type) +{ + struct sock *sk; + struct sk_buff *skb; + struct drop_event *event; + u64 addr; + + event = bpf_ringbuf_reserve(&perf_map, sizeof(struct drop_event), 0); + if (!event) + { + return 0; + } + event->type = drop_type; + skb = value->skb; + + bpf_probe_read(&sk, sizeof(sk), &skb->sk); + // 栈 + event->kstack_sz = + bpf_get_stack(ctx, event->kstack, sizeof(event->kstack), 0); + event->pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&event->comm, sizeof(event->comm)); + fill_sk_skb(event, sk, skb); + // 针对 iptables 的处理 + if (drop_type == DROP_IPTABLES_DROP) + { + struct xt_table *table = value->table; + addr = bpf_core_xt_table_name(table); + if (addr) + { + bpf_probe_read(event->name, sizeof(event->name), (void *)addr); + } + + event->hook = value->hook; + } + bpf_ringbuf_submit(event, 0); + return 1; +} + +static __always_inline int handle_drop_event(struct pt_regs *ctx, int ret, struct tid_map_value *value, u32 drop_type) +{ + if (ret != NF_DROP || !value) + { + return 0; + } + return submit_event(ctx, value, drop_type); +} + +static __always_inline int __ipt_do_table_ret(struct pt_regs *ctx, int ret) +{ + u32 tid = bpf_get_current_pid_tgid(); + struct tid_map_value *value = bpf_map_lookup_elem(&inner_tid_map, &tid); + + if (handle_drop_event(ctx, ret, value, DROP_IPTABLES_DROP)) + { + bpf_map_delete_elem(&inner_tid_map, &tid); + } + return 0; +} + +static __always_inline int __kfree_skb(struct trace_event_raw_kfree_skb *ctx) +{ + struct sk_buff *skb = ctx->skbaddr; + if (!skb) + return 0; + struct iphdr *ip = extract_iphdr(skb); + struct tcphdr *tcp = extract_tcphdr(skb); + struct event devent = {0}; + get_tcp_pkt_tuple(&devent, ip, tcp); + struct event *event; + event = bpf_ringbuf_reserve(&trace_all_drop, sizeof(*event), 0); + if (!event) + { + return 0; + } + event->location = (long)ctx->location; + // event->type == DROP_KFREE_SKB; + event->client_ip = devent.client_ip; + event->server_ip = devent.server_ip; + event->client_port = devent.client_port; + event->server_port = devent.server_port; + event->pid = get_current_tgid(); + event->protocol = ctx->protocol; + // bpf_printk("saddr:%u daddr:%u", event->client_ip, event->server_ip); + // 丢包调用栈 + // event->kstack_sz = + // bpf_get_stack(ctx, event->kstack, sizeof(event->kstack), 0); + bpf_ringbuf_submit(event, 0); + return 0; +} -- Gitee