From 166d93b306b36c3c4d20dc2821231b56fca16953 Mon Sep 17 00:00:00 2001 From: JofDiamonds Date: Mon, 17 Apr 2023 20:10:56 +0800 Subject: [PATCH] add proc file interface --- 0001-adapt-libbpf-0.8.1.patch | 19 +- 0003-add-proc-file-interface.patch | 735 +++++++++++++++++++++++++++++ README.md | 120 ++++- oncn-bwm.spec | 23 +- 4 files changed, 873 insertions(+), 24 deletions(-) create mode 100644 0003-add-proc-file-interface.patch diff --git a/0001-adapt-libbpf-0.8.1.patch b/0001-adapt-libbpf-0.8.1.patch index dfb319a..2fdfb1c 100644 --- a/0001-adapt-libbpf-0.8.1.patch +++ b/0001-adapt-libbpf-0.8.1.patch @@ -1,12 +1,12 @@ -From 0460a04fd537dffc527788613b03202191e10e8d Mon Sep 17 00:00:00 2001 +From 555072cbb13cb6cb45abb862f2b1a268c0fba660 Mon Sep 17 00:00:00 2001 From: JofDiamonds -Date: Tue, 15 Nov 2022 11:32:42 +0800 +Date: Mon, 17 Apr 2023 19:23:55 +0800 Subject: [PATCH] adapt libbpf-0.8.1 --- bpf/bwm_prio_kern.c | 14 ++++++------- - bwmcli.c | 50 ++++++++++++++++++++++++++++++--------------- - 2 files changed, 40 insertions(+), 24 deletions(-) + bwmcli.c | 51 ++++++++++++++++++++++++++++++--------------- + 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/bpf/bwm_prio_kern.c b/bpf/bwm_prio_kern.c index 1c59322..68127f9 100644 @@ -34,10 +34,10 @@ index 1c59322..68127f9 100644 SEC("cgroup_skb/egress") int _bwm_out_cg(struct __sk_buff *skb) diff --git a/bwmcli.c b/bwmcli.c -index f087ea6..75fbfab 100644 +index f087ea6..d9b59ad 100644 --- a/bwmcli.c +++ b/bwmcli.c -@@ -212,40 +212,59 @@ static int BreakArgs(char *s, char *arg1, char *arg2, unsigned long arg1Len, uns +@@ -212,40 +212,60 @@ static int BreakArgs(char *s, char *arg1, char *arg2, unsigned long arg1Len, uns return EXIT_OK; } @@ -103,6 +103,7 @@ index f087ea6..75fbfab 100644 + goto err; } ++ bpf_object__close(obj); return mapFd; +err: + bpf_object__close(obj); @@ -110,7 +111,7 @@ index f087ea6..75fbfab 100644 } static int GetMapFdByProgId(__u32 progId) -@@ -377,7 +396,6 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) +@@ -377,7 +397,6 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) int key = 0; int cgFd; int mapFd; @@ -118,7 +119,7 @@ index f087ea6..75fbfab 100644 int bpfprogFd; cgFd = open(cgrpPath, O_RDONLY); -@@ -392,7 +410,7 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) +@@ -392,7 +411,7 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) return EXIT_FAIL; } @@ -127,7 +128,7 @@ index f087ea6..75fbfab 100644 if (mapFd == -1) { goto err; } -@@ -416,11 +434,9 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) +@@ -416,11 +435,9 @@ static int CgrpV2PrioSet(const char *cgrpPath, int prio) BWM_LOG_INFO("set prio success\n"); (void)close(cgFd); diff --git a/0003-add-proc-file-interface.patch b/0003-add-proc-file-interface.patch new file mode 100644 index 0000000..be7ccb7 --- /dev/null +++ b/0003-add-proc-file-interface.patch @@ -0,0 +1,735 @@ +From 94b43d01f1f44e9fdea4c58510c386519763443a Mon Sep 17 00:00:00 2001 +From: JofDiamonds +Date: Mon, 17 Apr 2023 19:43:21 +0800 +Subject: [PATCH] add proc file interface + +--- + bwmcli.h | 6 +- + ko/Makefile | 16 ++ + ko/bwm.c | 599 ++++++++++++++++++++++++++++++++++++++++++++++++++++ + ko/bwm.h | 68 ++++++ + 4 files changed, 688 insertions(+), 1 deletion(-) + create mode 100644 ko/Makefile + create mode 100644 ko/bwm.c + create mode 100644 ko/bwm.h + +diff --git a/bwmcli.h b/bwmcli.h +index f7d4c62..a319130 100644 +--- a/bwmcli.h ++++ b/bwmcli.h +@@ -60,6 +60,10 @@ struct TcCmd { + + #define BWM_LOG_DEBUG(fmt, args...) syslog(LOG_DEBUG, "[BWM_DEBUG]: " fmt, ## args) + #define BWM_LOG_ERR(fmt, args...) syslog(LOG_ERR, "[BWM]: " fmt, ## args) +-#define BWM_LOG_INFO(fmt, args...) ((void)printf(fmt, ## args)) ++#define BWM_LOG_INFO(fmt, args...) \ ++ do { \ ++ (void)printf(fmt, ## args); \ ++ syslog(LOG_INFO, "[BWM]: " fmt, ## args); \ ++ } while (0) + + #endif /* __BWMCLI_H__ */ +diff --git a/ko/Makefile b/ko/Makefile +new file mode 100644 +index 0000000..7eae5f5 +--- /dev/null ++++ b/ko/Makefile +@@ -0,0 +1,16 @@ ++CONFIG_MODULE_SIG=n ++ ++obj-m := bwm.o ++ ++KERNELDIR ?= /lib/modules/$(shell uname -r)/build ++PWD := $(shell pwd) ++ ++LINUX_VER ?= $(shell ls /usr/src/ | grep linux) ++USERSRC ?= /usr/src/$(LINUX_VER) ++EXTRA_CFLAGS = -I$(USERSRC) ++ ++all: ++ $(MAKE) -C $(KERNELDIR) M=$(PWD) ++clean: ++ @rm -rf *.o *.mod *.mod.c *.mod.o *.ko *.order *.symvers .*.cmd ++ +diff --git a/ko/bwm.c b/ko/bwm.c +new file mode 100644 +index 0000000..c0bbc60 +--- /dev/null ++++ b/ko/bwm.c +@@ -0,0 +1,599 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * Description: Network bandwidth management tool ++ */ ++#include "bwm.h" ++ ++static char *envp[] = { "HOME=/", "PATH=/sbin:/usr/sbin:/bin:/usr/bin", NULL }; ++ ++static int qos_query_data(char *data) ++{ ++ int ret = 0; ++ struct file *fp; ++ ++ fp = filp_open(QOS_LOG_PATH, O_RDONLY, 0); ++ if (IS_ERR(fp)) { ++ BWM_LOG_ERR("open(%s) failed!", QOS_LOG_PATH); ++ return PTR_ERR(fp); ++ } ++ ++ ret = kernel_read(fp, data, MAX_DATA_SIZE, NULL); ++ if (ret < 0) { ++ BWM_LOG_ERR("read(%s) failed!", QOS_LOG_PATH); ++ filp_close(fp, NULL); ++ return ret; ++ } ++ filp_close(fp, NULL); ++ return 0; ++} ++ ++static ssize_t proc_value_get(const char __user *buffer, unsigned long count, char *value) ++{ ++ if (count == 0 || count >= MAX_BUF_SIZE) { ++ return -EINVAL; ++ } ++ ++ if (copy_from_user(value, buffer, count)) { ++ return -EINVAL; ++ } ++ ++ value[count - 1] = '\0'; ++ ++ return 0; ++} ++ ++static int proc_net_qos_enable_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s\n", net_qos_enable); ++ return 0; ++} ++ ++static int proc_net_qos_enable_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_enable_open, NULL); ++} ++ ++static int qos_enable_upcall(char *nspid) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -n -t %s %s -e", NSENTER_PATH, nspid, BWMCLI_PATH); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf enable qos cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("enable qos cmd:%s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t proc_net_qos_enable_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char nspid[MAX_BUF_SIZE] = { 0 }; ++ ++ ret = proc_value_get(buffer, count, nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ BWM_LOG_DEBUG("get nspid %s", nspid); ++ ++ ret = qos_enable_upcall(nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("qos_enable_upcall failed"); ++ } ++ ++ return count; ++} ++ ++static int proc_net_qos_disable_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s\n", net_qos_disable); ++ return 0; ++} ++ ++static int proc_net_qos_disable_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_disable_open, NULL); ++} ++ ++static int qos_disable_upcall(char *nspid) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -n -t %s %s -d", NSENTER_PATH, nspid, BWMCLI_PATH); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf disable qos cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("disable qos cmd:%s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t proc_net_qos_disable_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char nspid[MAX_BUF_SIZE] = { 0 }; ++ ++ ret = proc_value_get(buffer, count, nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ BWM_LOG_DEBUG("get nspid %s.", nspid); ++ ++ ret = qos_disable_upcall(nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("disable qos failed"); ++ } ++ ++ return count; ++} ++ ++static int proc_net_qos_bandwidth_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s\n", net_qos_bandwidth); ++ return 0; ++} ++ ++static int proc_net_qos_bandwidth_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_bandwidth_open, NULL); ++} ++ ++static int qos_bandwidth_upcall(char *bandwidth) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -s bandwidth %s", BWMCLI_PATH, bandwidth); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf bandwidth cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("set bandwidth cmd : %s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t proc_net_qos_bandwidth_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char bandwidth[MAX_BUF_SIZE] = { 0 }; ++ ++ ret = proc_value_get(buffer, count, bandwidth); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ BWM_LOG_DEBUG("change net_qos_bandwidth to %s.", bandwidth); ++ ++ ret = qos_bandwidth_upcall(bandwidth); ++ ret |= snprintf(net_qos_bandwidth, MAX_BUF_SIZE, "%s", bandwidth); ++ if (ret < 0) { ++ BWM_LOG_ERR("set bandwidth failed"); ++ return ret; ++ } ++ ++ return count; ++} ++ ++static int proc_net_qos_waterline_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s\n", net_qos_waterline); ++ return 0; ++} ++ ++static int proc_net_qos_waterline_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_waterline_open, NULL); ++} ++ ++static int qos_waterline_upcall(char *waterline) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -s waterline %s", BWMCLI_PATH, waterline); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf waterline cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("set waterline cmd : %s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t proc_net_qos_waterline_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char waterline[MAX_BUF_SIZE] = { 0 }; ++ ++ ret = proc_value_get(buffer, count, waterline); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ BWM_LOG_DEBUG("change net_qos_waterline to %s.", waterline); ++ ++ ret = qos_waterline_upcall(waterline); ++ ret |= snprintf(net_qos_waterline, MAX_BUF_SIZE, "%s", waterline); ++ if (ret < 0) { ++ BWM_LOG_ERR("set waterline failed"); ++ return ret; ++ } ++ ++ return count; ++} ++ ++static int proc_net_qos_devs_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s\n", net_qos_devs); ++ return 0; ++} ++ ++static int proc_net_qos_devs_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_devs_open, NULL); ++} ++ ++static int qos_devs_upcall(char *nspid) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -n -t %s %s -p devs", NSENTER_PATH, nspid, BWMCLI_PATH); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf devs status cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("devs status cmd:%s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } ++ ++ return ret; ++} ++ ++static ssize_t proc_net_qos_devs_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char nspid[MAX_BUF_SIZE] = { 0 }; ++ ++ ret = proc_value_get(buffer, count, nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ BWM_LOG_DEBUG("get nspid %s.", nspid); ++ ++ ret = qos_devs_upcall(nspid); ++ if (ret != 0) { ++ BWM_LOG_ERR("get devs status error"); ++ return ret; ++ } ++ ++ return count; ++} ++ ++static int proc_net_qos_stats_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%s", net_qos_stats); ++ return 0; ++} ++ ++static int proc_net_qos_stats_single_open(struct inode *inode, struct file *file) ++{ ++ int ret = 0; ++ char cmd[MAX_CMD_LEN] = { 0 }; ++ ++ ret = snprintf(cmd, MAX_CMD_LEN, "%s -p stats > %s", BWMCLI_PATH, QOS_LOG_PATH); ++ if (ret < 0) { ++ BWM_LOG_ERR("failed to snprintf stats cmd"); ++ return ret; ++ } ++ BWM_LOG_DEBUG("stats cmd:%s", cmd); ++ ++ char *argv[] = { ++ "/bin/bash", ++ "-c", ++ cmd, ++ NULL, ++ }; ++ ++ ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); ++ if (ret) { ++ BWM_LOG_ERR("call_usermodehelper failed, ret = %d", ret); ++ } else { ++ ret = qos_query_data(net_qos_stats); ++ if (ret != 0) { ++ BWM_LOG_ERR("qos_query_data failed, ret = %d", ret); ++ } ++ } ++ ++ return single_open(file, proc_net_qos_stats_open, NULL); ++} ++ ++static int proc_net_qos_version_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "version:%s\n", net_qos_version); ++ return 0; ++} ++ ++static int proc_net_qos_version_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_version_open, NULL); ++} ++ ++static int proc_net_qos_debug_open(struct seq_file *seq, void *offset) ++{ ++ seq_printf(seq, "%u\n", net_qos_debug); ++ return 0; ++} ++ ++static int proc_net_qos_debug_single_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_net_qos_debug_open, NULL); ++} ++ ++static ssize_t proc_net_qos_debug_write(struct file *file, const char __user *buffer, ++ unsigned long count, loff_t *ppos) ++{ ++ int ret = 0; ++ char debug_mode[MAX_BUF_SIZE] = { 0 }; ++ unsigned int net_qos_debug_new = net_qos_debug; ++ ++ ret = proc_value_get(buffer, count, debug_mode); ++ if (ret != 0) { ++ BWM_LOG_ERR("proc_value_get failed"); ++ return count; ++ } ++ ++ net_qos_debug_new = simple_strtoul(debug_mode, NULL, 0); ++ if (net_qos_debug_new != 0 && net_qos_debug_new != 1) { ++ BWM_LOG_ERR("invalid input of debug mode(%d), valid input: 0 or 1.", net_qos_debug_new); ++ return count; ++ } ++ ++ BWM_LOG_INFO("change debug mode, old is %u, new is %u.", net_qos_debug, net_qos_debug_new); ++ net_qos_debug = net_qos_debug_new; ++ ++ return count; ++} ++ ++static struct proc_ops bwm_proc_net_qos_enable_ops = { ++ .proc_open = proc_net_qos_enable_single_open, ++ .proc_write = proc_net_qos_enable_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_disable_ops = { ++ .proc_open = proc_net_qos_disable_single_open, ++ .proc_write = proc_net_qos_disable_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_bandwidth_ops = { ++ .proc_open = proc_net_qos_bandwidth_single_open, ++ .proc_write = proc_net_qos_bandwidth_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_waterline_ops = { ++ .proc_open = proc_net_qos_waterline_single_open, ++ .proc_write = proc_net_qos_waterline_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_devs_ops = { ++ .proc_open = proc_net_qos_devs_single_open, ++ .proc_write = proc_net_qos_devs_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_stats_ops = { ++ .proc_open = proc_net_qos_stats_single_open, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_version_ops = { ++ .proc_open = proc_net_qos_version_single_open, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct proc_ops bwm_proc_net_qos_debug_ops = { ++ .proc_open = proc_net_qos_debug_single_open, ++ .proc_write = proc_net_qos_debug_write, ++ .proc_read = seq_read, ++ .proc_release = single_release, ++}; ++ ++static struct bwm_proc g_proc_table[] = { ++ { ++ .proc_name = "net_qos_enable", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_enable_ops, ++ }, ++ { ++ .proc_name = "net_qos_disable", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_disable_ops, ++ }, ++ { ++ .proc_name = "net_qos_bandwidth", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_bandwidth_ops, ++ }, ++ { ++ .proc_name = "net_qos_waterline", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_waterline_ops, ++ }, ++ { ++ .proc_name = "net_qos_devs", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_devs_ops, ++ }, ++ { ++ .proc_name = "net_qos_stats", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_stats_ops, ++ }, ++ { ++ .proc_name = "net_qos_version", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_version_ops, ++ }, ++ { ++ .proc_name = "net_qos_debug", ++ .entry = NULL, ++ .ops = &bwm_proc_net_qos_debug_ops, ++ }, ++}; ++ ++static int bwm_proc_init(void) ++{ ++ unsigned int i; ++ int ret = 0; ++ ++ bwm_proc_root_dir = proc_mkdir(BWM_PROC_ROOTDIR, NULL); ++ if (!bwm_proc_root_dir) { ++ BWM_LOG_ERR("can't create /proc/%s.", BWM_PROC_ROOTDIR); ++ return -EFAULT; ++ } ++ ++ for (i = 0; i < sizeof(g_proc_table) / sizeof(struct bwm_proc); i++) { ++ g_proc_table[i].entry = proc_create_seq_data(g_proc_table[i].proc_name, BWM_CTL_PROC_PERM, ++ bwm_proc_root_dir, NULL, NULL); ++ if (!g_proc_table[i].entry) { ++ BWM_LOG_ERR("can't create file(%s).", g_proc_table[i].proc_name); ++ ret = -EFAULT; ++ break; ++ } ++ g_proc_table[i].entry->proc_ops = g_proc_table[i].ops; ++ } ++ ++ if (ret != 0) { ++ BWM_LOG_ERR("bwm_proc_init failed"); ++ } ++ ++ return ret; ++} ++ ++static void bwm_proc_clean(void) ++{ ++ unsigned int i; ++ ++ if (bwm_proc_root_dir) { ++ for (i = 0; i < sizeof(g_proc_table) / sizeof(struct bwm_proc); i++) { ++ if (g_proc_table[i].entry) { ++ remove_proc_entry(g_proc_table[i].proc_name, bwm_proc_root_dir); ++ g_proc_table[i].entry = NULL; ++ } ++ } ++ remove_proc_entry(BWM_PROC_ROOTDIR, NULL); ++ bwm_proc_root_dir = NULL; ++ } ++} ++ ++static int __init bwm_init(void) ++{ ++ int ret = 0; ++ ++ ret = bwm_proc_init(); ++ if (ret != 0) { ++ bwm_proc_clean(); ++ return ret; ++ } ++ ++ BWM_LOG_INFO("bwm loaded"); ++ return ret; ++} ++ ++static void __exit bwm_exit(void) ++{ ++ bwm_proc_clean(); ++ BWM_LOG_INFO("bwm unloaded"); ++} ++ ++module_init(bwm_init); ++module_exit(bwm_exit); ++MODULE_LICENSE("GPL"); +diff --git a/ko/bwm.h b/ko/bwm.h +new file mode 100644 +index 0000000..92ff2e6 +--- /dev/null ++++ b/ko/bwm.h +@@ -0,0 +1,68 @@ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * Description: Network bandwidth management tool ++ */ ++#ifndef NET_BWM_H ++#define NET_BWM_H ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#define BWM_PROC_ROOTDIR "qos" ++#define BWM_CTL_PROC_PERM 0644 ++ ++#define QOS_LOG_PATH "/tmp/qos.log" ++ ++#define NSENTER_PATH "/usr/bin/nsenter" ++#define BWMCLI_PATH "/usr/bin/bwmcli" ++ ++#define MAX_BUF_SIZE 128 ++#define MAX_CMD_LEN (256 + NAME_MAX) ++#define MAX_DATA_SIZE 1024 ++ ++static char net_qos_disable[] = "please write to net_qos_disable to disable qos"; ++static char net_qos_enable[] = "please write to net_qos_enable to enable qos"; ++static char net_qos_bandwidth[MAX_BUF_SIZE]; ++static char net_qos_waterline[MAX_BUF_SIZE]; ++static char net_qos_devs[] = "please write to net_qos_devs and read devs status in system log"; ++static char net_qos_stats[MAX_DATA_SIZE]; ++static char net_qos_version[] = "1.1"; ++static unsigned int net_qos_debug = 0; ++ ++static struct proc_dir_entry *bwm_proc_root_dir = NULL; ++ ++struct bwm_proc { ++ const char *proc_name; ++ struct proc_dir_entry *entry; ++ struct proc_ops *ops; ++}; ++ ++#define PFX "BWM" ++ ++#define BWM_LOG_INFO(fmt, ...) \ ++ do { \ ++ printk(KERN_INFO "[" PFX "] INFO:" fmt "[%s():%u]\n", ##__VA_ARGS__, __FUNCTION__, __LINE__); \ ++ } while (0) ++ ++#define BWM_LOG_NOTICE(fmt, ...) \ ++ do { \ ++ printk(KERN_NOTICE "[" PFX "] NOTICE:" fmt, ##__VA_ARGS__); \ ++ } while (0) ++ ++#define BWM_LOG_ERR(fmt, ...) \ ++ do { \ ++ printk(KERN_ERR "[" PFX "] ERROR:" fmt "[%s():%u]\n", ##__VA_ARGS__, __FUNCTION__, __LINE__); \ ++ } while (0) ++ ++#define BWM_LOG_DEBUG(fmt, ...) \ ++ do { \ ++ if (net_qos_debug == 1) { \ ++ printk(KERN_DEBUG "[" PFX "] DEBUG:" fmt "[%s():%u]\n", ##__VA_ARGS__, __FUNCTION__, __LINE__); \ ++ } \ ++ } while (0) ++ ++#endif +-- +2.33.0 + diff --git a/README.md b/README.md index 141266b..544d1c4 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ oncn-bwm │ ├─bpf # 两个ebpf程序文件实现带宽管理逻辑 │ +├─ko # 提供内核proc文件接口 +│ └─tools # 在离线带宽检测工具，基于bpftrace ``` @@ -43,21 +45,21 @@ oncn-bwm ## 使用说明 -### 接口说明 +### 命令行接口说明 **接口1** 说明 ``` -bwmcli –e/-d ethx 使能/除能某个网卡的Qos功能 -bwmcli –e/-d 使能/除能所有网卡的Qos功能 +bwmcli -e/-d ethx 使能/除能某个网卡的Qos功能 +bwmcli -e/-d 使能/除能所有网卡的Qos功能 ``` 示例 ``` -# bwmcli –e eth0 –e eth1 +# bwmcli -e eth0 -e eth1 enable eth0 success enable eth1 success -# bwmcli –d eth0 –d eth1 +# bwmcli -d eth0 -d eth1 disable eth0 success disable eth1 success ``` @@ -66,8 +68,8 @@ disable eth1 success 说明 ``` -bwmcli –s path 设置某个cgroup的优先级 -bwmcli –p path 查询某个cgroup的优先级 +bwmcli -s path 设置某个cgroup的优先级 +bwmcli -p path 查询某个cgroup的优先级 ``` 示例 ``` @@ -82,8 +84,8 @@ prio is 0 说明 ``` -bwmcli –s bandwidth 设置离线带宽 -bwmcli –p bandwidth 查询离线带宽 +bwmcli -s bandwidth 设置离线带宽 +bwmcli -p bandwidth 查询离线带宽 ``` 示例 ``` @@ -98,8 +100,8 @@ bandwidth is 31457280(B),104857600(B) 说明 ``` -bwmcli –s waterline 设置在线水线 -bwmcli –p waterline 查询在线水线 +bwmcli -s waterline 设置在线水线 +bwmcli -p waterline 查询在线水线 ``` 示例 ``` @@ -114,7 +116,7 @@ waterline is 20971520 (B) 说明 ``` -bwmcli –p stats 打印内部统计信息 +bwmcli -p stats 打印内部统计信息 ``` 示例 ``` @@ -130,15 +132,99 @@ offline rate past: 916194823 说明 ``` -bwmcli –p devs 描述系统上所有网卡的使能状态 +bwmcli -p devs 描述系统上所有网卡的使能状态 ``` 示例 ``` -# bwmcli –p devs +# bwmcli -p devs lo : disabled enp2s2 : disabled ``` +### proc文件接口说明 + +**接口1** + +说明 +``` +/proc/qos/net_qos_enable:使能网络qos功能 +``` + +示例 +``` +# 使能对应namespace中网络设备的qos功能 +echo $nspid > /proc/qos/net_qos_enable +``` + +**接口2** + +说明 +``` +/proc/qos/net_qos_disable:除能网络qos功能 +``` + +示例 +``` +# 除能对应namespace中网络设备的qos功能 +echo $nspid > /proc/qos/net_qos_disable +``` + +**接口3** + +说明 +``` +/proc/qos/net_qos_bandwidth:设置/查询离线业务带宽上下限 +``` + +示例 +``` +# 设置离线业务带宽上下限 +echo "$low,$high" > /proc/qos/net_qos_bandwidth +# 查询离线业务带宽上下限 +cat /proc/qos/net_qos_bandwidth +``` + +**接口4** + +说明 +``` +/proc/qos/net_qos_waterline:设置/查询在线业务带宽水线 +``` + +示例 +``` +# 设置在线业务带宽水线 +echo "$val" > /proc/qos/net_qos_waterline +# 查询在线业务带宽水线 +cat /proc/qos/net_qos_waterline +``` + +**接口5** + +说明 +``` +/proc/qos/net_qos_devs:查询对应namespace中网络设备qos功能使能状态，输入对应容器环境的nspid，结果在系统日志中体现 +``` + +示例 +``` +# 输入对应容器环境的nspid，qos使能状态在系统日志中体现 +echo $nspid > /proc/qos/net_qos_devs +``` + +**接口6** + +说明 +``` +/proc/qos/net_qos_stats:查询整个host环境上在离线业务的统计信息 +``` + +示例 +``` +# 查询在离线业务的统计信息 +cat /proc/qos/net_qos_stats +``` + ### 典型使用案例 ``` bwmcli -p devs 查询系统当前网卡使能状态 @@ -149,6 +235,12 @@ bwmcli -s bandwidth 20mb,1gb 配置离线业务带宽 bwmcli -s waterline 30mb 配置在线业务的水线 ``` +## 注意事项 + +1. 命令行接口和proc文件接口在设置离线业务带宽和在线业务水线上存在不同步的问题，通过proc文件接口设置的结果可以用命令行接口查询到，而通过命令行设置的结果不可以通过proc文件接口查询到。 +2. /proc/qos/net_qos_devs需要输入相应的nspid才能正确得到对应namespace中网络设备的qos功能状态，且其结果体现在系统日志中。 +3. /proc/qos/net_qos_stats接口在并发操作时可能存在查询结果不准确的情况，建议不要进行多并发操作。 + ## 参与贡献 1. Fork 本仓库 diff --git a/oncn-bwm.spec b/oncn-bwm.spec index a1367f4..7629621 100644 --- a/oncn-bwm.spec +++ b/oncn-bwm.spec @@ -1,6 +1,6 @@ Name: oncn-bwm Version: 1.1 -Release: 1 +Release: 2 Summary: Pod bandwidth management in mixed deployment scenarios of online and offline services License: GPL-2.0 URL: https://gitee.com/src-openeuler/oncn-bwm @@ -8,6 +8,7 @@ Source: %{name}-%{version}.tar.gz BuildRequires: libbpf-devel cmake gcc clang BuildRequires: libboundscheck +BuildRequires: uname-build-checks kernel-devel kernel-source Requires: iproute libbpf Requires(preun): bpftool @@ -15,6 +16,7 @@ Requires: libboundscheck Patch9001: 0001-adapt-libbpf-0.8.1.patch Patch9002: 0002-clean-code-and-use-securec-function.patch +Patch9003: 0003-add-proc-file-interface.patch %description Pod bandwidth management in mixed deployment scenarios of online and offline services @@ -33,6 +35,8 @@ devel tools for oncn-bwm mkdir build && cd build && cmake .. make +cd ../ko +make %install mkdir -p %{buildroot}/%{_bindir}/%{name} @@ -41,6 +45,12 @@ install -Dpm 0500 %{_builddir}/%{name}-%{version}/build/bpf/CMakeFiles/bwm_prio_ install -Dpm 0500 %{_builddir}/%{name}-%{version}/build/bpf/CMakeFiles/bwm_tc.dir/bwm_tc.c.o %{buildroot}/usr/share/bwmcli/bwm_tc.o install -Dpm 0500 %{_builddir}/%{name}-%{version}/build/bwmcli %{buildroot}/%{_bindir} install -Dpm 0500 %{_builddir}/%{name}-%{version}/tools/bwm_monitor.bt %{buildroot}/%{_bindir} +mkdir -p %{buildroot}/lib/modules/bwm +install %{_builddir}/%{name}-%{version}/ko/bwm.ko %{buildroot}/lib/modules/bwm + +%post +ln -sf /lib/modules/bwm/bwm.ko /lib/modules/`uname -r` +depmod -a %preun @@ -81,18 +91,29 @@ if [ $1 -eq 0 ]; then DisableAllDevices fi +%postun +if [ "$1" -ne "1" ]; then + rm -rf /lib/modules/`uname -r`/bwm.ko +fi +depmod -a + %files %defattr(-,root,root) %attr(0500,root,root) %{_bindir}/bwmcli %attr(0500,root,root) /usr/share/bwmcli %attr(0500,root,root) /usr/share/bwmcli/bwm_prio_kern.o %attr(0500,root,root) /usr/share/bwmcli/bwm_tc.o +%attr(0550,root,root) %dir /lib/modules/bwm +%attr(0440,root,root) /lib/modules/bwm/bwm.ko %files -n oncn-bwm-devel %attr(0500,root,root) %{_bindir}/bwm_monitor.bt %changelog +* Mon Apr 17 2023 JofDiamonds - 1.1-2 +- add proc file interface + * Wed Feb 15 2023 JofDiamonds - 1.1-1 - clean code and use securec function -- Gitee