From f1a5b90f5884d2f092ab5affb9d11fe88f9425e7 Mon Sep 17 00:00:00 2001 From: liqiang Date: Tue, 10 Sep 2024 15:51:59 +0800 Subject: [PATCH] add compile option of epoll proxy thread and fix some problems Signed-off-by: liqiang --- ...on-of-EPOLL_PROXY-and-fix-some-probl.patch | 370 ++++++++++++++++++ dpu-utilities.spec | 8 +- 2 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch diff --git a/0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch b/0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch new file mode 100644 index 0000000..8d02262 --- /dev/null +++ b/0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch @@ -0,0 +1,370 @@ +From 77a98b652e40ed72dbf8f72be0a17bc7f3b3967b Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Tue, 6 Aug 2024 19:52:06 +0800 +Subject: [PATCH 3/3] add compile option of EPOLL_PROXY and fix some problem of + vsock connection + +Signed-off-by: liqiang +--- + qtfs/qtfs/Makefile | 3 +++ + qtfs/qtfs/qtfs-mod.c | 13 ++++++++++++- + qtfs/qtfs_common/conn.c | 10 ++++++++++ + qtfs/qtfs_common/misc.c | 2 ++ + qtfs/qtfs_common/socket.c | 8 +++++++- + qtfs/qtfs_common/user_engine.c | 12 +++++++++++- + qtfs/qtfs_server/Makefile | 4 ++++ + qtfs/qtfs_server/qtfs-server.c | 12 ++++++++++++ + 8 files changed, 61 insertions(+), 3 deletions(-) + +diff --git a/qtfs/qtfs/Makefile b/qtfs/qtfs/Makefile +index 9b3ebe5..a6a6dca 100644 +--- a/qtfs/qtfs/Makefile ++++ b/qtfs/qtfs/Makefile +@@ -3,6 +3,9 @@ ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ip + else + ccflags-y += -I$(src)/../ -I$(src)/../utils/ -I$(src)/../include/ -I$(src)/../ipc/ -I$(src) -DQTFS_CLIENT + endif ++ifdef EPOLL_PROXY ++ccflags-y += -DEPOLL_PROXY ++endif + + KBUILD=/lib/modules/$(shell uname -r)/build/ + COMM=../qtfs_common/ +diff --git a/qtfs/qtfs/qtfs-mod.c b/qtfs/qtfs/qtfs-mod.c +index 0ce43fb..79256ad 100644 +--- a/qtfs/qtfs/qtfs-mod.c ++++ b/qtfs/qtfs/qtfs-mod.c +@@ -33,8 +33,9 @@ static struct file_system_type qtfs_fs_type = { + }; + MODULE_ALIAS_FS("qtfs"); + struct kmem_cache *qtfs_inode_priv_cache; ++#ifdef EPOLL_PROXY + struct task_struct *g_qtfs_epoll_thread = NULL; +- ++#endif + /* + * 转发框架层： + * 1. 调用者先在pvar里预留框架头后，填好自己的私有发送数据。 +@@ -124,6 +125,7 @@ retry: + return pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + } + ++#ifdef EPOLL_PROXY + static int qtfs_epoll_thread(void *data) + { + struct qtfs_conn_var_s *pvar = NULL; +@@ -202,6 +204,7 @@ end: + g_qtfs_epoll_thread = NULL; + return 0; + } ++#endif + + struct file_operations qtfs_misc_fops = { + .owner=THIS_MODULE, +@@ -236,12 +239,14 @@ static int __init qtfs_init(void) + goto conn_init_err; + } + qtfs_whitelist_initset(); ++#ifdef EPOLL_PROXY + g_qtfs_epoll_thread = kthread_run(qtfs_epoll_thread, NULL, "qtfs_epoll"); + if (IS_ERR_OR_NULL(g_qtfs_epoll_thread)) { + qtfs_err("qtfs epoll thread run failed.\n"); + ret = QTFS_PTR_ERR(g_qtfs_epoll_thread); + goto epoll_thread_err; + } ++#endif + qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL); + if (qtfs_diag_info == NULL) { + qtfs_err("kmalloc qtfs diag info failed."); +@@ -278,7 +283,9 @@ syscall_replace_err: + misc_register_err: + kfree(qtfs_diag_info); + diag_malloc_err: ++#ifdef EPOLL_PROXY + kthread_stop(g_qtfs_epoll_thread); ++#endif + epoll_thread_err: + qtfs_conn_param_fini(); + conn_init_err: +@@ -293,18 +300,22 @@ static void __exit qtfs_exit(void) + int ret; + qtfs_mod_exiting = true; + ++#ifdef EPOLL_PROXY + if (g_qtfs_epoll_thread) { + kthread_stop(g_qtfs_epoll_thread); + } ++#endif + + qtfs_conn_param_fini(); + qtfs_misc_destroy(); ++#ifdef EPOLL_PROXY + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + qtfs_epoll_var->conn_ops->conn_var_fini(qtfs_epoll_var); + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } ++#endif + qtfs_whitelist_exit(); + + kfree(qtfs_diag_info); +diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c +index 5f080cd..fd5f654 100644 +--- a/qtfs/qtfs_common/conn.c ++++ b/qtfs/qtfs_common/conn.c +@@ -42,7 +42,9 @@ static struct mutex g_param_mutex; + static struct mutex g_fifo_mutex; + int qtfs_mod_exiting = false; + struct qtfs_conn_var_s *qtfs_thread_var[QTFS_MAX_THREADS] = {NULL}; ++#ifdef EPOLL_PROXY + struct qtfs_conn_var_s *qtfs_epoll_var = NULL; ++#endif + #ifdef QTFS_SERVER + struct qtfs_server_userp_s *qtfs_userps = NULL; + #endif +@@ -520,7 +522,11 @@ void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir) + static int qtfs_sm_connecting(struct qtfs_conn_var_s *pvar) + { + int ret = QTERROR; ++#ifdef QTFS_CLIENT + int retry = 3; ++#else ++ int retry = 1; ++#endif + while (qtfs_mod_exiting == false && retry-- > 0) { + ret = pvar->conn_ops->conn_new_connection(&pvar->conn_var, pvar->user_type); + if (ret == 0) { +@@ -873,6 +879,7 @@ retry: + return pvar; + } + ++#ifdef EPOLL_PROXY + struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void) + { + struct qtfs_conn_var_s *pvar = NULL; +@@ -920,6 +927,7 @@ struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void) + qtfs_info("qtfs create new epoll param state:%s", QTCONN_CUR_STATE(pvar)); + return pvar; + } ++#endif + + void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar) + { +@@ -944,6 +952,7 @@ void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar) + mutex_unlock(&g_param_mutex); + } + ++#ifdef EPOLL_PROXY + void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar) + { + int ret = 0; +@@ -958,6 +967,7 @@ void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar) + qtfs_err("qtfs epoll put param exit failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + } + } ++#endif + + #ifdef QTFS_CLIENT + /* fifo的机制有所不同，每一个pvar对应唯一一个fifo的访问，生命周期贯穿 +diff --git a/qtfs/qtfs_common/misc.c b/qtfs/qtfs_common/misc.c +index e1c3520..3adfb57 100644 +--- a/qtfs/qtfs_common/misc.c ++++ b/qtfs/qtfs_common/misc.c +@@ -68,7 +68,9 @@ void qtfs_misc_flush_threadstate(void) + } + qtfs_diag_info->thread_state[i] = qtfs_thread_var[i]->state; + } ++#ifdef EPOLL_PROXY + qtfs_diag_info->epoll_state = (qtfs_epoll_var == NULL) ? -1 : qtfs_epoll_var->state; ++#endif + } + + void qtfs_req_size(void) +diff --git a/qtfs/qtfs_common/socket.c b/qtfs/qtfs_common/socket.c +index f247be0..233a1ce 100644 +--- a/qtfs/qtfs_common/socket.c ++++ b/qtfs/qtfs_common/socket.c +@@ -162,6 +162,11 @@ static int qtfs_conn_sock_server_accept(void *connvar, qtfs_conn_type_e type) + mutex_unlock(&qtfs_server_main_sock[type].lock); + return -EFAULT; + } ++ if (qtfs_server_thread_run == 0) { ++ qtfs_err("engine is exit now, return right now."); ++ mutex_unlock(&qtfs_server_main_sock[type].lock); ++ return -EINTR; ++ } + sock = qtfs_server_main_sock[type].sock; + + if (sock == NULL) { +@@ -246,7 +251,7 @@ static int qtfs_conn_sock_init(void *connvar, qtfs_conn_type_e type) + } + + qtfs_info("qtfs socket init sock OK, type:%d!", type); +- qtfs_sock_recvtimeo_set(sock, 0, 100000); ++ qtfs_sock_recvtimeo_set(sock, 1, 0); + qtfs_server_main_sock[type].sock = sock; + mutex_unlock(&qtfs_server_main_sock[type].lock); + return 0; +@@ -282,6 +287,7 @@ static int qtfs_conn_sock_client_connect(void *connvar, qtfs_conn_type_e type) + + ret = sock->ops->connect(sock, (struct sockaddr *)&saddr, sizeof(saddr), 0); + if (ret < 0) { ++ qtfs_err("qtfs client connect failed, ret:%d", ret); + return ret; + } + #ifdef QTFS_TEST_MODE +diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c +index a3edac2..a602980 100644 +--- a/qtfs/qtfs_common/user_engine.c ++++ b/qtfs/qtfs_common/user_engine.c +@@ -177,8 +177,8 @@ static void *qtfs_engine_kthread(void *arg) + break; + } + if (sig_int_flag == 1) { ++ sig_int_flag = 0; + engine_out("qtfs engine recv SIGINT."); +- + if (qtfs_fd < 0) { + engine_err("qtfs engine signal int file:%s open failed, fd:%d.", QTFS_SERVER_FILE, qtfs_fd); + continue; +@@ -204,6 +204,7 @@ static void qtfs_signal_int(int signum) + return; + } + ++#ifdef EPOLL_PROXY + static void *qtfs_engine_epoll_thread(void *data) + { + struct engine_arg *arg = (struct engine_arg *)data; +@@ -264,6 +265,7 @@ int qtfs_epoll_init(int fd) + + return epfd; + } ++#endif + + static void qtfs_whitelist_free_items(char **items, gsize count) + { +@@ -498,11 +500,13 @@ int main(int argc, char *argv[]) + } + qtfs_fd = fd; + // init epoll ++#ifdef EPOLL_PROXY + int epfd = qtfs_epoll_init(fd); + if (epfd < 0) { + close(fd); + return -1; + } ++#endif + ret = ioctl(fd, QTFS_IOCTL_EXIT, 1); + if (ret == QTERROR) { + goto end; +@@ -539,7 +543,9 @@ int main(int argc, char *argv[]) + arg[i].thread_idx = i; + (void)pthread_create(&texec[i], NULL, qtfs_engine_kthread, &arg[i]); + } ++#ifdef EPOLL_PROXY + (void)pthread_create(&tepoll, NULL, qtfs_engine_epoll_thread, &arg[0]); ++#endif + + #ifdef QTFS_TEST_MODE + fifo_arg.addr = argv[3]; +@@ -566,14 +572,18 @@ int main(int argc, char *argv[]) + pthread_join(texec[i], NULL); + engine_out("qtfs engine join thread %d.", i); + } ++#ifdef EPOLL_PROXY + pthread_join(tepoll, NULL); ++#endif + pthread_join(tfifo, NULL); + engine_free: + qtfs_engine_userp_free(userp, thread_nums); + engine_out("qtfs engine join epoll thread."); + end: + (void)ioctl(fd, QTFS_IOCTL_EXIT, 0); ++#ifdef EPOLL_PROXY + close(epfd); ++#endif + close(fd); + engine_out("qtfs engine over."); + return ret; +diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile +index 4608365..765567a 100644 +--- a/qtfs/qtfs_server/Makefile ++++ b/qtfs/qtfs_server/Makefile +@@ -4,6 +4,10 @@ CFLAGS += -DUDS_TEST_MODE -DQTFS_TEST_MODE + else + ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER + endif ++ifdef EPOLL_PROXY ++ccflags-y += -DEPOLL_PROXY ++CFLAGS += -DEPOLL_PROXY ++endif + + CFLAGS += -g -O2 + CFLAGS += -fstack-protector-strong +diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c +index f4e201c..48248ba 100644 +--- a/qtfs/qtfs_server/qtfs-server.c ++++ b/qtfs/qtfs_server/qtfs-server.c +@@ -59,6 +59,7 @@ void qtfs_server_epoll_exit(void) + return; + } + ++#ifdef EPOLL_PROXY + long qtfs_server_epoll_thread(struct qtfs_conn_var_s *pvar) + { + int n; +@@ -170,6 +171,7 @@ long qtfs_server_epoll_init(void) + + return QTOK; + } ++#endif + + int qtfs_server_fd_bitmap_init(void) + { +@@ -308,11 +310,16 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a + write_unlock(&qtfs_epoll_rwlock); + break; + case QTFS_IOCTL_EPOLL_THREAD_INIT: ++#ifdef EPOLL_PROXY + write_lock(&qtfs_epoll_rwlock); + ret = qtfs_server_epoll_init(); + write_unlock(&qtfs_epoll_rwlock); ++#else ++ qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it."); ++#endif + break; + case QTFS_IOCTL_EPOLL_THREAD_RUN: ++#ifdef EPOLL_PROXY + write_lock(&qtfs_epoll_rwlock); + if (qtfs_epoll_var == NULL) { + qtfs_err("qtfs epoll thread run failed, var is invalid."); +@@ -326,6 +333,9 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a + qtfs_info("qtfs epoll thread exit."); + qtfs_epoll_cut_conn(qtfs_epoll_var); + } ++#else ++ qtfs_warn("Qtfs not support epoll proxy, please compile with EPOLL_PROXY=1 to enable it."); ++#endif + break; + case QTFS_IOCTL_EXIT: + if (arg != 0 && arg != 1) { +@@ -403,6 +413,7 @@ static void __exit qtfs_server_exit(void) + + qtfs_conn_param_fini(); + ++#ifdef EPOLL_PROXY + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + qtfs_conn_fini(qtfs_epoll_var); +@@ -410,6 +421,7 @@ static void __exit qtfs_server_exit(void) + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } ++#endif + if (qtfs_diag_info != NULL) { + kfree(qtfs_diag_info); + qtfs_diag_info = NULL; +-- +2.37.1 (Apple Git-137.1) + diff --git a/dpu-utilities.spec b/dpu-utilities.spec index 8116475..f72f9fd 100644 --- a/dpu-utilities.spec +++ b/dpu-utilities.spec @@ -1,7 +1,7 @@ Name: dpu-utilities Summary: openEuler dpu utilities Version: 1.10 -Release: 4 +Release: 5 License: GPL-2.0 Source: https://gitee.com/openeuler/dpu-utilities/repository/archive/v%{version}.tar.gz ExclusiveOS: linux @@ -16,6 +16,7 @@ BuildRequires: kernel-devel >= 5.10, gcc, make, json-c-devel, glib2-devel Patch1: 0001-refactor-syscall-wrapper-for-aarch64-reduce-the-memo.patch Patch2: 0002-fix-readdir-bug-in-devtmpfs.patch Patch3: 0003-Change-the-maximum-number-of-qtfs-links-from-16-to-6.patch +Patch4: 0004-add-compile-option-of-EPOLL_PROXY-and-fix-some-probl.patch %description This package contains the software utilities on dpu. @@ -143,13 +144,16 @@ sed -i '/# product cut_conf/a\dpuos kiwi/minios/cfg_dpuos yes' /opt/imageT sed -i '//a\dpuos 1 rpm-dir euler_base' /opt/imageTailor/repos/RepositoryRule.conf %changelog +* Tue Sep 10 2024 liqiang 1.10-5 +- Add compile option for epoll proxy thread + * Tue Jul 23 2024 liqiang 1.10-4 - Change the maximum number of qtfs links from 16 to 64 * Wed Jul 10 2024 liqiang 1.10-3 - Fix readdir bug in devtmpfs -* Tue Jun 25 2024 liqiang 1.10-2 +* Mon Jun 24 2024 liqiang 1.10-2 - Refactor syscall wrapper for aarch64 reduce the memory * Fri May 24 2024 liqiang 1.10-1 -- Gitee