diff --git a/0001-rewrite-client-rexec_run.patch b/0001-rewrite-client-rexec_run.patch new file mode 100644 index 0000000000000000000000000000000000000000..4c4b0a79ba07105927d04c1890d6a95240e08ed1 --- /dev/null +++ b/0001-rewrite-client-rexec_run.patch @@ -0,0 +1,547 @@ +From 47fdab7bf180b058f6bbed10dd17e9a4c784eecc Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Thu, 1 Jun 2023 15:46:05 +0800 +Subject: rewrite client rexec_run + +Signed-off-by: liqiang +--- + qtfs/rexec/rexec.c | 256 +++++++++++++++++++++++++++++--------- + qtfs/rexec/rexec.h | 8 ++ + qtfs/rexec/rexec_server.c | 51 +++++--- + 3 files changed, 236 insertions(+), 79 deletions(-) + +diff --git a/qtfs/rexec/rexec.c b/qtfs/rexec/rexec.c +index 4dd206d..489ebec 100644 +--- a/qtfs/rexec/rexec.c ++++ b/qtfs/rexec/rexec.c +@@ -42,6 +42,21 @@ + #define REXEC_MSG_LEN 1024 + FILE *rexec_logfile = NULL; + ++struct rexec_global_var { ++ int rexec_hs_fd[2]; ++}; ++ ++struct rexec_global_var g_rexec; ++ ++ ++struct rexec_client_event { ++ int fd; ++ int outfd; // for stdin out err and other pipe ++ int (*handler)(struct rexec_client_event *); ++ int *exit_status; ++ int *pidfd; ++}; ++ + #define REXEC_PIDMAP_PATH "/var/run/rexec/pids" + #define REXEC_PIDMAP_PATH_LEN 64 + #define REXEC_PID_LEN 16 +@@ -84,37 +99,39 @@ static int rexec_msg_fill_argv(int argc, char *argv[], char *msg) + return offset; + } + +-static int rexec_io(int infd, int outfd, char *buf, int buflen) ++static int rexec_io(struct rexec_client_event *evt) + { ++#define MAX_MSG_LEN 256 ++ char buf[MAX_MSG_LEN]; + int len; + int ret; +- while ((len = read(infd, buf, buflen)) > 0) { +- ret = write(outfd, buf, len); ++ while ((len = read(evt->fd, buf, MAX_MSG_LEN)) > 0) { ++ ret = write(evt->outfd, buf, len); + if (ret <= 0) { +- rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", infd, len, outfd, ret); +- return -1; ++ rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", evt->fd, len, evt->outfd, ret); ++ return REXEC_EVENT_EXIT; + } + if (ret != len) { +- rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", infd, len, outfd, ret); ++ rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", evt->fd, len, evt->outfd, ret); + } + } +- return 0; ++ return REXEC_EVENT_OK; + } + + // return -1 means process exit. +-static int rexec_conn_msg(int connfd, int *exit_status, int *pidfd) ++static int rexec_conn_msg(struct rexec_client_event *evt) + { + struct rexec_msg head; +- int ret = recv(connfd, &head, sizeof(struct rexec_msg), MSG_WAITALL); ++ int ret = recv(evt->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); + if (ret <= 0) { + rexec_err("Rexec conn recv err:%d errno:%d", ret, errno); +- return -1; ++ return REXEC_EVENT_EXIT; + } + switch (head.msgtype) { + case REXEC_KILL: +- *exit_status = head.exit_status; ++ *evt->exit_status = head.exit_status; + rexec_err("Rexec conn recv kill msg, exit:%d now.", head.exit_status); +- return -1; ++ return REXEC_EVENT_EXIT; + case REXEC_PIDMAP: { + int mypid = getpid(); + int peerpid = head.pid; +@@ -122,9 +139,9 @@ static int rexec_conn_msg(int connfd, int *exit_status, int *pidfd) + char buf[REXEC_PID_LEN] = {0}; + int fd; + int err; +- if (*pidfd > 0) { ++ if (*evt->pidfd > 0) { + rexec_err("Rexec pidmap msg > 1 error."); +- return 0; ++ return REXEC_EVENT_OK; + } + sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, mypid); + fd = open(path, O_CREAT|O_WRONLY, 0600); +@@ -133,23 +150,41 @@ static int rexec_conn_msg(int connfd, int *exit_status, int *pidfd) + mypid, peerpid, path, fd); + break; + } +- *pidfd = fd; ++ *evt->pidfd = fd; + if ((err = flock(fd, LOCK_EX)) != 0) { + rexec_err("Rexec flock file:%s failed, errno:%d rexec exit.", path, err); +- return -1; ++ return REXEC_EVENT_EXIT; + } + if ((err = ftruncate(fd, 0)) != 0) { + rexec_err("Rexec pidmap file:%s clear failed errno:%d rexec exit.", path, err); +- return -1; ++ return REXEC_EVENT_EXIT; + } + if ((err = lseek(fd, 0, SEEK_SET)) < 0) { + rexec_err("Rexec pidmap file:%s lseek 0 failed errno:%d rexec exit", path, err); +- return -1; ++ return REXEC_EVENT_EXIT; + } + sprintf(buf, "%d", peerpid); + if ((err = write(fd, buf, strlen(buf))) <= 0) { + rexec_err("Rexec pidmap file:%s write pid:%d failed errno:%d rexec exit.", path, peerpid, err); +- return -1; ++ return REXEC_EVENT_EXIT; ++ } ++ if (g_rexec.rexec_hs_fd[PIPE_WRITE] != -1 && g_rexec.rexec_hs_fd[PIPE_READ] != -1) { ++ err = write(g_rexec.rexec_hs_fd[PIPE_WRITE], "1", 1); ++ if (err <= 0) { ++ rexec_err("rexec handshake write 1 failed, hs write:%d.", g_rexec.rexec_hs_fd[PIPE_WRITE]); ++ return REXEC_EVENT_ERR; ++ } ++ } else { ++ char msg[sizeof(struct rexec_msg) + 1]; ++ struct rexec_msg *hs = msg; ++ char *ok = hs->msg; ++ hs->msgtype = REXEC_HANDSHAKE; ++ hs->msglen = 1; ++ *ok = '1'; ++ if (write(evt->fd, hs, sizeof(struct rexec_msg) + 1) <= 0) { ++ rexec_err("send handshake failed, remote process will die"); ++ return REXEC_EVENT_EXIT; ++ } + } + break; + } +@@ -159,6 +194,35 @@ static int rexec_conn_msg(int connfd, int *exit_status, int *pidfd) + + rexec_log("Rexec conn recv msgtype:%d argc:%d stdno:%d msglen:%d", + head.msgtype, head.argc, head.stdno, head.msglen); ++ return REXEC_EVENT_OK; ++} ++ ++static struct rexec_client_event *rexec_add_event(int efd, int fd, int outfd, int (*handler)(struct rexec_client_event *)) ++{ ++ struct rexec_client_event *event = (struct rexec_client_event *)malloc(sizeof(struct rexec_client_event)); ++ if (event == NULL) { ++ rexec_err("malloc failed."); ++ return NULL; ++ } ++ event->fd = fd; ++ event->outfd = outfd; ++ event->handler = handler; ++ struct epoll_event evt; ++ evt.data.ptr = (void *)event; ++ evt.events = EPOLLIN; ++ if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { ++ rexec_err("epoll ctl add fd:%d event failed.", event->fd); ++ free(event); ++ return NULL; ++ } ++ return event; ++} ++ ++static int rexec_del_event(struct rexec_client_event *event) ++{ ++ // close will del fd in epoll list ++ close(event->fd); ++ free(event); + return 0; + } + +@@ -166,48 +230,48 @@ enum { + REPOL_IN_INDEX = 0, + REPOL_OUT_INDEX, + REPOL_ERR_INDEX, +- REPOL_CONN_INDEX, + REPOL_INV_INDEX, + }; +-static int rexec_run(int rstdin, int rstdout, int rstderr, int connfd, char *argv[]) ++static int rexec_std_event(int efd, int rstdin, int rstdout, int rstderr) + { +- int exit_status = EXIT_FAILURE; +-#define REXEC_MAX_EVENTS 4 +- int infds[4] = {STDIN_FILENO, rstdout, rstderr, connfd}; +- int outfds[4] = {rstdin, STDOUT_FILENO, STDERR_FILENO, connfd}; ++ #define REXEC_MAX_EVENTS 4 ++ int infds[REPOL_INV_INDEX] = {STDIN_FILENO, rstdout, rstderr}; ++ int outfds[REPOL_INV_INDEX] = {rstdin, STDOUT_FILENO, STDERR_FILENO}; + +- int efd = epoll_create1(0); +- if (efd == -1) { +- rexec_err("epoll create1 failed, errno:%d.", errno); +- return exit_status; +- } +- struct epoll_event evt; + for (int i = 0; i < REPOL_INV_INDEX; i++) { +- evt.data.u32 = i; +- evt.events = EPOLLIN; +- if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, infds[i], &evt)) { ++ if (NULL == rexec_add_event(efd, infds[i], outfds[i], rexec_io)) { + rexec_err("epoll ctl add fd:%d event failed and ignore this mistake.", infds[i]); + continue; + } else { + if (rexec_set_nonblock(infds[i], 1) != 0) { + rexec_err("rexec set fd:%d i:%d non block failed.", infds[i], i); +- return exit_status; ++ return -1; + } + } + } ++ return 0; ++} ++ ++static int rexec_run(int efd, int connfd, char *argv[]) ++{ ++ int pidfd = -1; ++ int exit_status = EXIT_FAILURE; ++ ++ struct rexec_client_event *connevt = rexec_add_event(efd, connfd, -1, rexec_conn_msg); ++ if (NULL == connevt || rexec_set_nonblock(connfd, 1) != 0) { ++ // process will exit, fd or mem resource will free by kernel soon ++ rexec_err("rexec add connfd event failed"); ++ return exit_status; ++ } ++ // 这两个指针只能在当前函数上下文使用,是当前函数栈指针 ++ connevt->exit_status = &exit_status; ++ connevt->pidfd = &pidfd; + + struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + rexec_err("init calloc evts failed."); + goto end; + } +- int buflen = REXEC_MSG_LEN; +- char *buf = (char *)malloc(buflen); +- int pidfd = -1; +- if (buf == NULL) { +- rexec_err("Rexec malloc failed."); +- goto free_end; +- } + rexec_log("Rexec process start run, as proxy of remote %s", argv[1]); + while (1) { + int n = epoll_wait(efd, evts, REXEC_MAX_EVENTS, 1000); +@@ -219,23 +283,16 @@ static int rexec_run(int rstdin, int rstdout, int rstderr, int connfd, char *arg + continue; + } + for (int i = 0; i < n; i++) { +- int infd = -1; +- int outfd = -1; +- if (evts[i].data.u32 >= REPOL_INV_INDEX) { +- rexec_err("invalid epoll events index data:%d", evts[i].data.u32); +- continue; ++ struct rexec_client_event *evt = (struct rexec_client_event *)evts[i].data.ptr; ++ int ret = evt->handler(evt); ++ if (evts[i].events & EPOLLHUP || ret == REXEC_EVENT_EXIT) { ++ process_exit = 1; + } +- infd = infds[evts[i].data.u32]; +- outfd = outfds[evts[i].data.u32]; +- if (infd == connfd) { +- if (evts[i].events & EPOLLHUP || rexec_conn_msg(connfd, &exit_status, &pidfd) == -1) +- process_exit = 1; +- } else { +- if (rexec_io(infd, outfd, buf, buflen) == -1) { +- close(infd); +- } ++ if (ret == REXEC_EVENT_DEL) { ++ rexec_del_event(evt); + } + } ++ // process will exit, and free all resource and exit + if (process_exit) { + rexec_log("Rexec process %s exit.", argv[1]); + break; +@@ -250,8 +307,6 @@ static int rexec_run(int rstdin, int rstdout, int rstderr, int connfd, char *arg + remove(path); + } + +- free(buf); +- + free_end: + free(evts); + +@@ -319,7 +374,7 @@ struct rexec_fdinfo { + int offset; + }; + +-static inline int rexec_is_reg_file(int fd) ++static inline unsigned int rexec_fd_mode(int fd) + { + struct stat st; + char path[32] = {0}; +@@ -327,9 +382,13 @@ static inline int rexec_is_reg_file(int fd) + rexec_err("get fd:%d fstat failed, errno:%d", fd, errno); + return 0; + } +- if (S_ISREG(st.st_mode)) { ++ return st.st_mode; ++} ++ ++static inline int rexec_is_reg_file(int fd) ++{ ++ if (S_ISREG(rexec_fd_mode(fd))) + return 1; +- } + return 0; + } + +@@ -429,16 +488,85 @@ err_end: + return NULL; + } + ++static int rexec_handshake_proc(struct rexec_client_event *evt) ++{ ++ char msg[sizeof(struct rexec_msg) + 1]; ++ struct rexec_msg *hs = msg; ++ int ret = read(evt->fd, hs->msg, 1); ++ if (ret <= 0) { ++ rexec_err("read from handshake pipe failed, ret:%d err:%d", ret, errno); ++ return REXEC_EVENT_DEL; ++ } ++ hs->msgtype = REXEC_HANDSHAKE; ++ hs->msglen = 1; ++ ret = write(evt->outfd, hs, sizeof(struct rexec_msg) + 1); ++ if (ret < 0) { ++ rexec_err("send handshake failed, connfd:%d.", evt->outfd); ++ } ++ return REXEC_EVENT_OK; ++} ++ ++static int rexec_handshake_init(int efd, int connfd) ++{ ++ char *hs_read = getenv("REXEC_HANDSHAKE_RD"); ++ char *hs_write = getenv("REXEC_HANDSHAKE_WR"); ++ ++ if (hs_read == NULL || hs_write == NULL) { ++ rexec_log("handshake not in effect, read:%lx write%lx", hs_read, hs_write); ++ return 0; ++ } ++ g_rexec.rexec_hs_fd[PIPE_READ] = atoi(hs_read); ++ g_rexec.rexec_hs_fd[PIPE_WRITE] = atoi(hs_write); ++ if (g_rexec.rexec_hs_fd[PIPE_READ] <= STDERR_FILENO || g_rexec.rexec_hs_fd[PIPE_WRITE] <= STDERR_FILENO) { ++ rexec_log("handshake invalid fd read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); ++ goto err_end; ++ } ++ if (!S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_READ])) || !S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_WRITE]))) { ++ rexec_err("handshake fd mode not fifo:%d %d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); ++ goto err_end; ++ } ++ if (rexec_add_event(efd, g_rexec.rexec_hs_fd[PIPE_READ], connfd, rexec_handshake_proc) == NULL) { ++ rexec_err("add handshake pipe read fd:%d to epoll failed", g_rexec.rexec_hs_fd[PIPE_READ]); ++ goto err_end; ++ } ++ rexec_log("handshake effect read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); ++ return 0; ++err_end: ++ g_rexec.rexec_hs_fd[PIPE_READ] = -1; ++ g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; ++ return -1; ++} ++ ++static void rexec_global_var_init() ++{ ++ memset(&g_rexec, 0, sizeof(g_rexec)); ++ g_rexec.rexec_hs_fd[PIPE_READ] = -1; ++ g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; ++ return; ++} ++ + int main(int argc, char *argv[]) + { + rexec_log_init(); + rexec_clear_pids(); + ++ int efd = epoll_create1(0); ++ if (efd == -1) { ++ rexec_err("epoll create1 failed, errno:%d.", errno); ++ return -1; ++ } ++ rexec_global_var_init(); ++ + int connfd = rexec_conn_to_server(); + if (connfd < 0) { + rexec_err("Rexec connect to server failed, errno:%d", errno); + return -1; + } ++ ++ if (rexec_handshake_init(efd, connfd) != 0) { ++ rexec_err("Rexec handshake environment set but get error."); ++ return -1; ++ } + rexec_log("Remote exec binary:%s", argv[1]); + + int arglen = rexec_calc_argv_len(argc - 1, &argv[1]); +@@ -513,7 +641,11 @@ int main(int argc, char *argv[]) + close(rstdin[0]); + close(rstdout[1]); + close(rstderr[1]); +- exit_status = rexec_run(rstdin[1], rstdout[0], rstderr[0], connfd, argv); ++ if (rexec_std_event(efd, rstdin[1], rstdout[0], rstderr[0]) != 0) { ++ rexec_err("add std event failed"); ++ goto err_end; ++ } ++ exit_status = rexec_run(efd, connfd, argv); + close(rstdin[1]); + close(rstdout[0]); + close(rstderr[0]); +diff --git a/qtfs/rexec/rexec.h b/qtfs/rexec/rexec.h +index ba7c2be..ce1280a 100644 +--- a/qtfs/rexec/rexec.h ++++ b/qtfs/rexec/rexec.h +@@ -24,6 +24,13 @@ enum { + PIPE_WRITE, + }; + ++enum { ++ REXEC_EVENT_OK, ++ REXEC_EVENT_DEL, // del this event ++ REXEC_EVENT_EXIT, // exit process ++ REXEC_EVENT_ERR, ++}; ++ + enum { + REXEC_STDIN = 0x5a, + REXEC_STDOUT, +@@ -45,6 +52,7 @@ enum rexec_msgtype { + REXEC_KILL, // kill process + REXEC_PIPE, // client send a pipefd as stdin/out/err to server + REXEC_PIDMAP, // server send remote process's pid to client ++ REXEC_HANDSHAKE, + }; + + struct rexec_msg { +diff --git a/qtfs/rexec/rexec_server.c b/qtfs/rexec/rexec_server.c +index 686c051..2aa3275 100644 +--- a/qtfs/rexec/rexec_server.c ++++ b/qtfs/rexec/rexec_server.c +@@ -65,12 +65,6 @@ struct rexec_event { + int (*handler)(struct rexec_event *); + }; + +-enum { +- REXEC_EVENT_OK, +- REXEC_EVENT_ERR, +- REXEC_EVENT_DEL, +-}; +- + static int rexec_add_event(int efd, int fd, int pid, int (*handler)(struct rexec_event *)) + { + struct rexec_event *event = (struct rexec_event *)malloc(sizeof(struct rexec_event)); +@@ -86,6 +80,7 @@ static int rexec_add_event(int efd, int fd, int pid, int (*handler)(struct rexec + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + rexec_err("epoll ctl add fd:%d event failed.", event->fd); ++ free(event); + return -1; + } + return 0; +@@ -136,15 +131,6 @@ static int rexec_event_handshake(struct rexec_event *event) + rexec_log("Rexec recv son pid:%d, connfd:%d", sonpid, connfd); + + rexec_hash_insert_direct(child_hash, sonpid, connfd); +- +- struct rexec_msg head; +- head.msgtype = REXEC_PIDMAP; +- head.msglen = 0; +- head.pid = sonpid; +- ret = write(connfd, &head, sizeof(struct rexec_msg)); +- if (ret <= 0) { +- rexec_err("Rexec send son pid:%d to client failed, ret:%d errno:%d", sonpid, ret, errno); +- } + rexec_add_event(main_epoll_fd, connfd, sonpid, rexec_event_process_manage); + + // 成功后同样要删除这个pipe监听事件,删除时会close掉fd +@@ -326,7 +312,7 @@ static int rexec_start_new_process(int newconnfd) + int scmfd = -1; + int len = sizeof(struct rexec_msg); + memset(&head, 0, sizeof(struct rexec_msg)); +- int ret = rexec_recvmsg(newconnfd, (char *)&head, len, &scmfd, MSG_WAITALL); ++ ret = rexec_recvmsg(newconnfd, (char *)&head, len, &scmfd, MSG_WAITALL); + if (ret <= 0) { + rexec_log("recvmsg ret:%d, errno:%d", ret, errno); + goto err_to_parent; +@@ -375,14 +361,45 @@ static int rexec_start_new_process(int newconnfd) + goto err_free; + } + ++ char *ack; + int mypid = getpid(); ++ char msg[sizeof(struct rexec_msg) + 1]; ++ struct rexec_msg *pm = msg; ++ pm->msgtype = REXEC_PIDMAP; ++ pm->msglen = 0; ++ pm->pid = mypid; ++ ret = write(newconnfd, pm, sizeof(struct rexec_msg)); ++ if (ret <= 0) { ++ rexec_err("Rexec send son pid:%d to client failed, ret:%d errno:%d", mypid, ret, errno); ++ } else { ++retry: ++ rexec_log("Waiting for rexec client handshake..."); ++ ret = read(newconnfd, pm, sizeof(struct rexec_msg) + 1); ++ if (ret <= 0) { ++ rexec_err("Recv handshake failed, ret:%d err:%d", ret, errno); ++ goto err_to_parent; ++ } ++ if (pm->msgtype != REXEC_HANDSHAKE) { ++ rexec_err("Recv unexpected msg:%d", pm->msgtype); ++ goto retry; ++ } ++ ack = pm->msg; ++ if (*ack != '1') { ++ rexec_err("recv error handshake ack from client:%c, exit now", *ack); ++ goto err_to_parent; ++ } ++ } + // 写会PID必须放在基于newconnfd接收完所有消息之后, + // 后面newconnfd的控制权交回父进程rexec server服务进程 +- write(pipefd[PIPE_WRITE], &mypid, sizeof(int)); ++ if (write(pipefd[PIPE_WRITE], &mypid, sizeof(int)) <= 0) { ++ rexec_err("write pid to parent failed, pipefd:%d.", pipefd[PIPE_WRITE]); ++ } + // 子进程不再使用pipe write和connfd + close(pipefd[PIPE_WRITE]); + close(newconnfd); + ++ rexec_log("handshake over normaly, continue to exec new process:%s.", binary); ++ + // rexec_shim_entry argv like: + // argv[0]: binary + // argv[1]: -f +-- +2.33.0 + diff --git a/0002-uds-add-vsock-mode.patch b/0002-uds-add-vsock-mode.patch new file mode 100644 index 0000000000000000000000000000000000000000..2d18fd31a50e398fdfc569223f1589d0e53f4298 --- /dev/null +++ b/0002-uds-add-vsock-mode.patch @@ -0,0 +1,171 @@ +From 9816a45d35ffa187b5830a417147afa9695dcead Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Sat, 3 Jun 2023 11:01:23 +0800 +Subject: uds add vsock mode + +Signed-off-by: liqiang +--- + qtfs/ipc/Makefile | 7 +++++++ + qtfs/ipc/uds_main.c | 43 ++++++++++++++++++++++++++++++++++++++----- + qtfs/ipc/uds_main.h | 12 +++++++++--- + 3 files changed, 54 insertions(+), 8 deletions(-) + +diff --git a/qtfs/ipc/Makefile b/qtfs/ipc/Makefile +index 2136f4f..3b851f3 100644 +--- a/qtfs/ipc/Makefile ++++ b/qtfs/ipc/Makefile +@@ -3,6 +3,9 @@ CFLAGS += -g -O2 + CFLAGS += -fstack-protector-strong + CFLAGS += -fPIE -pie -fPIC + CFLAGS += -D_FORTIFY_SOURCE=2 ++ifdef UDS_TEST_MODE ++CFLAGS += -DUDS_TEST_MODE ++endif + LDFLAGS += -s + LDFLAGS += -Wl,-z,now + LDFLAGS += -Wl,-z,noexecstack +@@ -12,6 +15,10 @@ all: udsproxyd libudsproxy.so + + udsproxyd: uds_event.o uds_main.o + gcc $(LDFLAGS) -o udsproxyd $^ -I../ $(DEPGLIB) ++ @test -z $(UDS_TEST_MODE) || echo "Important risk warning: The test mode is turned on,\ ++ and udsproxyd will expose the network port, which will bring security risks and is only for\ ++ testing! If you do not understand the risks, please don't use or compile again without\ ++ UDS_TEST_MODE." + + uds_event.o: + cc $(CFLAGS) -c -o uds_event.o uds_event.c $(DEPGLIB) +diff --git a/qtfs/ipc/uds_main.c b/qtfs/ipc/uds_main.c +index 2af4f06..adf0936 100644 +--- a/qtfs/ipc/uds_main.c ++++ b/qtfs/ipc/uds_main.c +@@ -37,6 +37,7 @@ + #include + #include + #include ++#include + + #include "comm.h" + #include "uds_main.h" +@@ -179,15 +180,24 @@ void uds_main_loop(int efd, struct uds_thread_arg *arg) + #define UDS_MAX_LISTEN_NUM 64 + int uds_build_tcp_connection(struct uds_conn_arg *arg) + { ++ int family = AF_VSOCK; + if (arg->cs > UDS_SOCKET_SERVER) { + uds_err("cs type %d is error.", arg->cs); + return -1; + } +- struct sockaddr_in sock_addr = { +- .sin_family = AF_INET, +- }; +- int sock_fd = socket(AF_INET, SOCK_STREAM, 0); ++#ifdef UDS_TEST_MODE ++ family = AF_INET; ++ struct sockaddr_in sock_addr; ++ memset(&sock_addr, 0, sizeof(sock_addr)); ++ sock_addr.sin_family = AF_INET; ++#else ++ family = AF_VSOCK; ++ struct sockaddr_vm sock_addr; ++ memset(&sock_addr, 0, sizeof(sock_addr)); ++ sock_addr.svm_family = AF_VSOCK; ++#endif + ++ int sock_fd = socket(family, SOCK_STREAM, 0); + if (sock_fd < 0) { + uds_err("As %s failed, socket fd: %d, errno:%d.", + (arg->cs == UDS_SOCKET_CLIENT) ? "client" : "server", +@@ -197,8 +207,13 @@ int uds_build_tcp_connection(struct uds_conn_arg *arg) + arg->sockfd = sock_fd; + + if (arg->cs == UDS_SOCKET_SERVER) { ++#ifdef UDS_TEST_MODE + sock_addr.sin_port = htons(p_uds_var->tcp.port); + sock_addr.sin_addr.s_addr = inet_addr(p_uds_var->tcp.addr); ++#else ++ sock_addr.svm_port = p_uds_var->vsock.port; ++ sock_addr.svm_cid = p_uds_var->vsock.cid; ++#endif + if (bind(sock_fd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + uds_err("As tcp server failed, bind error, errno:%d.", + errno); +@@ -209,13 +224,22 @@ int uds_build_tcp_connection(struct uds_conn_arg *arg) + goto close_and_return; + } + } else { ++#ifdef UDS_TEST_MODE + sock_addr.sin_port = htons(p_uds_var->tcp.peerport); + sock_addr.sin_addr.s_addr = inet_addr(p_uds_var->tcp.peeraddr); +- if (connect(arg->sockfd, (struct sockaddr *)&sock_addr, sizeof(struct sockaddr_in)) < 0) { ++#else ++ sock_addr.svm_port = p_uds_var->vsock.peerport; ++ sock_addr.svm_cid = p_uds_var->vsock.peercid; ++#endif ++ if (connect(arg->sockfd, (struct sockaddr *)&sock_addr, sizeof(sock_addr)) < 0) { + goto close_and_return; + } + arg->connfd = sock_fd; ++#ifdef UDS_TEST_MODE + uds_log("Connect to tcp server successed, ip:%s port:%u", p_uds_var->tcp.peeraddr, p_uds_var->tcp.peerport); ++#else ++ uds_log("Connect to vsock server successed, cid:%u port:%u", p_uds_var->vsock.peercid, p_uds_var->vsock.peerport); ++#endif + } + + return 0; +@@ -599,6 +623,7 @@ static int uds_glob_var_init(char *argv[]) + uds_err("work thread var malloc failed."); + return -1; + } ++#ifdef UDS_TEST_MODE + p_uds_var->tcp.port = atoi(argv[3]); + strncpy(p_uds_var->tcp.addr, argv[2], sizeof(p_uds_var->tcp.addr) - 1); + p_uds_var->tcp.peerport = atoi(argv[5]); +@@ -607,6 +632,14 @@ static int uds_glob_var_init(char *argv[]) + uds_log("uds proxy param thread num:%d ip:%s port:%u peerip:%s port:%u", + p_uds_var->work_thread_num, p_uds_var->tcp.addr, p_uds_var->tcp.port, + p_uds_var->tcp.peeraddr, p_uds_var->tcp.peerport); ++#else ++ // vsock param: ++ // port and peerport is checked before ++ p_uds_var->vsock.cid = atoi(argv[2]); ++ p_uds_var->vsock.port = myport; ++ p_uds_var->vsock.peercid = atoi(argv[4]); ++ p_uds_var->vsock.peerport = peerport; ++#endif + g_event_var = (struct uds_event_global_var *)malloc(sizeof(struct uds_event_global_var) * p_uds_var->work_thread_num); + if (g_event_var == NULL) { + free(p_uds_var->efd); +diff --git a/qtfs/ipc/uds_main.h b/qtfs/ipc/uds_main.h +index 3903ec8..de400f8 100644 +--- a/qtfs/ipc/uds_main.h ++++ b/qtfs/ipc/uds_main.h +@@ -111,15 +111,21 @@ struct uds_global_var { + struct uds_thread_arg *work_thread; + int loglevel; + char **logstr; ++#ifdef UDS_TEST_MODE + struct _tcp { + char addr[20]; + unsigned short port; + char peeraddr[20]; + unsigned short peerport; + } tcp; +- struct _uds { +- char sun_path[UDS_SUN_PATH_LEN]; +- } uds; ++#else ++ struct _vsock { ++ unsigned int cid; ++ unsigned int port; ++ unsigned int peercid; ++ unsigned int peerport; ++ } vsock; ++#endif + }; + enum uds_cs { + UDS_SOCKET_CLIENT = 1, +-- +2.33.0 + diff --git a/0003-fix-a-rexec_server-bug-on-child-process.patch b/0003-fix-a-rexec_server-bug-on-child-process.patch new file mode 100644 index 0000000000000000000000000000000000000000..6cbf848dd4113cd9832131d00bdd4b0a1d9a7649 --- /dev/null +++ b/0003-fix-a-rexec_server-bug-on-child-process.patch @@ -0,0 +1,26 @@ +From 754345d6dffc581e81670f4d1b18396003349a18 Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Mon, 5 Jun 2023 17:24:10 +0800 +Subject: fix a rexec_server bug on child process + +Signed-off-by: liqiang +--- + qtfs/rexec/rexec_server.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/qtfs/rexec/rexec_server.c b/qtfs/rexec/rexec_server.c +index 2aa3275..bdc6ea4 100644 +--- a/qtfs/rexec/rexec_server.c ++++ b/qtfs/rexec/rexec_server.c +@@ -431,7 +431,7 @@ err_to_parent: + write(pipefd[PIPE_WRITE], &errpid, sizeof(int)); + } while (0); + +- return ret; ++ exit(0); + } + + // 道生一 +-- +2.33.0 + diff --git a/0004-add-uds-test-mode-in-engine-compile.patch b/0004-add-uds-test-mode-in-engine-compile.patch new file mode 100644 index 0000000000000000000000000000000000000000..7cff0d96665d81e1790584ff5aa3725ca199a5be --- /dev/null +++ b/0004-add-uds-test-mode-in-engine-compile.patch @@ -0,0 +1,24 @@ +From 5185a08ec742ee044c4efa7fa5e1ae9fb454f80f Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Mon, 5 Jun 2023 20:53:39 +0800 +Subject: add uds test mode in engine compile + +Signed-off-by: liqiang +--- + qtfs/qtfs_server/Makefile | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile +index 0a8ed39..d00ce66 100644 +--- a/qtfs/qtfs_server/Makefile ++++ b/qtfs/qtfs_server/Makefile +@@ -1,5 +1,6 @@ + ifdef QTFS_TEST_MODE + ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER -DQTFS_TEST_MODE ++CFLAGS += -DUDS_TEST_MODE + else + ccflags-y += -I$(src)/../ -I$(src) -I$(src)/../ipc/ -I$(src)/../include/ -DQTFS_SERVER + endif +-- +2.33.0 + diff --git a/0005-modify-qtfs_dentry_revalidate.patch b/0005-modify-qtfs_dentry_revalidate.patch new file mode 100644 index 0000000000000000000000000000000000000000..99c3731016252bd33318fc1c28b13409aa346f52 --- /dev/null +++ b/0005-modify-qtfs_dentry_revalidate.patch @@ -0,0 +1,69 @@ +From 607c1e329ac107087fdadca0c997d3e0f66b3b06 Mon Sep 17 00:00:00 2001 +From: yangxin <245051644@qq.com> +Date: Thu, 8 Jun 2023 06:54:09 +0000 +Subject: modify qtfs_dentry_revalidate + +Signed-off-by: yangxin <245051644@qq.com> +--- + qtfs/qtfs/sb.c | 32 ++++++++++++++++++++++---------- + 1 file changed, 22 insertions(+), 10 deletions(-) + +diff --git a/qtfs/qtfs/sb.c b/qtfs/qtfs/sb.c +index b6bb3d7..2038c55 100644 +--- a/qtfs/qtfs/sb.c ++++ b/qtfs/qtfs/sb.c +@@ -1510,29 +1510,41 @@ const struct xattr_handler *qtfs_xattr_handlers[] = { + int qtfs_dentry_revalidate(struct dentry *dentry, unsigned int flags) + { + struct qtfs_conn_var_s *pvar = NULL; +- struct qtreq_mount *req = NULL; +- struct qtrsp_mount *rsp = NULL; +- +- // 1 means valid; 0 means invalid +- if (dentry && dentry->d_inode && S_ISDIR(dentry->d_inode->i_mode)) { ++ struct qtreq_getattr *req; ++ struct qtrsp_getattr *rsp; ++ struct inode *inode = dentry->d_inode; ++ if (dentry && dentry->d_inode) { + if (jiffies - dentry->d_time < 2000) + return 1; +- + pvar = qtfs_conn_get_param(); +- if (!pvar) ++ if (!pvar) { ++ qtfs_err("Failed to get qtfs sock var\n"); + return 0; ++ } + + req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); + qtfs_fullname(req->path, dentry, PATH_MAX); +- rsp = qtfs_remote_run(pvar, QTFS_REQ_MOUNT, strlen(req->path)); +- if (IS_ERR_OR_NULL(rsp) || rsp->ret != QTFS_OK) { ++ req->request_mask = STATX_BASIC_STATS; ++ req->query_flags = 0; ++ ++ rsp = qtfs_remote_run(pvar, QTFS_REQ_GETATTR, QTFS_SEND_SIZE(struct qtreq_getattr, req->path)); ++ if (IS_ERR_OR_NULL(rsp)) { ++ qtfs_conn_put_param(pvar); ++ return 0; ++ } ++ if (rsp->ret) { + qtfs_conn_put_param(pvar); + return 0; + } + ++ if (!inode || inode->i_ino != rsp->stat.ino || inode->i_mode != rsp->stat.mode) { ++ if (inode->i_nlink > 0) ++ drop_nlink(inode); ++ qtfs_conn_put_param(pvar); ++ return 0; ++ } + qtfs_conn_put_param(pvar); + dentry->d_time = jiffies; +- return 1; + } + return 1; + } +-- +2.33.0 + diff --git a/0006-add-cmake-compile-for-user-binary-and-so.patch b/0006-add-cmake-compile-for-user-binary-and-so.patch new file mode 100644 index 0000000000000000000000000000000000000000..daa931c5ed30afa8cd8f2bd2373efd9ff30491c4 --- /dev/null +++ b/0006-add-cmake-compile-for-user-binary-and-so.patch @@ -0,0 +1,55 @@ +From fbde8e2d0b6f66daf7aacb7348dffb97721bca7c Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Thu, 8 Jun 2023 15:58:09 +0800 +Subject: add cmake compile for user binary and so + +Signed-off-by: liqiang +--- + qtfs/CMakeLists.txt | 34 ++++++++++++++++++++++++++++++++++ + 1 file changed, 34 insertions(+) + create mode 100644 qtfs/CMakeLists.txt + +diff --git a/qtfs/CMakeLists.txt b/qtfs/CMakeLists.txt +new file mode 100644 +index 0000000..d5b65ec +--- /dev/null ++++ b/qtfs/CMakeLists.txt +@@ -0,0 +1,34 @@ ++cmake_minimum_required(VERSION 3.0.0) ++ ++project(qtfs) ++ ++set(CMAKE_C_FLAGS "-g -O2 -fstack-protector-strong -fPIE -pie -fPIC -D_FORTIFY_SOURCE=2 -s -Wl,-z,now -Wl,-z,noexecstack") ++ ++# Build rexec and rexec_server ++add_executable(rexec rexec/rexec.c rexec/rexec_sock.c) ++add_executable(rexec_server rexec/rexec_server.c rexec/rexec_sock.c rexec/rexec_shim.c) ++target_include_directories(rexec_server PRIVATE /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) ++target_link_libraries(rexec PRIVATE json-c) ++target_link_libraries(rexec_server PRIVATE json-c glib-2.0) ++ ++# Build udsproxyd and libudsproxy.so ++add_executable(udsproxyd ipc/uds_event.c ipc/uds_main.c) ++add_library(udsproxy SHARED ipc/uds_connector.c) ++target_include_directories(udsproxyd PRIVATE include/ /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) ++target_link_libraries(udsproxyd PRIVATE pthread glib-2.0) ++ ++# Build engine ++add_executable(engine ipc/uds_main.c ipc/uds_event.c qtfs_common/user_engine.c) ++target_include_directories(engine PRIVATE include/ ./ ipc/ /usr/include/glib-2.0 /usr/lib64/glib-2.0/include) ++target_link_libraries(engine PRIVATE glib-2.0 pthread) ++target_compile_options(engine PRIVATE "-DQTFS_SERVER") ++ ++if(DEFINED UDS_TEST_MODE OR DEFINED QTFS_TEST_MODE) ++target_compile_options(engine PRIVATE "-DUDS_TEST_MODE") ++target_compile_options(udsproxyd PRIVATE "-DUDS_TEST_MODE") ++message(WARNING "Important risk warning: the test mode is turned on, and qtfs will expose the network port, \ ++ which will bring security risks and is only for testing! If you do not understand the risks,\ ++ please don't use or compile again without test mode macro!") ++endif() ++ ++set(ignoreMe "${QTFS_TEST_MODE}${UDS_TEST_MODE}") +\ No newline at end of file +-- +2.33.0 + diff --git a/0007-uds-remove-MSG_WAITALL-flags-to-timeout.patch b/0007-uds-remove-MSG_WAITALL-flags-to-timeout.patch new file mode 100644 index 0000000000000000000000000000000000000000..46499d3afeb27404aac8dd5409a38d0970573022 --- /dev/null +++ b/0007-uds-remove-MSG_WAITALL-flags-to-timeout.patch @@ -0,0 +1,150 @@ +From 596a9553d90516375de169acdd513e4df26eab9d Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Thu, 8 Jun 2023 15:27:06 +0800 +Subject: uds remove MSG_WAITALL flags to timeout + +Signed-off-by: liqiang +--- + qtfs/ipc/uds_event.c | 16 ++++++++-------- + qtfs/ipc/uds_main.c | 36 ++++++++++++++++++++++++++++++++++++ + qtfs/ipc/uds_main.h | 1 + + 3 files changed, 45 insertions(+), 8 deletions(-) + +diff --git a/qtfs/ipc/uds_event.c b/qtfs/ipc/uds_event.c +index b046b69..047949b 100644 +--- a/qtfs/ipc/uds_event.c ++++ b/qtfs/ipc/uds_event.c +@@ -214,7 +214,7 @@ int uds_event_build_step2(void *arg, int epfd, struct uds_event_global_var *p_ev + struct uds_proxy_remote_conn_rsp rsp; + int len; + memset(buf, 0, sizeof(buf)); +- len = recv(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req), MSG_WAITALL); ++ len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req)); + if (len == 0) { + uds_err("recv err msg:%d errno:%d", len, errno); + return EVENT_DEL; +@@ -390,7 +390,7 @@ int uds_event_tcp_listener(void *arg, int epfd, struct uds_event_global_var *p_e + int uds_build_connect2uds(struct uds_event *evt, struct uds_proxy_remote_conn_req *msg) + { + struct uds_conn_arg targ; +- int len = recv(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req), MSG_WAITALL); ++ int len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_proxy_remote_conn_req)); + if (len <= 0) { + uds_err("recv failed, len:%d errno:%d", len, errno); + return EVENT_ERR; +@@ -441,7 +441,7 @@ err_ack: + + int uds_build_pipe_proxy(int efd, struct uds_event *evt, struct uds_stru_scm_pipe *msg) + { +- int len = recv(evt->fd, msg, sizeof(struct uds_stru_scm_pipe), MSG_WAITALL); ++ int len = uds_recv_with_timeout(evt->fd, msg, sizeof(struct uds_stru_scm_pipe)); + if (len <= 0) { + uds_err("recv failed, len:%d errno:%d", len, errno); + return EVENT_ERR; +@@ -477,7 +477,7 @@ int uds_event_remote_build(void *arg, int epfd, struct uds_event_global_var *p_e + int len; + int ret = EVENT_OK; + memset(p_event_var->iov_base, 0, p_event_var->iov_len); +- len = recv(evt->fd, bdmsg, sizeof(struct uds_tcp2tcp), MSG_WAITALL); ++ len = uds_recv_with_timeout(evt->fd, bdmsg, sizeof(struct uds_tcp2tcp)); + if (len <= 0) { + uds_err("read no msg from sock:%d, len:%d", evt->fd, len); + return EVENT_DEL; +@@ -711,7 +711,7 @@ int uds_msg_tcp2uds_scm_pipe(struct uds_tcp2tcp *p_msg, struct uds_event *evt, i + int scmfd; + int fd[SCM_PIPE_NUM]; + struct uds_stru_scm_pipe *p_pipe = (struct uds_stru_scm_pipe *)p_msg->data; +- int len = recv(evt->fd, p_pipe, p_msg->msglen, MSG_WAITALL); ++ int len = uds_recv_with_timeout(evt->fd, p_pipe, p_msg->msglen); + if (len <= 0) { + uds_err("recv data failed, len:%d", len); + return EVENT_DEL; +@@ -913,7 +913,7 @@ int uds_event_tcp2uds(void *arg, int epfd, struct uds_event_global_var *p_event_ + msg.msg_controllen = p_event_var->msg_controlsendlen; + + while (1) { +- int len = recv(evt->fd, p_msg, sizeof(struct uds_tcp2tcp), MSG_WAITALL); ++ int len = uds_recv_with_timeout(evt->fd, p_msg, sizeof(struct uds_tcp2tcp)); + if (len <= 0) { + uds_err("recv no msg maybe sock is closed, delete this tcp2uds event, len:%d.", len); + goto close_event; +@@ -932,7 +932,7 @@ int uds_event_tcp2uds(void *arg, int epfd, struct uds_event_global_var *p_event_ + uds_err("normal msg repeat recv fd:%d", evt->fd); + goto err; + } +- normal_msg_len = recv(evt->fd, p_event_var->iov_base_send, p_msg->msglen, MSG_WAITALL); ++ normal_msg_len = uds_recv_with_timeout(evt->fd, p_event_var->iov_base_send, p_msg->msglen); + if (normal_msg_len <= 0) { + uds_err("recv msg error:%d fd:%d", len, evt->fd); + goto close_event; +@@ -950,7 +950,7 @@ int uds_event_tcp2uds(void *arg, int epfd, struct uds_event_global_var *p_event_ + } + memset(p_scm->path, 0, sizeof(p_scm->path)); + // SCM RIGHTS msg proc +- len = recv(evt->fd, p_msg->data, p_msg->msglen, MSG_WAITALL); ++ len = uds_recv_with_timeout(evt->fd, p_msg->data, p_msg->msglen); + if (len <= 0) { + uds_err("recv data failed len:%d", p_msg->msglen); + return EVENT_DEL; +diff --git a/qtfs/ipc/uds_main.c b/qtfs/ipc/uds_main.c +index adf0936..65ae81d 100644 +--- a/qtfs/ipc/uds_main.c ++++ b/qtfs/ipc/uds_main.c +@@ -88,6 +88,42 @@ int uds_event_delete(int efd, int fd) + return 0; + } + ++int uds_recv_with_timeout(int fd, char *msg, int len) ++{ ++#define TMOUT_BLOCK_SIZE 1024 ++#define TMOUT_UNIT_MS 20 ++#define TMOUT_INTERVAL 1 ++#define TMOUT_MAX_MS 1000 ++ int total_recv = 0; ++ int ret; ++ int tmout_ms = ((len / TMOUT_BLOCK_SIZE) + 1) * TMOUT_UNIT_MS; ++ if (len <= 0 || msg == NULL || fd < 0) { ++ uds_err("invalid param fd:%d len:%d or %s", fd, len, (msg == NULL) ? "msg is NULL" : "msg is not NULL"); ++ return 0; ++ } ++ if (tmout_ms > TMOUT_MAX_MS) ++ tmout_ms = TMOUT_MAX_MS; ++ do { ++ ret = recv(fd, &msg[total_recv], len - total_recv, 0); ++ if (ret < 0) { ++ uds_err("recv failed ret:%d errno:%d", ret, errno); ++ return ret; ++ } ++ total_recv += ret; ++ if (total_recv > len) { ++ uds_err("fatal error total recv:%d longger than target len:%d", total_recv, len); ++ return 0; ++ } ++ if (total_recv == len) { ++ return total_recv; ++ } ++ usleep(TMOUT_INTERVAL * 1000); ++ tmout_ms -= TMOUT_INTERVAL; ++ } while (tmout_ms > 0); ++ uds_err("Fatal error, the target recv len:%d and only %d length is received when it time out", len, total_recv); ++ return 0; ++} ++ + #pragma GCC diagnostic ignored "-Wpointer-to-int-cast" + int uds_event_tmout_item(gpointer key, gpointer value, gpointer data) + { +diff --git a/qtfs/ipc/uds_main.h b/qtfs/ipc/uds_main.h +index de400f8..f69e4b3 100644 +--- a/qtfs/ipc/uds_main.h ++++ b/qtfs/ipc/uds_main.h +@@ -152,6 +152,7 @@ int uds_event_insert(int efd, struct uds_event *event); + int uds_hash_insert_dirct(GHashTable *table, int key, struct uds_event *value); + void *uds_hash_lookup_dirct(GHashTable *table, int key); + int uds_hash_remove_dirct(GHashTable *table, int key); ++int uds_recv_with_timeout(int fd, char *msg, int len); + + #ifdef QTFS_SERVER + int uds_proxy_main(int argc, char *argv[]); +-- +2.33.0 + diff --git a/0008-Modify-the-logic-of-rexec-proxy-process-exit-by-mist.patch b/0008-Modify-the-logic-of-rexec-proxy-process-exit-by-mist.patch new file mode 100644 index 0000000000000000000000000000000000000000..ea1e1b54968500552f29aa50d2958af9c7643d3e --- /dev/null +++ b/0008-Modify-the-logic-of-rexec-proxy-process-exit-by-mist.patch @@ -0,0 +1,35 @@ +From 802188d8d9a06a0c963ecee0b7d30b379dfe2814 Mon Sep 17 00:00:00 2001 +From: yangxin <245051644@qq.com> +Date: Thu, 8 Jun 2023 12:22:58 +0000 +Subject: Modify the logic of rexec proxy process exit by mistake + +Signed-off-by: yangxin <245051644@qq.com> +--- + qtfs/rexec/rexec.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/qtfs/rexec/rexec.c b/qtfs/rexec/rexec.c +index 489ebec..5439fac 100644 +--- a/qtfs/rexec/rexec.c ++++ b/qtfs/rexec/rexec.c +@@ -109,7 +109,7 @@ static int rexec_io(struct rexec_client_event *evt) + ret = write(evt->outfd, buf, len); + if (ret <= 0) { + rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", evt->fd, len, evt->outfd, ret); +- return REXEC_EVENT_EXIT; ++ return REXEC_EVENT_DEL; + } + if (ret != len) { + rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", evt->fd, len, evt->outfd, ret); +@@ -285,7 +285,7 @@ static int rexec_run(int efd, int connfd, char *argv[]) + for (int i = 0; i < n; i++) { + struct rexec_client_event *evt = (struct rexec_client_event *)evts[i].data.ptr; + int ret = evt->handler(evt); +- if (evts[i].events & EPOLLHUP || ret == REXEC_EVENT_EXIT) { ++ if (ret == REXEC_EVENT_EXIT) { + process_exit = 1; + } + if (ret == REXEC_EVENT_DEL) { +-- +2.33.0 + diff --git a/0009-Keep-doc-same-with-docs-in-openEuler.patch b/0009-Keep-doc-same-with-docs-in-openEuler.patch new file mode 100644 index 0000000000000000000000000000000000000000..d7ae771f1db0db0826b0c6619fc0176d18fc2a26 --- /dev/null +++ b/0009-Keep-doc-same-with-docs-in-openEuler.patch @@ -0,0 +1,325 @@ +From 67b7259d810c9e07227e7a1475acdc1141e4424e Mon Sep 17 00:00:00 2001 +From: Weifeng Su +Date: Thu, 8 Jun 2023 14:17:00 +0000 +Subject: Keep doc same with docs in openEuler + +Signed-off-by: Weifeng Su +--- + qtfs/doc/overview.md | 11 ++ + ...77\347\224\250\346\211\213\345\206\214.md" | 69 ++++++++ + ...40\346\204\237\345\215\270\350\275\275.md" | 31 ++++ + ...50\347\275\262\346\214\207\345\257\274.md" | 166 ++++++++++++++++++ + 7 files changed, 277 insertions(+) + create mode 100644 qtfs/doc/overview.md + create mode 100644 "qtfs/doc/qtfs\345\205\261\344\272\253\346\226\207\344\273\266\347\263\273\347\273\237\346\236\266\346\236\204\345\217\212\344\275\277\347\224\250\346\211\213\345\206\214.md" + create mode 100644 "qtfs/doc/\345\256\271\345\231\250\347\256\241\347\220\206\351\235\242\346\227\240\346\204\237\345\215\270\350\275\275.md" + create mode 100644 "qtfs/doc/\346\227\240\346\204\237\345\215\270\350\275\275\351\203\250\347\275\262\346\214\207\345\257\274.md" + +diff --git a/qtfs/doc/overview.md b/qtfs/doc/overview.md +new file mode 100644 +index 0000000..518deb0 +--- /dev/null ++++ b/qtfs/doc/overview.md +@@ -0,0 +1,11 @@ ++# 容器管理面DPU无感卸载指南 ++ ++本文档介绍基于openEuler操作系统的容器管理面DPU无感卸载功能特性及安装部署方法,该特性可以通过操作系统提供的统一抽象层,屏蔽容器管理面跨主机资源访问的差异,实现容器管理面业务无感卸载到DPU上。 ++ ++本文档适用于使用openEuler系统并希望了解和使用操作系统内核及容器的社区开发者、开源爱好者以及相关合作伙伴。使用人员需要具备以下经验和技能: ++ ++- 熟悉Linux基本操作 ++ ++- 熟悉linux内核文件系统相关基础机制 ++ ++- 对kubernetes和docker有一定了解,熟悉docker及kubernetes部署及使用 +\ No newline at end of file + +diff --git "a/qtfs/doc/qtfs\345\205\261\344\272\253\346\226\207\344\273\266\347\263\273\347\273\237\346\236\266\346\236\204\345\217\212\344\275\277\347\224\250\346\211\213\345\206\214.md" "b/qtfs/doc/qtfs\345\205\261\344\272\253\346\226\207\344\273\266\347\263\273\347\273\237\346\236\266\346\236\204\345\217\212\344\275\277\347\224\250\346\211\213\345\206\214.md" +new file mode 100644 +index 0000000..8088f48 +--- /dev/null ++++ "b/qtfs/doc/qtfs\345\205\261\344\272\253\346\226\207\344\273\266\347\263\273\347\273\237\346\236\266\346\236\204\345\217\212\344\275\277\347\224\250\346\211\213\345\206\214.md" +@@ -0,0 +1,69 @@ ++# qtfs ++ ++## 介绍 ++ ++qtfs是一个共享文件系统项目,可部署在host-dpu的硬件架构上,也可以部署在2台服务器之间。以客户端服务器的模式工作,使客户端能通过qtfs访问服务端的指定文件系统,得到本地文件访问一致的体验。 ++ ++qtfs的特性: ++ +++ 支持挂载点传播; ++ +++ 支持proc、sys、cgroup等特殊文件系统的共享; ++ +++ 支持远程文件读写的共享; ++ +++ 支持在客户端对服务端的文件系统进行远程挂载; ++ +++ 支持特殊文件的定制化处理; ++ +++ 支持远端fifo、unix-socket等,并且支持epoll,使客户端和服务端像本地通信一样使用这些文件; ++ +++ 支持基于host-dpu架构通过PCIe协议底层通信,性能大大优于网络; ++ +++ 支持内核模块形式开发,无需对内核进行侵入式修改。 ++ ++## 软件架构 ++ ++软件大体框架图: ++ ++![qtfs-arch](./figures/qtfs-arch.png) ++ ++## 安装教程 ++ ++目录说明: ++ +++ **qtfs**: 客户端内核模块相关代码,直接在该目录下编译客户端ko。 ++ +++ **qtfs_server**: 服务端内核模块相关代码,直接在该目录下编译服务端ko和相关程序。 ++ +++ **qtinfo**: 诊断工具,支持查询文件系统的工作状态以及修改log级别等。 ++ +++ **demo**、**test**、**doc**: 测试程序、演示程序以及项目资料等。 ++ +++ 根目录: 客户端与服务端通用的公共模块代码。 ++ ++首先找两台服务器(或虚拟机)配置内核编译环境: ++ ++ 1. 要求内核版本在5.10或更高版本。 ++  2. 安装内核开发包:yum install kernel-devel。 ++ ++服务端安装: ++ ++ 1. cd qtfs_server ++ 2. make clean && make ++ 3. insmod qtfs_server.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN ++ 4. ./engine 4096 16 ++ ++客户端安装: ++ ++ 1. cd qtfs ++ 2. make clean && make ++ 3. insmod qtfs.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN ++ ++## 使用说明 ++ ++安装完成后,客户端通过挂载把服务端的文件系统让客户端可见,例如: ++ ++ mount -t qtfs / /root/mnt/ ++ ++客户端进入"/root/mnt"后便可查看到server端的所有文件,以及对其进行相关操作。 +diff --git "a/qtfs/doc/\345\256\271\345\231\250\347\256\241\347\220\206\351\235\242\346\227\240\346\204\237\345\215\270\350\275\275.md" "b/qtfs/doc/\345\256\271\345\231\250\347\256\241\347\220\206\351\235\242\346\227\240\346\204\237\345\215\270\350\275\275.md" +new file mode 100644 +index 0000000..2e4be2f +--- /dev/null ++++ "b/qtfs/doc/\345\256\271\345\231\250\347\256\241\347\220\206\351\235\242\346\227\240\346\204\237\345\215\270\350\275\275.md" +@@ -0,0 +1,31 @@ ++# 容器管理面无感卸载介绍 ++ ++## 概述 ++ ++在数据中心及云场景下,随着摩尔定律失效,通用处理单元CPU算力增长速率放缓,而同时网络IO类速率及性能不断攀升,二者增长速率差异形成的剪刀差,即当前通用处理器的处理能力无法跟上网络、磁盘等IO处理的需求。传统数据中心下越来越多的通用CPU算力被IO及管理面等占用,这部分资源损耗称之为数据中心税(Data-center Tax)。据AWS统计,数据中心税可能占据数据中心算力的30%以上,部分场景下甚至可能更多。 ++ ++DPU的出现就是为了将这部分算力资源从主机CPU上解放出来,通过将管理面、网络、存储、安全等能力卸载到专有的处理器芯片(DPU)上进行处理加速,达成降本增效的结果。目前主流云厂商如AWS、阿里云、华为云都通过自研芯片完成管理面及相关数据面的卸载,达成数据中心计算资源100%售卖给客户。 ++ ++管理面进程卸载到DPU可以通过对组件源码进行拆分达成,将源码根据功能逻辑拆分成独立运行的两部分,分别运行在主机和DPU,达成组件卸载的目的。但是这种做法有以下问题:一是影响组件的软件兼容性,组件后续版本升级和维护需要自己维护相关patch,带来一定的维护工作量;二是卸载工作无法被其他组件继承,后续组件卸载后仍需要进行代码逻辑分析和拆分等工作。为解决上述问题,本方案提出DPU的无感卸载,通过OS提供的抽象层,屏蔽应用在主机和DPU间跨主机访问的差异,让业务进程近似0改动达成卸载到DPU运行的目标,且这部分工作属于操作系统通用层,与上层业务无关,其他业务进行DPU卸载时也可以继承。 ++ ++## 架构介绍 ++ ++#### 容器管理面DPU无感卸载架构 ++ ++**图1**容器管理面DPU无感卸载架构 ++ ++![offload-arch](./figures/offload-arch.png) ++ ++如图1所示,容器管理面卸载后,dockerd、kubelet等管理进程运行在DPU侧,容器进程本身运行在HOST,进程之间的交互关系由系统层提供对应的能力来保证: ++ ++* 通信层:DPU和主机之间可能通过PCIe或网络进行通信,需要基于底层物理连接提供通信接口层,为上层业务提供通信接口。 ++ ++* 内核共享文件系统qtfs:容器管理面组件kubelet、dockerd与容器进程之间的主要交互通过文件系统进行;管理面工具需要为容器进程准备rootfs、volume等数据面路径;还需要在运行时通过proc文件系统、cgroup文件系统等控制和监控容器进程的资源及状态。共享文件系统的详细介绍参考[共享文件系统介绍](qtfs共享文件系统架构及使用手册.md) ++ ++* 用户态卸载环境:用户态需要使用qtfs为容器管理面准备卸载后的运行时环境,将主机的容器管理及运行时相关目录远程挂载到DPU;另外由于需要挂载proc、sys、cgroup等系统管理文件系统,为防止对DPU原生系统功能的破坏,上述挂载动作都在chroot环境内完成。另外管理面(运行于DPU)和容器进程(运行于主机)之间仍存在调用关系,需要通过远程二进制执行工具(rexec)提供对应功能。 ++ ++容器管理面无感卸载的操作步骤可参考[部署指导文档](./无感卸载部署指导.md) ++ ++> ![](./public_sys-resources/icon-note.gif)**说明**: ++> ++> 上述操作指导涉及对容器管理面组件的少量改动和rexec工具修改,这些修改基于指定版本,其他版本可基于实际执行环境做适配修改。文档中提供的patch仅供验证指导使用,不具备实际商用的条件 +\ No newline at end of file +diff --git "a/qtfs/doc/\346\227\240\346\204\237\345\215\270\350\275\275\351\203\250\347\275\262\346\214\207\345\257\274.md" "b/qtfs/doc/\346\227\240\346\204\237\345\215\270\350\275\275\351\203\250\347\275\262\346\214\207\345\257\274.md" +new file mode 100644 +index 0000000..c15eed9 +--- /dev/null ++++ "b/qtfs/doc/\346\227\240\346\204\237\345\215\270\350\275\275\351\203\250\347\275\262\346\214\207\345\257\274.md" +@@ -0,0 +1,166 @@ ++ ++# 容器管理面无感卸载部署指导 ++ ++> ![](./public_sys-resources/icon-note.gif)**说明**: ++> ++> 本指导涉及对容器管理面组件的少量改动和rexec工具修改,这些修改基于指定版本,其他版本可基于实际执行环境做适配修改。文档中提供的patch仅供验证指导使用,不具备实际商用的条件。 ++ ++> ![](./public_sys-resources/icon-note.gif)**说明**: ++> ++> 当前共享文件系统之间通信通过网络完成,可通过网络互连的两台物理机器或VM模拟验证。 ++> ++> 建议用户验证前先搭建可正常使用的kubernetes集群和容器运行环境,针对其中单个节点的管理面进程进行卸载验证,卸载环境(DPU)可选择一台具备网络连接的物理机或VM。 ++ ++## 简介 ++ ++容器管理面,即kubernetes、dockerd、containerd、isulad等容器的管理工具,而容器管理面卸载,即是将容器管理面卸载到与容器所在机器(以下称为HOST)之外的另一台机器(当前场景下是指DPU,一个具备独立运行环境的硬件集合)上运行。 ++ ++我们使用共享文件系统qtfs将HOST上与容器运行相关的目录挂载到DPU上,使得容器管理面工具(运行在DPU)可以访问到这些目录,并为容器(运行在HOST)准备运行所需要的环境,此处,因为需要挂载远端的proc和sys等特殊文件系统,所以,我们创建了一个专门的rootfs以作为kubernetes、dockerd的运行环境(以下称为`/another_rootfs`)。 ++ ++并且通过rexec执行容器的拉起、删除等操作,使得可以将容器管理面和容器分离在不同的两台机器上,远程对容器进行管理。 ++ ++## 相关组件补丁介绍 ++ ++#### rexec介绍 ++ ++rexec是一个用go语言开发的远程执行工具,基于docker/libchan下的[rexec](https://github.com/docker/libchan/tree/master/examples/rexec)示例工具改造而成,实现远程调用远端二进制的功能,为方便使用在rexec中增加了环境变量传递和监控原进程退出等能力。 ++ ++rexec工具的具体使用方式为在服务器端用`CMD_NET_ADDR=tcp://0.0.0.0:<端口号> rexec_server`的方式拉起rexec服务进程,然后在客户端用`CMD_NET_ADDR=tcp://<服务端ip>:<端口号> rexec [要执行的指令] `的方式启动,便可以调用rexec_server执行需要执行的指令,并等待指令执行结果返回。 ++ ++#### dockerd相关改动介绍 ++ ++对dockerd的改动基于18.09版本。 ++ ++在containerd中,暂时注释掉了通过hook调用libnetwork-setkey的部分,此处不影响容器的拉起。并且,为了docker load的正常使用,注释掉了在mounter_linux.go 中mount函数中一处错误的返回。 ++ ++最后,因为在容器管理面的运行环境中,将`/proc`挂在了服务端的proc文件系统,而本地的proc文件系统则挂载在了`/local_proc`,所以,dockerd以及containerd中的对`/proc/self/xxx`或者`/proc/getpid()/xxx`或者相关的文件系统访问的部分,我们统统将`/proc`改为了`/local_proc`。 ++ ++#### containerd相关改动介绍 ++ ++对于containerd的改动基于containerd-1.2-rc.1版本。 ++ ++在获取mountinfo时,因为`/proc/self/mountinfo`只能获取到dockerd本身在本地的mountinfo,而无法获取到服务端的mountinfo,所以,将其改为了`/proc/1/mountinfo`,使其通过获取服务端1号进程mountinfo的方式得到服务端的mountinfo。 ++ ++在contaienrd-shim中,将与containerd通信的unix socket改为了用tcp通信,containerd通过`SHIM_HOST`环境变量获取containerd-shim所运行环境的ip,即服务端ip。用shim的哈希值计算出一个端口号,并以此作为通信的端口,来拉起containerd-shim. ++ ++并且,将原来的通过系统调用给contaienr-shim发信号的方式,改为了通过远程调用kill指令的方式向shim发信号,确保了docker杀死容器的行为可以正确的执行。 ++ ++#### kubernetes相关改动介绍 ++ ++kubelet暂不需要功能性改动,可能会遇到容器QoS管理器首次设置失败的错误,该错误不影响后续Pods拉起流程,暂时忽略该报错。 ++ ++## 容器管理面卸载操作指南 ++ ++在服务器端和客户端,都要拉起rexec_server。服务器端拉起rexec_server,主要是用于客户端创建容器时用rexec拉起containerd-shim,而客户端拉起rexec_server,则是为了执行containerd-shim对dockerd和containerd的调用。 ++ ++#### 服务器端 ++ ++创建容器管理面所需要的文件夹,然后插入qtfs_server.ko,并拉起engine进程。 ++ ++此外在服务器端,还需要创建rexec脚本/usr/bin/dockerd. ++ ++``` shell ++#!/bin/bash ++CMD_NET_ADDR=tcp://<客户端ip>: rexec /usr/bin/dockerd $* ++``` ++ ++#### 客户端 ++ ++需要准备一个rootfs,作为dockerd与containerd的运行环境,通过如下的脚本,将dockerd、containerd所需要的服务端目录挂载到客户端。并且,需要确保在以下脚本中被挂载的远程目录在服务端和客户端都存在。 ++ ++``` shell ++#!/bin/bash ++mkdir -p /another_rootfs/var/run/docker/containerd ++iptables -t nat -N DOCKER ++echo "---------insmod qtfs ko----------" ++insmod /YOUR/QTFS/PATH/qtfs.ko qtfs_server_ip=<服务端ip> qtfs_log_level=INFO ++ ++# chroot环境内的proc使用DPU的proc共享文件系统替换,需要将本机真实proc文件系统挂载到local_proc下使用 ++mount -t proc proc /another_rootfs/local_proc/ ++ ++# 将chroot内环境与外部环境bind,方便进行配置和运行 ++mount --bind /var/run/ /another_rootfs/var/run/ ++mount --bind /var/lib/ /another_rootfs/var/lib/ ++mount --bind /etc /another_rootfs/etc ++ ++mkdir -p /another_rootfs/var/lib/isulad ++ ++# 在chroot环境内创建并挂载dev、sys和cgroup文件系统 ++mount -t devtmpfs devtmpfs /another_rootfs/dev/ ++mount -t sysfs sysfs /another_rootfs/sys ++mkdir -p /another_rootfs/sys/fs/cgroup ++mount -t tmpfs tmpfs /another_rootfs/sys/fs/cgroup ++list="perf_event freezer files net_cls,net_prio hugetlb pids rdma cpu,cpuacct memory devices blkio cpuset" ++for i in $list ++do ++ echo $i ++ mkdir -p /another_rootfs/sys/fs/cgroup/$i ++ mount -t cgroup cgroup -o rw,nosuid,nodev,noexec,relatime,$i /another_rootfs/sys/fs/cgroup/$i ++done ++ ++## common system dir ++mount -t qtfs -o proc /proc /another_rootfs/proc ++echo "proc" ++mount -t qtfs /sys /another_rootfs/sys ++echo "cgroup" ++ ++# 挂载容器管理面所需要的共享目录 ++mount -t qtfs /var/lib/docker/containers /another_rootfs/var/lib/docker/containers ++mount -t qtfs /var/lib/docker/containerd /another_rootfs/var/lib/docker/containerd ++mount -t qtfs /var/lib/docker/overlay2 /another_rootfs/var/lib/docker/overlay2 ++mount -t qtfs /var/lib/docker/image /another_rootfs/var/lib/docker/image ++mount -t qtfs /var/lib/docker/tmp /another_rootfs/var/lib/docker/tmp ++mkdir -p /another_rootfs/run/containerd/io.containerd.runtime.v1.linux/ ++mount -t qtfs /run/containerd/io.containerd.runtime.v1.linux/ /another_rootfs/run/containerd/io.containerd.runtime.v1.linux/ ++mkdir -p /another_rootfs/var/run/docker/containerd ++mount -t qtfs /var/run/docker/containerd /another_rootfs/var/run/docker/containerd ++mount -t qtfs /var/lib/kubelet/pods /another_rootfs/var/lib/kubelet/pods ++``` ++ ++在/another_rootfs中,需要创建以下脚本,用来支持部分跨主机操作。 ++ ++* /another_rootfs/usr/local/bin/containerd-shim ++ ++``` shell ++#!/bin/bash ++CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/bin/containerd-shim $* ++``` ++ ++* /another_rootfs/usr/local/bin/remote_kill ++ ++``` shell ++#!/bin/bash ++CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/bin/kill $* ++``` ++ ++* /another_rootfs/usr/sbin/modprobe ++``` shell ++#!/bin/bash ++CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/rexec /usr/sbin/modprobe $* ++``` ++ ++在chroot到dockerd和containerd运行所需的rootfs后,用如下的命令拉起dockerd和containerd ++ ++* containerd ++``` shell ++#!/bin/bash ++SHIM_HOST=<服务端ip> containerd --config /var/run/docker/containerd/containerd.toml --address /var/run/containerd/containerd.sock ++``` ++ ++* dockerd ++``` shell ++#!/bin/bash ++SHIM_HOST=<服务端ip> CMD_NET_ADDR=tcp://<服务端ip>: /usr/bin/dockerd --containerd /var/run/containerd/containerd.sock ++``` ++ ++* kubelet ++ ++在chroot环境内使用原参数拉起kubelet即可。 ++ ++因为我们已经将/var/run/和/another_rootfs/var/run/绑定在了一起,所以可以在正常的rootfs下,通过docker来访问docker.sock接口进行容器管理。 ++ ++至此,完成容器管理面卸载到DPU,可以通过docker相关操作进行容器创建、删除等操作,也可以通过kubectl在当前节点进行pods调度和销毁,且实际容器业务进程运行在HOST侧。 ++ ++> ![](./public_sys-resources/icon-note.gif)**说明**: ++> ++> 本指导所述操作只涉及容器管理面进程卸载,不包含容器网络和数据卷volume等卸载,如有相关需求,需要通过额外的网络或存储卸载能力支持。本指导支持不带网络和存储的容器跨节点拉起。 +\ No newline at end of file +-- +2.33.0 + diff --git a/0010-update-libvirt-doc.patch b/0010-update-libvirt-doc.patch new file mode 100644 index 0000000000000000000000000000000000000000..68f933b6529d6eda49ee4b121940eacd8825d9ba --- /dev/null +++ b/0010-update-libvirt-doc.patch @@ -0,0 +1,56 @@ +From ab96a3f0279f6befda7c66ea80956defe4b90f40 Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Fri, 9 Jun 2023 15:20:31 +0800 +Subject: update libvirt doc + +Signed-off-by: liqiang +--- + ...\351\233\266\346\220\255\345\273\272v1.1.md" | 17 ++++++++++++++--- + 1 file changed, 14 insertions(+), 3 deletions(-) + +diff --git "a/usecases/libvirtd-offload/libvirt\347\233\264\350\277\236\350\201\232\345\220\210\347\216\257\345\242\203\344\273\216\351\233\266\346\220\255\345\273\272v1.1.md" "b/usecases/libvirtd-offload/libvirt\347\233\264\350\277\236\350\201\232\345\220\210\347\216\257\345\242\203\344\273\216\351\233\266\346\220\255\345\273\272v1.1.md" +index c1f9c87..85541dc 100644 +--- "a/usecases/libvirtd-offload/libvirt\347\233\264\350\277\236\350\201\232\345\220\210\347\216\257\345\242\203\344\273\216\351\233\266\346\220\255\345\273\272v1.1.md" ++++ "b/usecases/libvirtd-offload/libvirt\347\233\264\350\277\236\350\201\232\345\220\210\347\216\257\345\242\203\344\273\216\351\233\266\346\220\255\345\273\272v1.1.md" +@@ -2,16 +2,27 @@ + | ---- | --------- | ---- | ------------------------------------------------------------ | + | V1.0 | 2022/12/5 | 李强 | 创建文档 | + | V1.1 | 2023/2/6 | 李强 | 增加uds proxy组件部署介绍;增加rexec组件部署介绍;修改libvirt相关描述,适配uds proxy组件。 | +-| V1.2 | 2023/3/22 | 李强 | rexec重构后,更新rexec组件部署说明。 | +-| V1.3 | 2023/3/27 | 李强 | qtfs为connect syscall适配uds做改动 | ++| V1.2 | 2023/3/22 | 李强 | rexec重构后,更新rexec组件部署说明。 | ++| V1.3 | 2023/3/27 | 李强 | qtfs为connect syscall适配uds做改动。 | ++| V1.4 | 2023/6/9 | 李强 | 将默认的组件链接方式改为vsock,网络改为测试模式。 | + + + # **1** 硬件准备 + ++## 测试模式 ++ + 需准备2台物理机(虚机当前未试过),网络互通。 + + 其中一台作为DPU模拟,另一台作为HOST模拟。在本文档中用DPU和HOST指代这两台服务器。 + ++注意:测试模式因为会暴露网络端口且不做连接认证,存在网络安全风险,仅能用于内部测试验证,不要用于实际生产环境。 ++ ++## vsock模式 ++ ++需要DPU加HOST,且DPU能支持通过virtio提供vsock通信方式。 ++ ++目前还未基于真实的支持DPU vsock的环境调试过,本文档当前仅描述基于测试模式的方法,下面的内容依然默认使用测试模式。 ++ + + # **2** libvirt卸载架构图 + +@@ -40,7 +51,7 @@ udsproxyd是一个跨主机的unix domain socket代理服务,需要分别部 + ```bash + cd qtfs/ipc + +-make && make install ++make -j UDS_TEST_MODE=1 && make install + ``` + + 当前最新版本下,qtfs server侧的engine服务已经整合了udsproxyd的能力,所以server侧若部署了qtfs后不需要再额外启动udsproxyd。client侧则单独拉起udsproxyd服务: +-- +2.33.0 + diff --git a/0011-ioctl-enhance-and-rewrite-duplicate-code.patch b/0011-ioctl-enhance-and-rewrite-duplicate-code.patch new file mode 100644 index 0000000000000000000000000000000000000000..c975b0cf184fe80507ef699abd7cea1004c6b317 --- /dev/null +++ b/0011-ioctl-enhance-and-rewrite-duplicate-code.patch @@ -0,0 +1,286 @@ +From bf30c2c3dc20b2b5e9562313a4c50541ebe313de Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Sat, 10 Jun 2023 11:30:37 +0800 +Subject: ioctl enhance and rewrite duplicate code + +Signed-off-by: liqiang +--- + qtfs/include/req.h | 2 + + qtfs/qtfs/sb.c | 43 +++++++++----- + qtfs/qtfs_server/fsops.c | 122 ++++++++++++++++----------------------- + 3 files changed, 81 insertions(+), 86 deletions(-) + +diff --git a/qtfs/include/req.h b/qtfs/include/req.h +index 7e0a4b2..407faef 100644 +--- a/qtfs/include/req.h ++++ b/qtfs/include/req.h +@@ -135,6 +135,8 @@ struct qtreq_ioctl { + unsigned int cmd; + unsigned int size; + int fd; ++ int argtype; // 0--use pointer arg, 1--use long arg ++ unsigned long arg; // for long type arg + } d; + + char path[QTFS_TAIL_LEN(struct qtreq_ioctl_len)]; +diff --git a/qtfs/qtfs/sb.c b/qtfs/qtfs/sb.c +index 2038c55..9c03219 100644 +--- a/qtfs/qtfs/sb.c ++++ b/qtfs/qtfs/sb.c +@@ -20,7 +20,7 @@ + #include + #include + #include +- ++#include + + #include "conn.h" + #include "qtfs-mod.h" +@@ -561,7 +561,7 @@ int qtfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) + return 0; + } + +-long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsigned int size) ++long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsigned int size, int argtype) + { + struct qtfs_conn_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_ioctl *req; +@@ -575,19 +575,22 @@ long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsig + return -EINVAL; + } + +- if (size >= MAX_PATH_LEN) { +- WARN_ON(1); ++ req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); ++ rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); ++ if (size >= sizeof(req->path)) { ++ qtfs_err("do ioctl failed, size:%u too big:%u", size, sizeof(req->path)); + qtfs_conn_put_param(pvar); + return -EINVAL; + } +- req = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_SEND); +- rsp = pvar->conn_ops->get_conn_msg_buf(pvar, QTFS_RECV); + + priv = (struct private_data *)filp->private_data; + req->d.fd = priv->fd; +- ++ req->d.argtype = argtype; + req->d.cmd = cmd; +- if (size > 0) { ++ if (argtype) { ++ req->d.arg = arg; ++ len = sizeof(struct qtreq_ioctl) - sizeof(req->path); ++ } else if (size > 0) { + ret = copy_from_user(req->path, (char __user *)arg, size); + if (ret) { + qtfs_err("%s: copy_from_user, size %u failed.", __func__, size); +@@ -597,7 +600,7 @@ long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsig + len = sizeof(struct qtreq_ioctl) - sizeof(req->path) + size; + req->d.size = size; + } else { +- len = sizeof(struct qtreq_ioctl) - sizeof(req->path) + strlen(req->path) + 1; ++ len = sizeof(struct qtreq_ioctl) - sizeof(req->path); + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_IOCTL, len); +@@ -623,27 +626,41 @@ out: + return (long)ret; + } + ++#define QTFS_IOCTL_CASE_WITH_BREAK(size, argtype)\ ++ {\ ++ ret = qtfs_do_ioctl(filp, cmd, arg, size, argtype);\ ++ break;\ ++ } + long qtfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) + { ++ unsigned int size; ++ long ret; + switch(cmd) { ++ // all case of size 0 type 0 enter here + case FS_IOC_FSGETXATTR: + case TCGETS: +- return qtfs_do_ioctl(filp, cmd, arg, 0); ++ QTFS_IOCTL_CASE_WITH_BREAK(0, 0); ++ // all case of size 0 type 1 enter here ++ case TUNSETPERSIST: ++ QTFS_IOCTL_CASE_WITH_BREAK(0, 1); + case FS_IOC_FSSETXATTR: +- return qtfs_do_ioctl(filp, cmd, arg, sizeof(struct fsxattr)); ++ QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct fsxattr), 0); + case TCSETS: +- return qtfs_do_ioctl(filp, cmd, arg, sizeof(struct ktermios)); ++ QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ktermios), 0); ++ case TUNSETIFF: ++ QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ifreq), 0); + default: { + char *fullname = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!fullname) + return -ENOMEM; + memset(fullname, 0, MAX_PATH_LEN); + qtfs_fullname(fullname, filp->f_path.dentry, MAX_PATH_LEN); +- qtfs_err("qtfs ioctl get not support cmd:%d file:%s TCGETS:%d", cmd, fullname, TCGETS); ++ qtfs_err("qtfs ioctl get not support cmd:%d file:%s", cmd, fullname); + kfree(fullname); + return -EOPNOTSUPP; + } + } ++ return ret; + } + + loff_t qtfs_dir_file_llseek(struct file *file, loff_t offset, int whence) +diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c +index 0ae95f3..e8e5742 100644 +--- a/qtfs/qtfs_server/fsops.c ++++ b/qtfs/qtfs_server/fsops.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0)) + #include + #endif +@@ -105,8 +106,20 @@ static inline void qtfs_inode_info_fill(struct inode_info *ii, struct inode *ino + return; + } + ++#define QTFS_IOCTL_HANDLE_WITH_BREAK(rspsize)\ ++ {\ ++ ret = copy_from_user(rsp->buf, userp->userp, rspsize);\ ++ if (ret) {\ ++ qtfs_err("cmd:%d copy_from_user failed with:%d\n", req->d.cmd, ret);\ ++ rsp->errno = -EFAULT;\ ++ goto err;\ ++ }\ ++ rsp->size = rspsize;\ ++ break;\ ++ } + static int handle_ioctl(struct qtserver_arg *arg) + { ++ unsigned long ioctl_arg; + int ret; + int iret; + struct qtreq_ioctl *req = (struct qtreq_ioctl *)REQ(arg); +@@ -123,82 +136,45 @@ static int handle_ioctl(struct qtserver_arg *arg) + } + mutex_unlock(&fd_bitmap_lock); + +- switch (req->d.cmd) { +- case FS_IOC_FSGETXATTR: +- iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, (unsigned long)userp->userp); +- if (iret) { +- qtfs_err("fsgetxattr ioctl failed with %d\n", iret); +- rsp->errno = iret; +- goto err; +- } +- ret = copy_from_user(rsp->buf, userp->userp, sizeof(struct fsxattr)); +- if (ret) { +- qtfs_err("fsgetxattr copy_from_user failed with %d\n", ret); +- rsp->errno = -EFAULT; +- goto err; +- } +- rsp->size = sizeof(struct fsxattr); +- break; +- case FS_IOC_FSSETXATTR: +- if (req->d.size <= 0 || req->d.size > sizeof(req->path) || req->d.size >= userp->size) { +- rsp->errno = -EINVAL; +- goto err; +- } +- ret = copy_to_user(userp->userp, req->path, req->d.size); +- if (ret) { +- qtfs_err("fssetxattr copy_to_user failed with %d\n", ret); +- rsp->errno = -EFAULT; +- goto err; +- } +- iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, (unsigned long)userp->userp); +- if (iret) { +- qtfs_err("fssetxattr ioctl failed with %d\n", iret); +- rsp->errno = iret; +- goto err; +- } +- rsp->size = 0; +- break; +- case TCGETS: +- iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, (unsigned long)userp->userp); +- if (iret) { +- qtfs_err("ioctl TCGETS failed with %d\n", iret); +- rsp->errno = iret; +- goto err; +- } +- qtfs_info("ioctl TCGETS ret:%d", iret); +- +- ret = copy_from_user(rsp->buf, userp->userp, sizeof(struct ktermios)); +- if (ret) { +- qtfs_err("fsgetxattr copy_from_user failed with %d\n", ret); +- rsp->errno = -EFAULT; +- goto err; +- } +- rsp->size = sizeof(struct ktermios); +- break; +- case TCSETS: +- if (req->d.size <= 0 || req->d.size > sizeof(req->path)) { +- rsp->errno = -EINVAL; +- goto err; +- } +- ret = copy_to_user(userp->userp, req->path, req->d.size); +- if (ret) { +- qtfs_err("tcsets copy_to_user failed with %d\n", ret); +- rsp->errno = -EFAULT; +- goto err; +- } +- qtfs_info("tcsets size:%u sizeof ktermios:%lu", req->d.size, sizeof(struct ktermios)); +- iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, (unsigned long)userp->userp); +- if (iret) { +- qtfs_err("tcsets ioctl failed with %d\n", iret); +- rsp->errno = iret; +- goto err; ++ if (req->d.argtype) { ++ ioctl_arg = req->d.arg; ++ } else { ++ if (req->d.size) { ++ if (req->d.size <= 0 || req->d.size > sizeof(req->path) || req->d.size >= userp->size) { ++ rsp->errno = -EINVAL; ++ goto err; ++ } ++ ret = copy_to_user(userp->userp, req->path, req->d.size); ++ if (ret) { ++ qtfs_err("cmd:%d copy_to_user failed with:%d", req->d.cmd, ret); ++ rsp->errno = -EFAULT; ++ goto err; ++ } + } +- rsp->size = 0; +- break; +- default: +- rsp->errno = -EOPNOTSUPP; ++ ioctl_arg = (unsigned long)userp->userp; ++ } ++ iret = qtfs_syscall_ioctl(req->d.fd, req->d.cmd, ioctl_arg); ++ if (iret) { ++ qtfs_err("ioctl fd:%d cmd:%d failed with %d", req->d.fd, req->d.cmd, iret); ++ rsp->errno = iret; + goto err; + } ++ qtfs_info("ioctl fd:%d cmd:%d argtype:%d arg:%lx size:%u successed", req->d.fd, req->d.cmd, req->d.argtype, req->d.arg, req->d.size); ++ switch (req->d.cmd) { ++ case TUNSETPERSIST: ++ case TUNSETIFF: ++ case TCSETS: ++ case FS_IOC_FSSETXATTR: ++ rsp->size = 0; ++ break; ++ case FS_IOC_FSGETXATTR: ++ QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct fsxattr)); ++ case TCGETS: ++ QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct ktermios)); ++ default: ++ rsp->errno = -EOPNOTSUPP; ++ goto err; ++ } + rsp->ret = QTFS_OK; + rsp->errno = iret; + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf) + rsp->size; +-- +2.33.0 + diff --git a/0012-port-in-use.patch b/0012-port-in-use.patch new file mode 100644 index 0000000000000000000000000000000000000000..89df6477e88ad9e873e422d085482eb48eab90d9 --- /dev/null +++ b/0012-port-in-use.patch @@ -0,0 +1,127 @@ +From 890082069c4e43fd8d4ec6e96bf851d185195b7d Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Sat, 10 Jun 2023 17:02:53 +0800 +Subject: port in use + +Signed-off-by: liqiang +--- + qtfs/include/conn.h | 12 ++++++++++-- + qtfs/qtfs_common/conn.c | 10 +++++----- + qtfs/qtfs_common/user_engine.c | 5 ++++- + qtfs/qtfs_server/qtfs-server.c | 8 ++++++++ + 4 files changed, 27 insertions(+), 8 deletions(-) + +diff --git a/qtfs/include/conn.h b/qtfs/include/conn.h +index 3b8f1f4..afbb00f 100644 +--- a/qtfs/include/conn.h ++++ b/qtfs/include/conn.h +@@ -46,7 +46,16 @@ extern bool qtfs_epoll_mode; + extern struct qtsock_wl_stru qtsock_wl; + extern struct qtfs_pvar_ops_s qtfs_conn_sock_pvar_ops; + +-#define qtfs_conn_get_param(void) _qtfs_conn_get_param(__func__) ++struct qtfs_conn_var_s *_qtfs_conn_get_param(const char *); ++static inline struct qtfs_conn_var_s *__qtfs_conn_get_param(const char *who_using) ++{ ++ struct qtfs_conn_var_s *p = _qtfs_conn_get_param(who_using); ++ if (IS_ERR_OR_NULL(p)) ++ return NULL; ++ return p; ++} ++#define qtfs_conn_get_param(void) __qtfs_conn_get_param(__func__) ++#define qtfs_conn_get_param_errcode(void) _qtfs_conn_get_param(__func__) + + #define QTFS_CONN_SOCK_TYPE "socket" + #define QTFS_CONN_PCIE_TYPE "pcie" +@@ -172,7 +181,6 @@ void *qtfs_conn_msg_buf(struct qtfs_conn_var_s *pvar, int dir); + void qtfs_conn_param_init(void); + void qtfs_conn_param_fini(void); + +-struct qtfs_conn_var_s *_qtfs_conn_get_param(const char *); + void qtfs_conn_put_param(struct qtfs_conn_var_s *pvar); + struct qtfs_conn_var_s *qtfs_epoll_establish_conn(void); + void qtfs_epoll_cut_conn(struct qtfs_conn_var_s *pvar); +diff --git a/qtfs/qtfs_common/conn.c b/qtfs/qtfs_common/conn.c +index 56b2b12..ab84181 100644 +--- a/qtfs/qtfs_common/conn.c ++++ b/qtfs/qtfs_common/conn.c +@@ -580,7 +580,7 @@ retry: + } + if (ret != 0) { + qtfs_conn_put_param(pvar); +- return NULL; ++ return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + strlcpy(pvar->who_using, func, QTFS_FUNCTION_LEN); + return pvar; +@@ -636,18 +636,18 @@ retry: + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + qtfs_conn_put_param(pvar); +- return NULL; ++ return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + qtfs_thread_var[pvar->cur_threadidx] = pvar; + #else + pvar->cs = QTFS_CONN_SOCK_SERVER; + if (!pvar->conn_ops->conn_inited(pvar)) { +- if (qtfs_sm_active(pvar)) { ++ if ((ret = qtfs_sm_active(pvar)) != 0) { + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + mutex_unlock(&g_param_mutex); + qtfs_conn_put_param(pvar); +- return NULL; ++ return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + mutex_unlock(&g_param_mutex); + } else { +@@ -657,7 +657,7 @@ retry: + if (ret) { + qtfs_err("qtfs get param active connection failed, ret:%d curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + qtfs_conn_put_param(pvar); +- return NULL; ++ return (IS_ERR_VALUE((long)ret) ? ERR_PTR((long)ret) : NULL); + } + } + #endif +diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c +index 521ebe4..919da11 100644 +--- a/qtfs/qtfs_common/user_engine.c ++++ b/qtfs/qtfs_common/user_engine.c +@@ -105,9 +105,12 @@ static struct qtfs_server_userp_s *qtfs_engine_thread_init(int fd, int thread_nu + } + } + struct qtfs_thread_init_s init_userp; ++ int ret; + init_userp.thread_nums = thread_nums; + init_userp.userp = userp; +- if (ioctl(fd, QTFS_IOCTL_THREAD_INIT, (unsigned long)&init_userp) == QTERROR) { ++ ret = ioctl(fd, QTFS_IOCTL_THREAD_INIT, (unsigned long)&init_userp); ++ if (ret != QTOK) { ++ engine_err("Engine thread init failed reason:%s", (ret == EADDRINUSE) ? strerror(EADDRINUSE) : "userp init failed."); + goto rollback; + } + return userp; +diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c +index 003c361..729097b 100644 +--- a/qtfs/qtfs_server/qtfs-server.c ++++ b/qtfs/qtfs_server/qtfs-server.c +@@ -192,6 +192,14 @@ long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long a + struct qtfs_thread_init_s init_userp; + switch (cmd) { + case QTFS_IOCTL_THREAD_INIT: ++ pvar = qtfs_conn_get_param_errcode(); ++ if (IS_ERR_OR_NULL(pvar)) { ++ qtfs_err("init pvar get failed, pvar:%ld", (long)pvar); ++ if (PTR_ERR(pvar) == -EADDRINUSE) ++ return EADDRINUSE; ++ } else { ++ qtfs_conn_put_param(pvar); ++ } + if (!write_trylock(&g_userp_rwlock)) { + qtfs_err("try lock userps failed."); + return QTERROR; +-- +2.33.0 + diff --git a/0013-ioctl-add-cmd.patch b/0013-ioctl-add-cmd.patch new file mode 100644 index 0000000000000000000000000000000000000000..f87dfd0a865e4c42b61648e8073962ebf55e19a7 --- /dev/null +++ b/0013-ioctl-add-cmd.patch @@ -0,0 +1,39 @@ +From 31262c9f982d59bf7f407f2f698933515dc4c8c7 Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Mon, 12 Jun 2023 11:28:27 +0800 +Subject: ioctl add cmd + +Signed-off-by: liqiang +--- + qtfs/qtfs/sb.c | 1 + + qtfs/qtfs_server/fsops.c | 2 ++ + 2 files changed, 3 insertions(+) + +diff --git a/qtfs/qtfs/sb.c b/qtfs/qtfs/sb.c +index 9c03219..f557e6d 100644 +--- a/qtfs/qtfs/sb.c ++++ b/qtfs/qtfs/sb.c +@@ -648,6 +648,7 @@ long qtfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) + case TCSETS: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ktermios), 0); + case TUNSETIFF: ++ case SIOCGIFHWADDR: + QTFS_IOCTL_CASE_WITH_BREAK(sizeof(struct ifreq), 0); + default: { + char *fullname = kmalloc(MAX_PATH_LEN, GFP_KERNEL); +diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c +index e8e5742..727c4e1 100644 +--- a/qtfs/qtfs_server/fsops.c ++++ b/qtfs/qtfs_server/fsops.c +@@ -171,6 +171,8 @@ static int handle_ioctl(struct qtserver_arg *arg) + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct fsxattr)); + case TCGETS: + QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct ktermios)); ++ case SIOCGIFHWADDR: ++ QTFS_IOCTL_HANDLE_WITH_BREAK(sizeof(struct ifreq)); + default: + rsp->errno = -EOPNOTSUPP; + goto err; +-- +2.33.0 + diff --git a/0014-engine-running-lock.patch b/0014-engine-running-lock.patch new file mode 100644 index 0000000000000000000000000000000000000000..2b6abad53fd5c1cbaa0355b55e440159652d0506 --- /dev/null +++ b/0014-engine-running-lock.patch @@ -0,0 +1,54 @@ +From a79ba9896c37d30cd1901f7f54e0507adb9aabda Mon Sep 17 00:00:00 2001 +From: liqiang +Date: Mon, 12 Jun 2023 11:34:09 +0800 +Subject: engine running lock + +Signed-off-by: liqiang +--- + qtfs/qtfs_common/user_engine.c | 16 +++++++++++++++- + 1 file changed, 15 insertions(+), 1 deletion(-) + +diff --git a/qtfs/qtfs_common/user_engine.c b/qtfs/qtfs_common/user_engine.c +index 919da11..7455413 100644 +--- a/qtfs/qtfs_common/user_engine.c ++++ b/qtfs/qtfs_common/user_engine.c +@@ -33,7 +33,7 @@ + #include + #include + #include +- ++#include + #include + + #include "comm.h" +@@ -68,6 +68,16 @@ struct engine_arg { + + #define QTFS_USERP_SIZE QTFS_USERP_MAXSIZE + #define QTFS_SERVER_FILE "/dev/qtfs_server" ++#define ENGINE_LOCK_ADDR "/var/run/qtfs/engine.lock" ++ ++int engine_socket_lock(void) ++{ ++ int lock_fd = open(ENGINE_LOCK_ADDR, O_RDONLY | O_CREAT, 0600); ++ if (lock_fd == -1) ++ return -EINVAL; ++ ++ return flock(lock_fd, LOCK_EX | LOCK_NB); ++} + + int qtfs_fd; + int engine_run = 1; +@@ -365,6 +375,10 @@ int main(int argc, char *argv[]) + engine_out(" Example: %s 16 1 192.168.10.10 12121 192.168.10.11 12121.", argv[0]); + return -1; + } ++ if (engine_socket_lock() < 0) { ++ engine_err("Engine is running."); ++ return -1; ++ } + if (qtfs_engine_env_check(argv) < 0) { + engine_err("Environment check failed, engine exit."); + return -1; +-- +2.33.0 + diff --git a/dpu-utilities.spec b/dpu-utilities.spec index 099d511b6b13b8393103610640b5a5d483ca1253..e9cc6b8e93a80f49c54d82b05a7751abd001d932 100644 --- a/dpu-utilities.spec +++ b/dpu-utilities.spec @@ -1,7 +1,7 @@ Name: dpu-utilities Summary: openEuler dpu utilities Version: 1.4 -Release: 1 +Release: 2 License: GPL-2.0 Source: https://gitee.com/openeuler/dpu-utilities/repository/archive/v%{version}.tar.gz ExclusiveOS: linux @@ -12,6 +12,21 @@ Provides: %{name} = %{version}-%{release} %define kernel_version %(ver=`rpm -qa|grep kernel-devel`;echo ${ver#*kernel-devel-}) BuildRequires: kernel-devel >= 5.10, gcc, make, json-c-devel, glib2-devel +Patch1: 0001-rewrite-client-rexec_run.patch +Patch2: 0002-uds-add-vsock-mode.patch +Patch3: 0003-fix-a-rexec_server-bug-on-child-process.patch +Patch4: 0004-add-uds-test-mode-in-engine-compile.patch +Patch5: 0005-modify-qtfs_dentry_revalidate.patch +Patch6: 0006-add-cmake-compile-for-user-binary-and-so.patch +Patch7: 0007-uds-remove-MSG_WAITALL-flags-to-timeout.patch +Patch8: 0008-Modify-the-logic-of-rexec-proxy-process-exit-by-mist.patch +Patch9: 0009-Keep-doc-same-with-docs-in-openEuler.patch +Patch10: 0010-update-libvirt-doc.patch +Patch11: 0011-ioctl-enhance-and-rewrite-duplicate-code.patch +Patch12: 0012-port-in-use.patch +Patch13: 0013-ioctl-add-cmd.patch +Patch14: 0014-engine-running-lock.patch + %description This package contains the software utilities on dpu. @@ -131,6 +146,9 @@ sed -i '/# product cut_conf/a\dpuos kiwi/minios/cfg_dpuos yes' /opt/imageT sed -i '//a\dpuos 1 rpm-dir euler_base' /opt/imageTailor/repos/RepositoryRule.conf %changelog +* Mon Jun 12 2023 Weifeng Su 1.4-2 +- Sync patches from source + * Fri Jun 2 2023 Weifeng Su 1.4-1 - Upgrade dpu-utilities version to 1.4