From 9917a934d148dc4cb32e0fcfefedd2a58ad4dffd Mon Sep 17 00:00:00 2001 From: Zhihao Cheng Date: Fri, 3 Sep 2021 20:35:57 +0800 Subject: [PATCH] Add io_uring support --- ...configure-add-io_uring-configuration.patch | 69 ++ 0002-event-add-io_uring-module.patch | 651 ++++++++++++++++ ...-add_event_with_-iovec-buf-interface.patch | 695 ++++++++++++++++++ 0004-io_uring-add-io_uring-read-support.patch | 242 ++++++ ...d-iovs-to-structure-ngx_connection_s.patch | 95 +++ ...io_uring-add-io_uring-writev-support.patch | 427 +++++++++++ 0007-io_uring-add-keepalive-support.patch | 147 ++++ 0008-io_uring-add-register-file-support.patch | 277 +++++++ ...conf-add-io_uring-configure-for-test.patch | 141 ++++ 0010-config-set.patch | 72 ++ nginx.spec | 10 + 11 files changed, 2826 insertions(+) create mode 100644 0001-configure-add-io_uring-configuration.patch create mode 100644 0002-event-add-io_uring-module.patch create mode 100644 0003-event-add-new-add_event_with_-iovec-buf-interface.patch create mode 100644 0004-io_uring-add-io_uring-read-support.patch create mode 100644 0005-add-iovs-to-structure-ngx_connection_s.patch create mode 100644 0006-io_uring-add-io_uring-writev-support.patch create mode 100644 0007-io_uring-add-keepalive-support.patch create mode 100644 0008-io_uring-add-register-file-support.patch create mode 100644 0009-conf-add-io_uring-configure-for-test.patch create mode 100644 0010-config-set.patch diff --git a/0001-configure-add-io_uring-configuration.patch b/0001-configure-add-io_uring-configuration.patch new file mode 100644 index 0000000..0be8f29 --- /dev/null +++ b/0001-configure-add-io_uring-configuration.patch @@ -0,0 +1,69 @@ +From 1c67180155f1abd184575a3b12fa93fbc4614977 Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Fri, 21 Aug 2020 12:38:03 +0800 +Subject: [PATCH 01/10] configure: add io_uring configuration + +Signed-off-by: Jiufei Xue +--- + auto/options | 4 ++++ + auto/os/linux | 22 ++++++++++++++++++++++ + 2 files changed, 26 insertions(+) + +diff --git a/auto/options b/auto/options +index 521c9768d8b3..01a96842fbc4 100644 +--- a/auto/options ++++ b/auto/options +@@ -45,6 +45,8 @@ USE_THREADS=NO + + NGX_FILE_AIO=NO + ++NGX_IO_URING=NO ++ + HTTP=YES + + NGX_HTTP_LOG_PATH= +@@ -207,6 +209,8 @@ do + + --with-file-aio) NGX_FILE_AIO=YES ;; + ++ --with-io-uring) NGX_IO_URING=YES ;; ++ + --with-ipv6) + NGX_POST_CONF_MSG="$NGX_POST_CONF_MSG + $0: warning: the \"--with-ipv6\" option is deprecated" +diff --git a/auto/os/linux b/auto/os/linux +index 5e280eca75db..00da62ec579a 100644 +--- a/auto/os/linux ++++ b/auto/os/linux +@@ -88,6 +88,28 @@ if [ $ngx_found = yes ]; then + . auto/feature + fi + ++if [ $NGX_IO_URING = YES ]; then ++ ++ ngx_feature="Linux io_uring support" ++ ngx_feature_name="NGX_HAVE_IO_URING" ++ ngx_feature_incs="#include " ++ ngx_feature_path= ++ ngx_feature_libs="-luring" ++ ngx_feature_test="struct io_uring ring; ++ struct io_uring_params params; ++ int ret; ++ memset(¶ms, 0, sizeof(params)); ++ ret = io_uring_queue_init_params(64, &ring, ¶ms); ++ if (ret < 0) return 1; ++ if (!(params.features & IORING_FEAT_FAST_POLL)) return 1; ++ io_uring_queue_exit(&ring)" ++ . auto/feature ++ ++ if [ $ngx_found = yes ]; then ++ have=NGX_HAVE_EVENTFD . auto/have ++ CORE_LIBS="$CORE_LIBS -luring" ++ fi ++fi + + # O_PATH and AT_EMPTY_PATH were introduced in 2.6.39, glibc 2.14 + +-- +2.31.1 + diff --git a/0002-event-add-io_uring-module.patch b/0002-event-add-io_uring-module.patch new file mode 100644 index 0000000..7a1a3d6 --- /dev/null +++ b/0002-event-add-io_uring-module.patch @@ -0,0 +1,651 @@ +From 648b5e0f7941118e04cc6a683bbb65225f5876b0 Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Fri, 21 Aug 2020 12:42:08 +0800 +Subject: [PATCH 02/10] event: add io_uring module + +Signed-off-by: Jiufei Xue +--- + auto/os/linux | 2 + + auto/sources | 3 + + src/event/modules/ngx_io_uring_module.c | 483 ++++++++++++++++++++++++ + src/event/ngx_event.c | 27 +- + src/event/ngx_event.h | 1 - + src/event/ngx_event_accept.c | 19 + + src/os/unix/ngx_linux_config.h | 3 + + 7 files changed, 527 insertions(+), 11 deletions(-) + create mode 100644 src/event/modules/ngx_io_uring_module.c + +diff --git a/auto/os/linux b/auto/os/linux +index 00da62ec579a..bf75a6dd7ab7 100644 +--- a/auto/os/linux ++++ b/auto/os/linux +@@ -108,6 +108,8 @@ if [ $NGX_IO_URING = YES ]; then + if [ $ngx_found = yes ]; then + have=NGX_HAVE_EVENTFD . auto/have + CORE_LIBS="$CORE_LIBS -luring" ++ CORE_SRCS="$CORE_SRCS $IO_URING_SRCS" ++ EVENT_MODULES="$EVENT_MODULES $IO_URING_MODULE" + fi + fi + +diff --git a/auto/sources b/auto/sources +index 3dad11132c48..59ec3d720b0f 100644 +--- a/auto/sources ++++ b/auto/sources +@@ -120,6 +120,9 @@ EVENTPORT_SRCS=src/event/modules/ngx_eventport_module.c + EPOLL_MODULE=ngx_epoll_module + EPOLL_SRCS=src/event/modules/ngx_epoll_module.c + ++IO_URING_MODULE=ngx_io_uring_module ++IO_URING_SRCS=src/event/modules/ngx_io_uring_module.c ++ + IOCP_MODULE=ngx_iocp_module + IOCP_SRCS=src/event/modules/ngx_iocp_module.c + +diff --git a/src/event/modules/ngx_io_uring_module.c b/src/event/modules/ngx_io_uring_module.c +new file mode 100644 +index 000000000000..eaa222aa6f60 +--- /dev/null ++++ b/src/event/modules/ngx_io_uring_module.c +@@ -0,0 +1,483 @@ ++ ++/* ++ * Copyright (C) Igor Sysoev ++ * Copyright (C) Nginx, Inc. ++ */ ++ ++ ++#include ++#include ++#include ++ ++typedef struct { ++ ngx_uint_t entries; ++} ngx_io_uring_conf_t; ++ ++ ++#if (NGX_HAVE_EPOLLRDHUP) ++static void ngx_io_uring_epoll_test_rdhup(ngx_cycle_t *cycle); ++#endif ++ ++static ngx_int_t ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer); ++static void ngx_io_uring_done(ngx_cycle_t *cycle); ++static ngx_int_t ngx_io_uring_add_event(ngx_event_t *ev, ++ ngx_int_t event, ++ ngx_uint_t flags); ++static ngx_int_t ngx_io_uring_del_event(ngx_event_t *ev, ngx_int_t event, ++ ngx_uint_t flags); ++static ngx_int_t ngx_io_uring_add_connection(ngx_connection_t *c); ++static ngx_int_t ngx_io_uring_del_connection(ngx_connection_t *c, ++ ngx_uint_t flags); ++static ngx_int_t ngx_io_uring_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ++ ngx_uint_t flags); ++ ++static void *ngx_io_uring_create_conf(ngx_cycle_t *cycle); ++static char *ngx_io_uring_init_conf(ngx_cycle_t *cycle, void *conf); ++ ++static struct io_uring_cqe **cqes; ++static ngx_uint_t nevents; ++ ++struct io_uring ngx_ring; ++ ++static ngx_str_t io_uring_name = ngx_string("io_uring"); ++ ++static ngx_command_t ngx_io_uring_commands[] = { ++ ++ { ngx_string("io_uring_entries"), ++ NGX_EVENT_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_num_slot, ++ 0, ++ offsetof(ngx_io_uring_conf_t, entries), ++ NULL }, ++ ++ ngx_null_command ++}; ++ ++static ngx_event_module_t ngx_io_uring_module_ctx = { ++ &io_uring_name, ++ ngx_io_uring_create_conf, /* create configuration */ ++ ngx_io_uring_init_conf, /* init configuration */ ++ ++ { ++ ngx_io_uring_add_event, /* add an event */ ++ ngx_io_uring_del_event, /* delete an event */ ++ ngx_io_uring_add_event, /* enable an event */ ++ ngx_io_uring_del_event, /* disable an event */ ++ ngx_io_uring_add_connection, /* add an connection */ ++ ngx_io_uring_del_connection, /* delete an connection */ ++ NULL, /* trigger a notify */ ++ ngx_io_uring_process_events, /* process the events */ ++ ngx_io_uring_init, /* init the events */ ++ ngx_io_uring_done, /* done the events */ ++ } ++}; ++ ++ngx_module_t ngx_io_uring_module = { ++ NGX_MODULE_V1, ++ &ngx_io_uring_module_ctx, /* module context */ ++ ngx_io_uring_commands, /* module directives */ ++ NGX_EVENT_MODULE, /* module type */ ++ NULL, /* init master */ ++ NULL, /* init module */ ++ NULL, /* init process */ ++ NULL, /* init thread */ ++ NULL, /* exit thread */ ++ NULL, /* exit process */ ++ NULL, /* exit master */ ++ NGX_MODULE_V1_PADDING ++}; ++ ++#if (NGX_HAVE_EPOLLRDHUP) ++static void ngx_io_uring_epoll_test_rdhup(ngx_cycle_t *cycle) ++{ ++ int s[2], events, ret; ++ struct io_uring_cqe *cqe; ++ struct io_uring_sqe *sqe = io_uring_get_sqe(&ngx_ring); ++ ++ if (!sqe) ngx_abort(); ++ ++ if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "socketpair() failed"); ++ return; ++ } ++ ++ events = EPOLLET|EPOLLIN|EPOLLRDHUP; ++ io_uring_prep_poll_add(sqe, s[0], events); ++ ret = io_uring_submit(&ngx_ring); ++ if (ret < 0) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "io_uring_submit() failed"); ++ goto failed; ++ } ++ ++ if (close(s[1]) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "close() failed"); ++ s[1] = -1; ++ goto failed; ++ } ++ ++ s[1] = -1; ++ ++ ret = io_uring_wait_cqe(&ngx_ring, &cqe); ++ if (ret < 0) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "io_uring_wait_cqe() failed"); ++ goto failed; ++ } ++ ++ ngx_use_epoll_rdhup = cqe->res & EPOLLRDHUP; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "testing the EPOLLRDHUP flag: %s", ++ ngx_use_epoll_rdhup ? "success" : "fail"); ++ ++failed: ++ ++ if (s[1] != -1 && close(s[1]) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "close() failed"); ++ } ++ ++ if (close(s[0]) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ++ "close() failed"); ++ } ++} ++#endif ++ ++static ngx_int_t ++ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) ++{ ++ ngx_io_uring_conf_t *urcf; ++ ++ urcf = ngx_event_get_conf(cycle->conf_ctx, ngx_io_uring_module); ++ ++ if (ngx_ring.ring_fd == 0) { ++ if (io_uring_queue_init(urcf->entries, &ngx_ring, 0) < 0) { ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, ++ "io_uring_queue_init() failed"); ++ return NGX_ERROR; ++ } ++ } ++ ++#if (NGX_HAVE_EPOLLRDHUP) ++ ngx_io_uring_epoll_test_rdhup(cycle); ++#endif ++ ++ if (nevents < urcf->entries / 2) { ++ if (cqes) { ++ ngx_free(cqes); ++ } ++ ++ cqes = ngx_alloc(sizeof(struct io_uring_cqe *) * urcf->entries / 2, ++ cycle->log); ++ if (cqes == NULL) { ++ return NGX_ERROR; ++ } ++ } ++ ++ nevents = urcf->entries / 2; ++ ++ ngx_io = ngx_os_io; ++ ++ ngx_event_actions = ngx_io_uring_module_ctx.actions; ++ ++ ngx_event_flags = NGX_USE_CLEAR_EVENT ++ | NGX_USE_GREEDY_EVENT ++ | NGX_USE_EPOLL_EVENT; ++ ++ return NGX_OK; ++} ++ ++static void ++ngx_io_uring_done(ngx_cycle_t *cycle) ++{ ++ io_uring_queue_exit(&ngx_ring); ++ ngx_ring.ring_fd = 0; ++ ngx_free(cqes); ++} ++ ++static void ++io_uring_add_poll(struct io_uring *ring, int fd, ++ int event, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_poll_add(sqe, fd, event); ++ io_uring_sqe_set_data(sqe, data); ++} ++ ++static void ++io_uring_remove_poll(struct io_uring *ring, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_poll_remove(sqe, data); ++ io_uring_sqe_set_data(sqe, NULL); ++} ++ ++static ngx_int_t ++ngx_io_uring_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) ++{ ++ ngx_connection_t *c = ev->data; ++ ++ if (event == NGX_READ_EVENT) { ++#if (NGX_READ_EVENT != EPOLLIN|EPOLLRDHUP) ++ event = EPOLLIN|EPOLLRDHUP; ++#endif ++ } else { ++#if (NGX_WRITE_EVENT != EPOLLOUT) ++ event = EPOLLOUT; ++#endif ++ } ++ ++#if (NGX_HAVE_EPOLLEXCLUSIVE && NGX_HAVE_EPOLLRDHUP) ++ if (flags & NGX_EXCLUSIVE_EVENT) { ++ event &= ~EPOLLRDHUP; ++ } ++#endif ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_uring add event: fd:%d ev:%08XD, data:%p", ++ c->fd, event, (void *) ((uintptr_t) c | ev->instance)); ++ ++ io_uring_add_poll(&ngx_ring, c->fd, event | (uint32_t) flags, ++ (void *) ((uintptr_t) c | ev->instance)); ++ ev->active = 1; ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ++ngx_io_uring_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) ++{ ++ ngx_connection_t *c = ev->data; ++ ++ if (flags & NGX_CLOSE_EVENT) { ++ ev->active = 0; ++ return NGX_OK; ++ } ++ ++ io_uring_remove_poll(&ngx_ring, (void *) ((uintptr_t) c | ev->instance)); ++ ++ ev->active = 0; ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ++ngx_io_uring_add_connection(ngx_connection_t *c) ++{ ++ ngx_int_t event = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "io_uring add connection: fd:%d ev:%08XD", c->fd, event); ++ ++ io_uring_add_poll(&ngx_ring, c->fd, event, ++ (void *) ((uintptr_t) c | c->read->instance)); ++ c->read->active = 1; ++ c->write->active = 1; ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ++ngx_io_uring_del_connection(ngx_connection_t *c, ngx_uint_t flags) ++{ ++ /* ++ * when the file descriptor is closed the epoll automatically deletes ++ * it from its queue so we do not need to delete explicitly the event ++ * before the closing the file descriptor ++ */ ++ ++ if (flags & NGX_CLOSE_EVENT) { ++ c->read->active = 0; ++ c->write->active = 0; ++ return NGX_OK; ++ } ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "epoll del connection: fd:%d", c->fd); ++ ++ io_uring_remove_poll(&ngx_ring, (void *) ((uintptr_t) c | c->read->instance)); ++ ++ c->read->active = 0; ++ c->write->active = 0; ++ ++ return NGX_OK; ++} ++ ++static ngx_int_t ++ngx_io_uring_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) ++{ ++ uint32_t cqe_count, revents; ++ ngx_int_t instance, i; ++ ngx_uint_t level; ++ ngx_err_t err = NGX_OK; ++ ngx_event_t *rev, *wev; ++ ngx_queue_t *queue; ++ ngx_connection_t *c; ++ void *data; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring timer: %M", timer); ++ ++ if (io_uring_submit_and_wait(&ngx_ring, 1) < 0) ++ err = ngx_errno; ++ ++ if (flags & NGX_UPDATE_TIME || ngx_event_timer_alarm) { ++ ngx_time_update(); ++ } ++ ++ if (err && err != NGX_EBUSY) { ++ if (err == NGX_EINTR) { ++ ++ if (ngx_event_timer_alarm) { ++ ngx_event_timer_alarm = 0; ++ return NGX_OK; ++ } ++ ++ level = NGX_LOG_INFO; ++ ++ } else { ++ level = NGX_LOG_ALERT; ++ } ++ ++ ngx_log_error(level, cycle->log, err, "io_uring_submit_and_wait() failed"); ++ return NGX_ERROR; ++ } ++ ++ cqe_count = io_uring_peek_batch_cqe(&ngx_ring, cqes, nevents); ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "cqe count: %d", cqe_count); ++ ++ for (i = 0; i < cqe_count; i++) { ++ struct io_uring_cqe *cqe = cqes[i]; ++ ++ data = io_uring_cqe_get_data(cqe); ++ instance = (uintptr_t) data & 1; ++ c = (ngx_connection_t *) ((uintptr_t) data & (uintptr_t) ~1); ++ ++ rev = c->read; ++ ++ if (c->fd == -1 || rev->instance != instance) { ++ ++ /* ++ * the stale event from a file descriptor ++ * that was just closed in this iteration ++ */ ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring: stale event fd:%d d:%p instance:%d rev_instance:%d", ++ c, c->fd, instance, rev->instance); ++ io_uring_cqe_seen(&ngx_ring, cqe); ++ continue; ++ } ++ ++ revents = cqe->res; ++ io_uring_cqe_seen(&ngx_ring, cqe); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring_peek_cqe: fd:%d ev:%04XD d:%p listening:%p active:%d", ++ c->fd, revents, data, c->listening, rev->active); ++ ++ if (revents & (EPOLLERR|EPOLLHUP)) { ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring_peek_cqe error on fd:%d ev:%04XD", ++ c->fd, revents); ++ ++ /* ++ * if the error events were returned, add EPOLLIN and EPOLLOUT ++ * to handle the events at least in one active handler ++ */ ++ ++ revents |= EPOLLIN|EPOLLOUT; ++ } ++ ++ if ((revents & EPOLLIN) && rev->active) { ++ ++#if (NGX_HAVE_EPOLLRDHUP) ++ if (revents & EPOLLRDHUP) { ++ rev->pending_eof = 1; ++ } ++#endif ++ ++ rev->ready = 1; ++ rev->available = -1; ++ rev->active = 0; ++ ++ if (flags & NGX_POST_EVENTS) { ++ queue = rev->accept ? &ngx_posted_accept_events ++ : &ngx_posted_events; ++ ++ ngx_post_event(rev, queue); ++ ++ } else { ++ rev->handler(rev); ++ } ++ } ++ ++ wev = c->write; ++ ++ if ((revents & EPOLLOUT) && wev->active) { ++ ++ if (c->fd == -1 || wev->instance != instance) { ++ ++ /* ++ * the stale event from a file descriptor ++ * that was just closed in this iteration ++ */ ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "epoll: stale event %p", c); ++ continue; ++ } ++ ++ wev->ready = 1; ++ wev->active = 0; ++#if (NGX_THREADS) ++ wev->complete = 1; ++#endif ++ ++ if (flags & NGX_POST_EVENTS) { ++ ngx_post_event(wev, &ngx_posted_events); ++ ++ } else { ++ wev->handler(wev); ++ } ++ } ++ } ++ ++ return NGX_OK; ++} ++ ++static void * ++ngx_io_uring_create_conf(ngx_cycle_t *cycle) ++{ ++ ngx_io_uring_conf_t *urcf; ++ ++ urcf = ngx_palloc(cycle->pool, sizeof(ngx_io_uring_conf_t)); ++ if (urcf == NULL) { ++ return NULL; ++ } ++ ++ urcf->entries = NGX_CONF_UNSET; ++ ++ return urcf; ++} ++ ++static char * ++ngx_io_uring_init_conf(ngx_cycle_t *cycle, void *conf) ++{ ++ ngx_io_uring_conf_t *urcf = conf; ++ ++ ngx_conf_init_uint_value(urcf->entries, 512); ++ ++ return NGX_CONF_OK; ++} +diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c +index 402a7f5e28e4..9e9484e76f60 100644 +--- a/src/event/ngx_event.c ++++ b/src/event/ngx_event.c +@@ -18,6 +18,7 @@ extern ngx_module_t ngx_eventport_module; + extern ngx_module_t ngx_devpoll_module; + extern ngx_module_t ngx_epoll_module; + extern ngx_module_t ngx_select_module; ++extern ngx_module_t ngx_io_uring_module; + + + static char *ngx_event_init_conf(ngx_cycle_t *cycle, void *conf); +@@ -1262,30 +1263,36 @@ ngx_event_core_init_conf(ngx_cycle_t *cycle, void *conf) + + module = NULL; + ++#if (NGX_HAVE_IO_URING) ++ module = &ngx_io_uring_module; ++#endif ++ + #if (NGX_HAVE_EPOLL) && !(NGX_TEST_BUILD_EPOLL) + +- fd = epoll_create(100); ++ if (module == NULL) { ++ fd = epoll_create(100); + +- if (fd != -1) { +- (void) close(fd); +- module = &ngx_epoll_module; ++ if (fd != -1) { ++ (void) close(fd); ++ module = &ngx_epoll_module; + +- } else if (ngx_errno != NGX_ENOSYS) { +- module = &ngx_epoll_module; ++ } else if (ngx_errno != NGX_ENOSYS) { ++ module = &ngx_epoll_module; ++ } + } + + #endif + + #if (NGX_HAVE_DEVPOLL) && !(NGX_TEST_BUILD_DEVPOLL) + +- module = &ngx_devpoll_module; ++ if (module == NULL) ++ module = &ngx_devpoll_module; + + #endif + + #if (NGX_HAVE_KQUEUE) +- +- module = &ngx_kqueue_module; +- ++ if (module == NULL) ++ module = &ngx_kqueue_module; + #endif + + #if (NGX_HAVE_SELECT) +diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h +index 97f9673c90da..9d3c155bb511 100644 +--- a/src/event/ngx_event.h ++++ b/src/event/ngx_event.h +@@ -271,7 +271,6 @@ extern ngx_uint_t ngx_use_epoll_rdhup; + */ + #define NGX_USE_VNODE_EVENT 0x00002000 + +- + /* + * The event filter is deleted just before the closing file. + * Has no meaning for select and poll. +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index 4364240938fa..9a859c25cccd 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -312,6 +312,25 @@ ngx_event_accept(ngx_event_t *ev) + } + + } while (ev->available); ++ ++ /* re-add event poll for listening */ ++#if (HAVE_IO_URING) ++#if (NGX_HAVE_EPOLLEXCLUSIVE) ++ if (ngx_event_flags & NGX_USE_EPOLL_EVENT) { ++ if (ngx_add_event(ev, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT) ++ == NGX_ERROR) { ++ ngx_log_error(NGX_LOG_ALERT, ev->log, 0, ++ "add event failed"); ++ } ++ ++ } ++#else ++ if (ngx_add_event(ev, NGX_READ_EVENT, 0) == NGX_ERROR) { ++ ngx_log_error(NGX_LOG_ALERT, ev->log, 0, ++ "add event failed"); ++ } ++#endif ++#endif + } + + +diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h +index 3036caebf630..4bca66d86d06 100644 +--- a/src/os/unix/ngx_linux_config.h ++++ b/src/os/unix/ngx_linux_config.h +@@ -98,6 +98,9 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size); + typedef struct iocb ngx_aiocb_t; + #endif + ++#if (NGX_HAVE_IO_URING) ++#include ++#endif + + #if (NGX_HAVE_CAPABILITIES) + #include +-- +2.31.1 + diff --git a/0003-event-add-new-add_event_with_-iovec-buf-interface.patch b/0003-event-add-new-add_event_with_-iovec-buf-interface.patch new file mode 100644 index 0000000..bd33585 --- /dev/null +++ b/0003-event-add-new-add_event_with_-iovec-buf-interface.patch @@ -0,0 +1,695 @@ +From d6d8039c2b697885dae8142933ef76d565b06555 Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Wed, 19 Aug 2020 13:00:08 +0800 +Subject: [PATCH 03/10] event: add new add_event_with_[iovec|buf] interface + +Signed-off-by: Jiufei Xue +--- + src/event/modules/ngx_devpoll_module.c | 2 + + src/event/modules/ngx_epoll_module.c | 2 + + src/event/modules/ngx_eventport_module.c | 2 + + src/event/modules/ngx_io_uring_module.c | 331 +++++++++++++------- + src/event/modules/ngx_kqueue_module.c | 2 + + src/event/modules/ngx_poll_module.c | 2 + + src/event/modules/ngx_select_module.c | 2 + + src/event/modules/ngx_win32_poll_module.c | 2 + + src/event/modules/ngx_win32_select_module.c | 2 + + src/event/ngx_event.c | 41 ++- + src/event/ngx_event.h | 28 +- + src/event/ngx_event_accept.c | 2 +- + 12 files changed, 301 insertions(+), 117 deletions(-) + +diff --git a/src/event/modules/ngx_devpoll_module.c b/src/event/modules/ngx_devpoll_module.c +index 590eb28e1464..7c8338af78db 100644 +--- a/src/event/modules/ngx_devpoll_module.c ++++ b/src/event/modules/ngx_devpoll_module.c +@@ -85,6 +85,8 @@ static ngx_event_module_t ngx_devpoll_module_ctx = { + + { + ngx_devpoll_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_devpoll_del_event, /* delete an event */ + ngx_devpoll_add_event, /* enable an event */ + ngx_devpoll_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c +index 98e3ce7c826f..131ef9c8490c 100644 +--- a/src/event/modules/ngx_epoll_module.c ++++ b/src/event/modules/ngx_epoll_module.c +@@ -183,6 +183,8 @@ static ngx_event_module_t ngx_epoll_module_ctx = { + + { + ngx_epoll_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_epoll_del_event, /* delete an event */ + ngx_epoll_add_event, /* enable an event */ + ngx_epoll_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_eventport_module.c b/src/event/modules/ngx_eventport_module.c +index f67c70457f49..e9d66f96e662 100644 +--- a/src/event/modules/ngx_eventport_module.c ++++ b/src/event/modules/ngx_eventport_module.c +@@ -178,6 +178,8 @@ static ngx_event_module_t ngx_eventport_module_ctx = { + + { + ngx_eventport_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_eventport_del_event, /* delete an event */ + ngx_eventport_add_event, /* enable an event */ + ngx_eventport_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_io_uring_module.c b/src/event/modules/ngx_io_uring_module.c +index eaa222aa6f60..7a96b4188f4a 100644 +--- a/src/event/modules/ngx_io_uring_module.c ++++ b/src/event/modules/ngx_io_uring_module.c +@@ -14,15 +14,19 @@ typedef struct { + } ngx_io_uring_conf_t; + + +-#if (NGX_HAVE_EPOLLRDHUP) +-static void ngx_io_uring_epoll_test_rdhup(ngx_cycle_t *cycle); +-#endif +- + static ngx_int_t ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer); + static void ngx_io_uring_done(ngx_cycle_t *cycle); + static ngx_int_t ngx_io_uring_add_event(ngx_event_t *ev, + ngx_int_t event, + ngx_uint_t flags); ++static ngx_int_t ngx_io_uring_add_event_with_iovec(ngx_event_t *ev, ++ ngx_int_t event, ++ struct iovec *iov, ++ ngx_int_t count); ++static ngx_int_t ngx_io_uring_add_event_with_buf(ngx_event_t *ev, ++ ngx_int_t event, ++ u_char *buf, ++ size_t size); + static ngx_int_t ngx_io_uring_del_event(ngx_event_t *ev, ngx_int_t event, + ngx_uint_t flags); + static ngx_int_t ngx_io_uring_add_connection(ngx_connection_t *c); +@@ -60,6 +64,8 @@ static ngx_event_module_t ngx_io_uring_module_ctx = { + + { + ngx_io_uring_add_event, /* add an event */ ++ ngx_io_uring_add_event_with_iovec, ++ ngx_io_uring_add_event_with_buf, + ngx_io_uring_del_event, /* delete an event */ + ngx_io_uring_add_event, /* enable an event */ + ngx_io_uring_del_event, /* disable an event */ +@@ -87,66 +93,6 @@ ngx_module_t ngx_io_uring_module = { + NGX_MODULE_V1_PADDING + }; + +-#if (NGX_HAVE_EPOLLRDHUP) +-static void ngx_io_uring_epoll_test_rdhup(ngx_cycle_t *cycle) +-{ +- int s[2], events, ret; +- struct io_uring_cqe *cqe; +- struct io_uring_sqe *sqe = io_uring_get_sqe(&ngx_ring); +- +- if (!sqe) ngx_abort(); +- +- if (socketpair(AF_UNIX, SOCK_STREAM, 0, s) == -1) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "socketpair() failed"); +- return; +- } +- +- events = EPOLLET|EPOLLIN|EPOLLRDHUP; +- io_uring_prep_poll_add(sqe, s[0], events); +- ret = io_uring_submit(&ngx_ring); +- if (ret < 0) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "io_uring_submit() failed"); +- goto failed; +- } +- +- if (close(s[1]) == -1) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "close() failed"); +- s[1] = -1; +- goto failed; +- } +- +- s[1] = -1; +- +- ret = io_uring_wait_cqe(&ngx_ring, &cqe); +- if (ret < 0) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "io_uring_wait_cqe() failed"); +- goto failed; +- } +- +- ngx_use_epoll_rdhup = cqe->res & EPOLLRDHUP; +- +- ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, +- "testing the EPOLLRDHUP flag: %s", +- ngx_use_epoll_rdhup ? "success" : "fail"); +- +-failed: +- +- if (s[1] != -1 && close(s[1]) == -1) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "close() failed"); +- } +- +- if (close(s[0]) == -1) { +- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, +- "close() failed"); +- } +-} +-#endif +- + static ngx_int_t + ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + { +@@ -162,10 +108,6 @@ ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + } + } + +-#if (NGX_HAVE_EPOLLRDHUP) +- ngx_io_uring_epoll_test_rdhup(cycle); +-#endif +- + if (nevents < urcf->entries / 2) { + if (cqes) { + ngx_free(cqes); +@@ -186,7 +128,8 @@ ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + + ngx_event_flags = NGX_USE_CLEAR_EVENT + | NGX_USE_GREEDY_EVENT +- | NGX_USE_EPOLL_EVENT; ++ | NGX_USE_EPOLL_EVENT ++ | NGX_USE_IO_URING_EVENT; + + return NGX_OK; + } +@@ -222,6 +165,109 @@ io_uring_remove_poll(struct io_uring *ring, void *data) + io_uring_sqe_set_data(sqe, NULL); + } + ++static void ++io_uring_add_recv(struct io_uring *ring, int fd, ++ u_char *buf, ++ size_t size, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_recv(sqe, fd, buf, size, 0); ++ io_uring_sqe_set_data(sqe, data); ++} ++ ++static void ++io_uring_add_send(struct io_uring *ring, int fd, ++ u_char *buf, ++ size_t size, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_send(sqe, fd, buf, size, 0); ++ io_uring_sqe_set_data(sqe, data); ++} ++ ++static void ++io_uring_add_readv(struct io_uring *ring, int fd, ++ struct iovec *iov, ++ ngx_int_t count, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_readv(sqe, fd, iov, count, 0); ++ io_uring_sqe_set_data(sqe, data); ++} ++ ++static void ++io_uring_add_writev(struct io_uring *ring, int fd, ++ struct iovec *iov, ++ ngx_int_t count, void *data) ++{ ++ struct io_uring_sqe *sqe = io_uring_get_sqe(ring); ++ ++ /* TODO: !sqe ? */ ++ if (!sqe) ngx_abort(); ++ io_uring_prep_writev(sqe, fd, iov, count, 0); ++ io_uring_sqe_set_data(sqe, data); ++} ++ ++static ngx_int_t ngx_io_uring_add_event_with_buf(ngx_event_t *ev, ++ ngx_int_t event, ++ u_char *buf, ++ size_t size) ++{ ++ ngx_connection_t *c = ev->data; ++ ++ ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_uring add event with buf: fd:%d ev:%08XD, data:%p, size %uz", ++ c->fd, event, (void *) ((uintptr_t) c), size); ++ ++ if (event == NGX_READ_EVENT) { ++ io_uring_add_recv(&ngx_ring, c->fd, buf, size, ++ (void *) ((uintptr_t) c | ev->instance | 0x6)); ++ } else { ++ io_uring_add_send(&ngx_ring, c->fd, buf, size, ++ (void *) ((uintptr_t) c | ev->instance | 0x2)); ++ } ++ ++ ev->active = 1; ++ ev->complete = 0; ++ ++ return NGX_OK; ++ ++} ++ ++static ngx_int_t ngx_io_uring_add_event_with_iovec(ngx_event_t *ev, ++ ngx_int_t event, ++ struct iovec *iov, ++ ngx_int_t count) ++{ ++ ngx_connection_t *c = ev->data; ++ ++ ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, ++ "io_uring add event with iovec: fd:%d ev:%08XD, data:%p", ++ c->fd, event, (void *) ((uintptr_t) c)); ++ ++ if (event == NGX_READ_EVENT) { ++ io_uring_add_readv(&ngx_ring, c->fd, iov, count, ++ (void *) ((uintptr_t) c | ev->instance | 0x6)); ++ } else { ++ io_uring_add_writev(&ngx_ring, c->fd, iov, count, ++ (void *) ((uintptr_t) c | ev->instance | 0x2)); ++ } ++ ++ ev->active = 1; ++ ev->complete = 0; ++ ++ return NGX_OK; ++} ++ + static ngx_int_t + ngx_io_uring_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) + { +@@ -317,7 +363,7 @@ static ngx_int_t + ngx_io_uring_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + { + uint32_t cqe_count, revents; +- ngx_int_t instance, i; ++ ngx_int_t instance, i, rw; + ngx_uint_t level; + ngx_err_t err = NGX_OK; + ngx_event_t *rev, *wev; +@@ -362,7 +408,8 @@ ngx_io_uring_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t fla + + data = io_uring_cqe_get_data(cqe); + instance = (uintptr_t) data & 1; +- c = (ngx_connection_t *) ((uintptr_t) data & (uintptr_t) ~1); ++ rw = (uintptr_t) data & 0x2; ++ c = (ngx_connection_t *) ((uintptr_t) data & (uintptr_t) ~0x7); + + rev = c->read; + +@@ -380,74 +427,130 @@ ngx_io_uring_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t fla + continue; + } + +- revents = cqe->res; +- io_uring_cqe_seen(&ngx_ring, cqe); +- + ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0, +- "io_uring_peek_cqe: fd:%d ev:%04XD d:%p listening:%p active:%d", +- c->fd, revents, data, c->listening, rev->active); ++ "io_uring_peek_cqe: fd:%d res:%d d:%p listening:%p active:%d", ++ c->fd, cqe->res, data, c->listening, rev->active); + +- if (revents & (EPOLLERR|EPOLLHUP)) { +- ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, +- "io_uring_peek_cqe error on fd:%d ev:%04XD", +- c->fd, revents); ++ if (!rw) { /* poll_add request */ ++ revents = cqe->res; ++ io_uring_cqe_seen(&ngx_ring, cqe); + +- /* +- * if the error events were returned, add EPOLLIN and EPOLLOUT +- * to handle the events at least in one active handler +- */ ++ if (cqe->res < 0) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, revents, "poll request failed"); ++ continue; ++ } ++ if (revents & (EPOLLERR|EPOLLHUP)) { ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring_peek_cqe error on fd:%d ev:%04XD", ++ c->fd, revents); + +- revents |= EPOLLIN|EPOLLOUT; +- } ++ /* ++ * if the error events were returned, add EPOLLIN and EPOLLOUT ++ * to handle the events at least in one active handler ++ */ ++ ++ revents |= EPOLLIN|EPOLLOUT; ++ } ++ ++ if ((revents & EPOLLIN) && rev->active) { ++ rev->ready = 1; ++ rev->available = -1; ++ rev->active = 0; ++ ++ if (flags & NGX_POST_EVENTS) { ++ queue = rev->accept ? &ngx_posted_accept_events ++ : &ngx_posted_events; + +- if ((revents & EPOLLIN) && rev->active) { ++ ngx_post_event(rev, queue); + +-#if (NGX_HAVE_EPOLLRDHUP) +- if (revents & EPOLLRDHUP) { +- rev->pending_eof = 1; ++ } else { ++ rev->handler(rev); ++ } + } +-#endif + +- rev->ready = 1; +- rev->available = -1; +- rev->active = 0; ++ wev = c->write; + +- if (flags & NGX_POST_EVENTS) { +- queue = rev->accept ? &ngx_posted_accept_events +- : &ngx_posted_events; ++ if ((revents & EPOLLOUT) && wev->active) { + +- ngx_post_event(rev, queue); ++ if (c->fd == -1 || wev->instance != instance) { + +- } else { +- rev->handler(rev); ++ /* ++ * the stale event from a file descriptor ++ * that was just closed in this iteration ++ */ ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "epoll: stale event %p", c); ++ continue; ++ } ++ ++ wev->ready = 1; ++ wev->active = 0; ++#if (NGX_THREADS) ++ wev->complete = 1; ++#endif ++ ++ if (flags & NGX_POST_EVENTS) { ++ ngx_post_event(wev, &ngx_posted_events); ++ ++ } else { ++ wev->handler(wev); ++ } + } ++ continue; + } + +- wev = c->write; +- +- if ((revents & EPOLLOUT) && wev->active) { ++ /* readv/writev request */ ++ if (((uintptr_t) data) & 0x4) { ++ if (!rev->active) { ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "readv: not active %p", c); + +- if (c->fd == -1 || wev->instance != instance) { ++ io_uring_cqe_seen(&ngx_ring, cqe); ++ continue; ++ } ++ rev->nbytes = cqe->res; ++ rev->available = -1; ++ rev->active = 0; ++ rev->complete = 1; ++ io_uring_cqe_seen(&ngx_ring, cqe); + +- /* +- * the stale event from a file descriptor +- * that was just closed in this iteration +- */ ++ ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring_read_ready: fd:%d d:%p connection:%p active:%d nbytes:%uz", ++ c->fd, data, c, rev->active, rev->nbytes); + ++ if (flags & NGX_POST_EVENTS) { ++ ngx_post_event(rev, &ngx_posted_events); ++ } else { ++ rev->handler(rev); ++ } ++ } else { ++ wev = c->write; ++ if (!wev->active) { + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, +- "epoll: stale event %p", c); ++ "writev: not active %p", c); ++ ++ io_uring_cqe_seen(&ngx_ring, cqe); + continue; + } + +- wev->ready = 1; +- wev->active = 0; +-#if (NGX_THREADS) ++ if (c->fd == -1 || wev->instance != instance) { ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "writev: stale event %p", c); ++ io_uring_cqe_seen(&ngx_ring, cqe); ++ continue; ++ } ++ wev->nbytes = cqe->res; + wev->complete = 1; +-#endif ++ wev->active = 0; ++ io_uring_cqe_seen(&ngx_ring, cqe); ++ ++ ngx_log_debug5(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "io_uring_write_ready: fd:%d d:%p connection:%p active:%d nbytes:%uz", ++ c->fd, data, c, wev->active, wev->nbytes); + + if (flags & NGX_POST_EVENTS) { + ngx_post_event(wev, &ngx_posted_events); +- + } else { + wev->handler(wev); + } +diff --git a/src/event/modules/ngx_kqueue_module.c b/src/event/modules/ngx_kqueue_module.c +index 9c7244c4587a..33319f1225fe 100644 +--- a/src/event/modules/ngx_kqueue_module.c ++++ b/src/event/modules/ngx_kqueue_module.c +@@ -80,6 +80,8 @@ static ngx_event_module_t ngx_kqueue_module_ctx = { + + { + ngx_kqueue_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_kqueue_del_event, /* delete an event */ + ngx_kqueue_add_event, /* enable an event */ + ngx_kqueue_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_poll_module.c b/src/event/modules/ngx_poll_module.c +index c16f0242e4a4..416bc121f45c 100644 +--- a/src/event/modules/ngx_poll_module.c ++++ b/src/event/modules/ngx_poll_module.c +@@ -34,6 +34,8 @@ static ngx_event_module_t ngx_poll_module_ctx = { + + { + ngx_poll_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_poll_del_event, /* delete an event */ + ngx_poll_add_event, /* enable an event */ + ngx_poll_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_select_module.c b/src/event/modules/ngx_select_module.c +index b9fceb3b648b..cf7ad34e2757 100644 +--- a/src/event/modules/ngx_select_module.c ++++ b/src/event/modules/ngx_select_module.c +@@ -42,6 +42,8 @@ static ngx_event_module_t ngx_select_module_ctx = { + + { + ngx_select_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_select_del_event, /* delete an event */ + ngx_select_add_event, /* enable an event */ + ngx_select_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_win32_poll_module.c b/src/event/modules/ngx_win32_poll_module.c +index 2fbc1b376aef..fca1d0288f2b 100644 +--- a/src/event/modules/ngx_win32_poll_module.c ++++ b/src/event/modules/ngx_win32_poll_module.c +@@ -36,6 +36,8 @@ static ngx_event_module_t ngx_poll_module_ctx = { + + { + ngx_poll_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_poll_del_event, /* delete an event */ + ngx_poll_add_event, /* enable an event */ + ngx_poll_del_event, /* disable an event */ +diff --git a/src/event/modules/ngx_win32_select_module.c b/src/event/modules/ngx_win32_select_module.c +index 962514a3668f..ace4f4ede72c 100644 +--- a/src/event/modules/ngx_win32_select_module.c ++++ b/src/event/modules/ngx_win32_select_module.c +@@ -44,6 +44,8 @@ static ngx_event_module_t ngx_select_module_ctx = { + + { + ngx_select_add_event, /* add an event */ ++ NULL, ++ NULL, + ngx_select_del_event, /* delete an event */ + ngx_select_add_event, /* enable an event */ + ngx_select_del_event, /* disable an event */ +diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c +index 9e9484e76f60..7f15280f1baf 100644 +--- a/src/event/ngx_event.c ++++ b/src/event/ngx_event.c +@@ -171,7 +171,7 @@ static ngx_event_module_t ngx_event_core_module_ctx = { + ngx_event_core_create_conf, /* create configuration */ + ngx_event_core_init_conf, /* init configuration */ + +- { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } ++ { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } + }; + + +@@ -265,6 +265,45 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle) + ngx_event_process_posted(cycle, &ngx_posted_events); + } + ++ngx_int_t ++ngx_handle_read_event_with_buf(ngx_event_t *rev, u_char *buf, size_t size) ++{ ++ if (!rev->active && !rev->complete) { ++ if (ngx_add_event_with_buf(rev, NGX_READ_EVENT, buf, size) == NGX_ERROR) ++ return NGX_ERROR; ++ } ++ return NGX_OK; ++} ++ ++ngx_int_t ++ngx_handle_write_event_with_buf(ngx_event_t *wev, u_char *buf, size_t size) ++{ ++ if (!wev->active && !wev->complete) { ++ if (ngx_add_event_with_buf(wev, NGX_WRITE_EVENT, buf, size) == NGX_ERROR) ++ return NGX_ERROR; ++ } ++ return NGX_OK; ++} ++ ++ngx_int_t ++ngx_handle_read_event_with_iovec(ngx_event_t *rev, struct iovec *iov, ngx_int_t count) ++{ ++ if (!rev->active && !rev->complete) { ++ if (ngx_add_event_with_iovec(rev, NGX_READ_EVENT, iov, count) == NGX_ERROR) ++ return NGX_ERROR; ++ } ++ return NGX_OK; ++} ++ ++ngx_int_t ++ngx_handle_write_event_with_iovec(ngx_event_t *wev, struct iovec *iov, ngx_int_t count) ++{ ++ if (!wev->active && !wev->complete) { ++ if (ngx_add_event_with_iovec(wev, NGX_WRITE_EVENT, iov, count) == NGX_ERROR) ++ return NGX_ERROR; ++ } ++ return NGX_OK; ++} + + ngx_int_t + ngx_handle_read_event(ngx_event_t *rev, ngx_uint_t flags) +diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h +index 9d3c155bb511..87467615cf77 100644 +--- a/src/event/ngx_event.h ++++ b/src/event/ngx_event.h +@@ -116,6 +116,10 @@ struct ngx_event_s { + /* the posted queue */ + ngx_queue_t queue; + ++#if (NGX_HAVE_IO_URING) ++ ssize_t nbytes; ++#endif ++ + #if 0 + + /* the threads support */ +@@ -169,6 +173,12 @@ struct ngx_event_aio_s { + + typedef struct { + ngx_int_t (*add)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); ++ ngx_int_t (*add_with_iovec)(ngx_event_t *ev, ++ ngx_int_t event, struct iovec *iov, ++ ngx_int_t count); ++ ngx_int_t (*add_with_buf)(ngx_event_t *ev, ++ ngx_int_t event, u_char *buf, ++ size_t size); + ngx_int_t (*del)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); + + ngx_int_t (*enable)(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); +@@ -271,6 +281,9 @@ extern ngx_uint_t ngx_use_epoll_rdhup; + */ + #define NGX_USE_VNODE_EVENT 0x00002000 + ++/* The event filter support io_uring */ ++#define NGX_USE_IO_URING_EVENT 0x00004000 ++ + /* + * The event filter is deleted just before the closing file. + * Has no meaning for select and poll. +@@ -404,6 +417,8 @@ extern ngx_uint_t ngx_use_epoll_rdhup; + #define ngx_done_events ngx_event_actions.done + + #define ngx_add_event ngx_event_actions.add ++#define ngx_add_event_with_iovec ngx_event_actions.add_with_iovec ++#define ngx_add_event_with_buf ngx_event_actions.add_with_buf + #define ngx_del_event ngx_event_actions.del + #define ngx_add_conn ngx_event_actions.add_conn + #define ngx_del_conn ngx_event_actions.del_conn +@@ -513,7 +528,18 @@ void ngx_debug_accepted_connection(ngx_event_conf_t *ecf, ngx_connection_t *c); + void ngx_process_events_and_timers(ngx_cycle_t *cycle); + ngx_int_t ngx_handle_read_event(ngx_event_t *rev, ngx_uint_t flags); + ngx_int_t ngx_handle_write_event(ngx_event_t *wev, size_t lowat); +- ++ngx_int_t ngx_handle_read_event_with_iovec(ngx_event_t *rev, ++ struct iovec *iov, ++ ngx_int_t count); ++ngx_int_t ngx_handle_write_event_with_iovec(ngx_event_t *wev, ++ struct iovec *iov, ++ ngx_int_t count); ++ngx_int_t ngx_handle_read_event_with_buf(ngx_event_t *rev, ++ u_char *buf, ++ size_t size); ++ngx_int_t ngx_handle_write_event_with_buf(ngx_event_t *wev, ++ u_char *buf, ++ size_t size); + + #if (NGX_WIN32) + void ngx_event_acceptex(ngx_event_t *ev); +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index 9a859c25cccd..752cc5a97ce5 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -314,7 +314,7 @@ ngx_event_accept(ngx_event_t *ev) + } while (ev->available); + + /* re-add event poll for listening */ +-#if (HAVE_IO_URING) ++#if (NGX_HAVE_IO_URING) + #if (NGX_HAVE_EPOLLEXCLUSIVE) + if (ngx_event_flags & NGX_USE_EPOLL_EVENT) { + if (ngx_add_event(ev, NGX_READ_EVENT, NGX_EXCLUSIVE_EVENT) +-- +2.31.1 + diff --git a/0004-io_uring-add-io_uring-read-support.patch b/0004-io_uring-add-io_uring-read-support.patch new file mode 100644 index 0000000..47e649d --- /dev/null +++ b/0004-io_uring-add-io_uring-read-support.patch @@ -0,0 +1,242 @@ +From 46e5d70ce3024d7e98eebbd2c338b23f98826b1c Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Thu, 20 Aug 2020 13:32:49 +0800 +Subject: [PATCH 04/10] io_uring: add io_uring read support + +Signed-off-by: Jiufei Xue +--- + src/event/ngx_event_accept.c | 4 + + src/http/ngx_http_request.c | 153 ++++++++++++++++++++++++++++++++++- + 2 files changed, 155 insertions(+), 2 deletions(-) + +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index 752cc5a97ce5..efa2f39163c2 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -235,6 +235,10 @@ ngx_event_accept(ngx_event_t *ev) + rev->ready = 1; + } + ++ if (ngx_event_flags & NGX_USE_IO_URING_EVENT) { ++ rev->ready = 1; ++ } ++ + if (ev->deferred_accept) { + rev->ready = 1; + #if (NGX_HAVE_KQUEUE || NGX_HAVE_EPOLLRDHUP) +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index 6feb6cc31c30..bd8ebfb4b083 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -350,7 +350,7 @@ ngx_http_init_connection(ngx_connection_t *c) + } + + if (rev->ready) { +- /* the deferred accept(), iocp */ ++ /* the deferred accept(), iocp, io_uring */ + + if (ngx_use_accept_mutex) { + ngx_post_event(rev, &ngx_posted_events); +@@ -370,6 +370,134 @@ ngx_http_init_connection(ngx_connection_t *c) + } + } + ++#if (NGX_HAVE_IO_URING) ++static ssize_t ngx_get_async_read_bytes(ngx_event_t *rev) ++{ ++ ngx_err_t err; ++ ssize_t n; ++ ngx_connection_t *c = rev->data; ++ ++ n = rev->nbytes; ++ rev->complete = 0; ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "get read result: fd:%d %z", c->fd, n); ++ if (n == 0) { ++ rev->eof = 1; ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client closed connection"); ++ return 0; ++ } ++ ++ if (n > 0) ++ return n; ++ ++ err = -n; ++ if (err == NGX_EAGAIN || err == NGX_EINTR) { ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err,"readv() not ready"); ++ n = NGX_AGAIN; ++ } else { ++ c->log_error = NGX_ERROR_IGNORE_ECONNRESET; ++ n = ngx_connection_error(c, err, "readv() failed"); ++ } ++ ++ return n; ++} ++ ++static void ++ngx_http_read_request_handler(ngx_event_t *rev) ++{ ++ u_char *p; ++ size_t size; ++ ssize_t n; ++ ngx_buf_t *b; ++ ngx_connection_t *c; ++ ngx_http_connection_t *hc; ++ ngx_http_core_srv_conf_t *cscf; ++ ++ c = rev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http read request handler"); ++ ++ if (rev->timedout) { ++ ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ if (c->close) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ b = c->buffer; ++ hc = c->data; ++ n = ngx_get_async_read_bytes(rev); ++ ++ if (n == NGX_AGAIN) { ++ ++ if (!rev->timer_set) ++ ngx_add_timer(rev, c->listening->post_accept_timeout); ++ ++ cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_core_module); ++ size = cscf->client_header_buffer_size; ++ if (ngx_handle_read_event_with_buf(rev, b->last, size) != NGX_OK) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ return; ++ } ++ ++ if (n < 0) { ++ ngx_log_error(NGX_LOG_INFO, c->log, n, ++ "http read request failed"); ++ ngx_http_close_connection(c); ++ return; ++ } ++ if (n == 0) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client closed connection"); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ b->last += n; ++ ++ if (hc->proxy_protocol) { ++ hc->proxy_protocol = 0; ++ ++ p = ngx_proxy_protocol_read(c, b->pos, b->last); ++ ++ if (p == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ b->pos = p; ++ ++ if (b->pos == b->last) { ++ c->log->action = "waiting for request"; ++ b->pos = b->start; ++ b->last = b->start; ++ ngx_post_event(rev, &ngx_posted_events); ++ return; ++ } ++ } ++ ++ c->log->action = "reading client request line"; ++ ++ ngx_reusable_connection(c, 0); ++ ++ c->data = ngx_http_create_request(c); ++ if (c->data == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ rev->handler = ngx_http_process_request_line; ++ ngx_http_process_request_line(rev); ++} ++#endif + + static void + ngx_http_wait_request_handler(ngx_event_t *rev) +@@ -386,6 +514,7 @@ ngx_http_wait_request_handler(ngx_event_t *rev) + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http wait request handler"); + ++#if !(NGX_HAVE_IO_URING) + if (rev->timedout) { + ngx_log_error(NGX_LOG_INFO, c->log, NGX_ETIMEDOUT, "client timed out"); + ngx_http_close_connection(c); +@@ -396,6 +525,7 @@ ngx_http_wait_request_handler(ngx_event_t *rev) + ngx_http_close_connection(c); + return; + } ++#endif + + hc = c->data; + cscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_core_module); +@@ -426,6 +556,13 @@ ngx_http_wait_request_handler(ngx_event_t *rev) + b->end = b->last + size; + } + ++#if (NGX_HAVE_IO_URING) ++ rev->handler = ngx_http_read_request_handler; ++ if (ngx_handle_read_event_with_buf(rev, b->last, size) != NGX_OK) { ++ ngx_http_close_connection(c); ++ } ++ return; ++#endif + n = c->recv(c, b->last, size); + + if (n == NGX_AGAIN) { +@@ -1519,9 +1656,14 @@ ngx_http_read_request_header(ngx_http_request_t *r) + return n; + } + ++#if (NGX_HAVE_IO_URING) ++ if (rev->complete) { ++ n = ngx_get_async_read_bytes(rev); ++#else + if (rev->ready) { + n = c->recv(c, r->header_in->last, + r->header_in->end - r->header_in->last); ++#endif + } else { + n = NGX_AGAIN; + } +@@ -1532,11 +1674,18 @@ ngx_http_read_request_header(ngx_http_request_t *r) + ngx_add_timer(rev, cscf->client_header_timeout); + } + ++#if (NGX_HAVE_IO_URING) ++ if (ngx_handle_read_event_with_buf(rev, r->header_in->last, ++ r->header_in->end - r->header_in->last) != NGX_OK) { ++ ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); ++ return NGX_ERROR; ++ }; ++#else + if (ngx_handle_read_event(rev, 0) != NGX_OK) { + ngx_http_close_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); + return NGX_ERROR; + } +- ++#endif + return NGX_AGAIN; + } + +-- +2.31.1 + diff --git a/0005-add-iovs-to-structure-ngx_connection_s.patch b/0005-add-iovs-to-structure-ngx_connection_s.patch new file mode 100644 index 0000000..9e5493d --- /dev/null +++ b/0005-add-iovs-to-structure-ngx_connection_s.patch @@ -0,0 +1,95 @@ +From e1a2d723a6b6bbad9a55268179016e0f046774ff Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Thu, 20 Aug 2020 14:06:15 +0800 +Subject: [PATCH 05/10] add iovs to structure ngx_connection_s + +We should hold the memory of iov when send writev request +to io_uring. So it should not allocated on stack. + +Signed-off-by: Jiufei Xue +--- + src/core/ngx_connection.h | 2 ++ + src/os/unix/ngx_linux_sendfile_chain.c | 3 +-- + src/os/unix/ngx_udp_sendmsg_chain.c | 3 +-- + src/os/unix/ngx_writev_chain.c | 3 +-- + 4 files changed, 5 insertions(+), 6 deletions(-) + +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index ad6556d0c3f0..a8c3917406cf 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -193,6 +193,8 @@ struct ngx_connection_s { + #if (NGX_THREADS || NGX_COMPAT) + ngx_thread_task_t *sendfile_task; + #endif ++ ++ struct iovec iovs[NGX_IOVS_PREALLOCATE]; + }; + + +diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c +index 5695839b0a26..83984e1d3bfc 100644 +--- a/src/os/unix/ngx_linux_sendfile_chain.c ++++ b/src/os/unix/ngx_linux_sendfile_chain.c +@@ -55,7 +55,6 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + ngx_event_t *wev; + ngx_chain_t *cl; + ngx_iovec_t header; +- struct iovec headers[NGX_IOVS_PREALLOCATE]; + + wev = c->write; + +@@ -73,7 +72,7 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + + send = 0; + +- header.iovs = headers; ++ header.iovs = c->iovs; + header.nalloc = NGX_IOVS_PREALLOCATE; + + for ( ;; ) { +diff --git a/src/os/unix/ngx_udp_sendmsg_chain.c b/src/os/unix/ngx_udp_sendmsg_chain.c +index 3d1d6dde4c8b..7d18574c81cd 100644 +--- a/src/os/unix/ngx_udp_sendmsg_chain.c ++++ b/src/os/unix/ngx_udp_sendmsg_chain.c +@@ -23,7 +23,6 @@ ngx_udp_unix_sendmsg_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + ngx_chain_t *cl; + ngx_event_t *wev; + ngx_iovec_t vec; +- struct iovec iovs[NGX_IOVS_PREALLOCATE]; + + wev = c->write; + +@@ -50,7 +49,7 @@ ngx_udp_unix_sendmsg_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + + send = 0; + +- vec.iovs = iovs; ++ vec.iovs = c->iovs; + vec.nalloc = NGX_IOVS_PREALLOCATE; + + for ( ;; ) { +diff --git a/src/os/unix/ngx_writev_chain.c b/src/os/unix/ngx_writev_chain.c +index e38a3aae00dd..c82d6322667d 100644 +--- a/src/os/unix/ngx_writev_chain.c ++++ b/src/os/unix/ngx_writev_chain.c +@@ -18,7 +18,6 @@ ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + ngx_chain_t *cl; + ngx_event_t *wev; + ngx_iovec_t vec; +- struct iovec iovs[NGX_IOVS_PREALLOCATE]; + + wev = c->write; + +@@ -45,7 +44,7 @@ ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + + send = 0; + +- vec.iovs = iovs; ++ vec.iovs = c->iovs; + vec.nalloc = NGX_IOVS_PREALLOCATE; + + for ( ;; ) { +-- +2.31.1 + diff --git a/0006-io_uring-add-io_uring-writev-support.patch b/0006-io_uring-add-io_uring-writev-support.patch new file mode 100644 index 0000000..137d2c8 --- /dev/null +++ b/0006-io_uring-add-io_uring-writev-support.patch @@ -0,0 +1,427 @@ +From 07a8c9ed1ae9eb3f51b4eb9a9f6ea2a5d059d18a Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Thu, 20 Aug 2020 17:33:52 +0800 +Subject: [PATCH 06/10] io_uring: add io_uring writev support + +Signed-off-by: Jiufei Xue +--- + src/core/ngx_buf.h | 1 + + src/core/ngx_connection.h | 1 + + src/event/ngx_event.h | 3 + + src/event/ngx_event_accept.c | 3 + + src/http/ngx_http_request.c | 105 ++++++++++++++++++++++++ + src/http/ngx_http_write_filter_module.c | 6 ++ + src/os/unix/ngx_darwin_init.c | 6 ++ + src/os/unix/ngx_freebsd_init.c | 6 ++ + src/os/unix/ngx_linux_init.c | 6 ++ + src/os/unix/ngx_os.h | 5 ++ + src/os/unix/ngx_posix_init.c | 3 + + src/os/unix/ngx_solaris_init.c | 6 ++ + src/os/unix/ngx_writev_chain.c | 59 +++++++++++++ + 13 files changed, 210 insertions(+) + +diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h +index 12781a782198..48fd4b39f8ba 100644 +--- a/src/core/ngx_buf.h ++++ b/src/core/ngx_buf.h +@@ -123,6 +123,7 @@ typedef struct { + + + #define NGX_CHAIN_ERROR (ngx_chain_t *) NGX_ERROR ++#define NGX_CHAIN_EAGAIN (ngx_chain_t *) NGX_EAGAIN + + + #define ngx_buf_in_memory(b) (b->temporary || b->memory || b->mmap) +diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h +index a8c3917406cf..c406a730649f 100644 +--- a/src/core/ngx_connection.h ++++ b/src/core/ngx_connection.h +@@ -132,6 +132,7 @@ struct ngx_connection_s { + ngx_send_pt send; + ngx_recv_chain_pt recv_chain; + ngx_send_chain_pt send_chain; ++ ngx_send_chain_pt async_send_chain; + + ngx_listening_t *listening; + +diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h +index 87467615cf77..e66cca8c3666 100644 +--- a/src/event/ngx_event.h ++++ b/src/event/ngx_event.h +@@ -436,6 +436,9 @@ extern ngx_os_io_t ngx_io; + #define ngx_udp_recv ngx_io.udp_recv + #define ngx_send ngx_io.send + #define ngx_send_chain ngx_io.send_chain ++#if (NGX_HAVE_IO_URING) ++#define ngx_async_send_chain ngx_io.async_send_chain ++#endif + #define ngx_udp_send ngx_io.udp_send + #define ngx_udp_send_chain ngx_io.udp_send_chain + +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index efa2f39163c2..00221d050234 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -206,6 +206,9 @@ ngx_event_accept(ngx_event_t *ev) + c->send = ngx_send; + c->recv_chain = ngx_recv_chain; + c->send_chain = ngx_send_chain; ++#if (NGX_HAVE_IO_URING) ++ c->async_send_chain = ngx_async_send_chain; ++#endif + + c->log = log; + c->pool->log = log; +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index bd8ebfb4b083..b7e726846252 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -45,6 +45,9 @@ static void ngx_http_terminate_handler(ngx_http_request_t *r); + static void ngx_http_finalize_connection(ngx_http_request_t *r); + static ngx_int_t ngx_http_set_write_handler(ngx_http_request_t *r); + static void ngx_http_writer(ngx_http_request_t *r); ++#if (NGX_HAVE_IO_URING) ++static void ngx_http_writer_done(ngx_http_request_t *r); ++#endif + static void ngx_http_request_finalizer(ngx_http_request_t *r); + + static void ngx_http_set_keepalive(ngx_http_request_t *r); +@@ -2597,6 +2600,13 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) + c->error = 1; + } + ++#if (NGX_HAVE_IO_URING) ++ if (rc == NGX_AGAIN && c->write->active && !c->write->complete) { ++ r->write_event_handler = ngx_http_writer_done; ++ return; ++ } ++#endif ++ + if (rc == NGX_DECLINED) { + r->content_handler = NULL; + r->write_event_handler = ngx_http_core_run_phases; +@@ -2921,6 +2931,101 @@ ngx_http_set_write_handler(ngx_http_request_t *r) + return NGX_OK; + } + ++#if (NGX_HAVE_IO_URING) ++static ssize_t ngx_get_async_write_bytes(ngx_event_t *wev) ++{ ++ ngx_err_t err; ++ ssize_t n; ++ ngx_connection_t *c = wev->data; ++ ++ n = wev->nbytes; ++ wev->complete = 0; ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "get write result: fd:%d %z", c->fd, n); ++ if (n > 0) ++ return n; ++ ++ err = -n; ++ wev->error = 1; ++ ngx_connection_error(c, err, "writev() failed"); ++ return NGX_ERROR; ++} ++ ++static void ++ngx_http_writer_done(ngx_http_request_t *r) ++{ ++ ssize_t n, sent, nsent; ++ off_t limit; ++ ngx_connection_t *c = r->connection; ++ ngx_event_t *wev = c->write; ++ ngx_chain_t *cl, *ln, *chain; ++ ngx_msec_t delay; ++ ++ if (!wev->complete) ++ ngx_log_error(NGX_LOG_CRIT, c->log, NGX_ERROR, "write not complete"); ++ ++ n = ngx_get_async_write_bytes(wev); ++ sent = (n < 0) ? 0 : n; ++ c->sent += sent; ++ chain = ngx_chain_update_sent(r->out, sent); ++ ++ if (r->limit_rate) { ++ ++ nsent = c->sent; ++ ++ if (r->limit_rate_after) { ++ ++ sent -= r->limit_rate_after; ++ if (sent < 0) { ++ sent = 0; ++ } ++ ++ nsent -= r->limit_rate_after; ++ if (nsent < 0) { ++ nsent = 0; ++ } ++ } ++ ++ delay = (ngx_msec_t) ((nsent - sent) * 1000 / r->limit_rate); ++ ++ if (delay > 0) { ++ limit = 0; ++ c->write->delayed = 1; ++ ngx_add_timer(c->write, delay); ++ } ++ } ++ ++ if (limit ++ && c->write->ready ++ && c->sent - sent >= limit - (off_t) (2 * ngx_pagesize)) ++ { ++ c->write->delayed = 1; ++ ngx_add_timer(c->write, 1); ++ } ++ ++ for (cl = r->out; cl && cl != chain; /* void */) { ++ ln = cl; ++ cl = cl->next; ++ ngx_free_chain(r->pool, ln); ++ } ++ ++ r->out = chain; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, ++ "http write done chain %p", chain); ++ ++ /* TODO: should traverse the filter layters */ ++ if (chain) { ++ /* need to send the rest buf */ ++ c->buffered |= NGX_HTTP_WRITE_BUFFERED; ++ ngx_http_finalize_request(r, NGX_AGAIN); ++ } else { ++ c->buffered &= ~NGX_HTTP_WRITE_BUFFERED; ++ r->write_event_handler = ngx_http_request_empty_handler; ++ ngx_http_finalize_request(r, NGX_DONE); ++ } ++} ++#endif + + static void + ngx_http_writer(ngx_http_request_t *r) +diff --git a/src/http/ngx_http_write_filter_module.c b/src/http/ngx_http_write_filter_module.c +index 6a5d957b169f..1cca6a52554a 100644 +--- a/src/http/ngx_http_write_filter_module.c ++++ b/src/http/ngx_http_write_filter_module.c +@@ -291,7 +291,11 @@ ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *in) + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http write filter limit %O", limit); + ++#if (NGX_HAVE_IO_URING) ++ chain = c->async_send_chain(c, r->out, limit); ++#else + chain = c->send_chain(c, r->out, limit); ++#endif + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http write filter %p", chain); +@@ -300,6 +304,8 @@ ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *in) + c->error = 1; + return NGX_ERROR; + } ++ if (chain == NGX_CHAIN_EAGAIN) ++ return NGX_AGAIN; + + if (r->limit_rate) { + +diff --git a/src/os/unix/ngx_darwin_init.c b/src/os/unix/ngx_darwin_init.c +index aabe02ff963f..a8b3b558f212 100644 +--- a/src/os/unix/ngx_darwin_init.c ++++ b/src/os/unix/ngx_darwin_init.c +@@ -27,9 +27,15 @@ static ngx_os_io_t ngx_darwin_io = { + ngx_udp_unix_sendmsg_chain, + #if (NGX_HAVE_SENDFILE) + ngx_darwin_sendfile_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + NGX_IO_SENDFILE + #else + ngx_writev_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + 0 + #endif + }; +diff --git a/src/os/unix/ngx_freebsd_init.c b/src/os/unix/ngx_freebsd_init.c +index 1823f028139b..6e9fbdff1671 100644 +--- a/src/os/unix/ngx_freebsd_init.c ++++ b/src/os/unix/ngx_freebsd_init.c +@@ -36,9 +36,15 @@ static ngx_os_io_t ngx_freebsd_io = { + ngx_udp_unix_sendmsg_chain, + #if (NGX_HAVE_SENDFILE) + ngx_freebsd_sendfile_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + NGX_IO_SENDFILE + #else + ngx_writev_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + 0 + #endif + }; +diff --git a/src/os/unix/ngx_linux_init.c b/src/os/unix/ngx_linux_init.c +index a8cf6a048d13..10b3478a0809 100644 +--- a/src/os/unix/ngx_linux_init.c ++++ b/src/os/unix/ngx_linux_init.c +@@ -22,9 +22,15 @@ static ngx_os_io_t ngx_linux_io = { + ngx_udp_unix_sendmsg_chain, + #if (NGX_HAVE_SENDFILE) + ngx_linux_sendfile_chain, ++#if (NGX_HAVE_IO_URING) ++ ngx_async_writev_chain, ++#endif + NGX_IO_SENDFILE + #else + ngx_writev_chain, ++#if (NGX_HAVE_IO_URING) ++ ngx_async_writev_chain, ++#endif + 0 + #endif + }; +diff --git a/src/os/unix/ngx_os.h b/src/os/unix/ngx_os.h +index 3b328191bb5a..ce485ed3af07 100644 +--- a/src/os/unix/ngx_os.h ++++ b/src/os/unix/ngx_os.h +@@ -31,6 +31,9 @@ typedef struct { + ngx_send_pt udp_send; + ngx_send_chain_pt udp_send_chain; + ngx_send_chain_pt send_chain; ++#if (NGX_HAVE_IO_URING) ++ ngx_send_chain_pt async_send_chain; ++#endif + ngx_uint_t flags; + } ngx_os_io_t; + +@@ -49,6 +52,8 @@ ssize_t ngx_udp_unix_recv(ngx_connection_t *c, u_char *buf, size_t size); + ssize_t ngx_unix_send(ngx_connection_t *c, u_char *buf, size_t size); + ngx_chain_t *ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, + off_t limit); ++ngx_chain_t *ngx_async_writev_chain(ngx_connection_t *c, ngx_chain_t *in, ++ off_t limit); + ssize_t ngx_udp_unix_send(ngx_connection_t *c, u_char *buf, size_t size); + ngx_chain_t *ngx_udp_unix_sendmsg_chain(ngx_connection_t *c, ngx_chain_t *in, + off_t limit); +diff --git a/src/os/unix/ngx_posix_init.c b/src/os/unix/ngx_posix_init.c +index 7824735d0bae..401b163d36c2 100644 +--- a/src/os/unix/ngx_posix_init.c ++++ b/src/os/unix/ngx_posix_init.c +@@ -27,6 +27,9 @@ ngx_os_io_t ngx_os_io = { + ngx_udp_unix_send, + ngx_udp_unix_sendmsg_chain, + ngx_writev_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + 0 + }; + +diff --git a/src/os/unix/ngx_solaris_init.c b/src/os/unix/ngx_solaris_init.c +index 65d787539aa8..75099f40304a 100644 +--- a/src/os/unix/ngx_solaris_init.c ++++ b/src/os/unix/ngx_solaris_init.c +@@ -23,9 +23,15 @@ static ngx_os_io_t ngx_solaris_io = { + ngx_udp_unix_sendmsg_chain, + #if (NGX_HAVE_SENDFILE) + ngx_solaris_sendfilev_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + NGX_IO_SENDFILE + #else + ngx_writev_chain, ++#if (NGX_HAVE_IO_URING) ++ NULL, ++#endif + 0 + #endif + }; +diff --git a/src/os/unix/ngx_writev_chain.c b/src/os/unix/ngx_writev_chain.c +index c82d6322667d..d26ff5e7472b 100644 +--- a/src/os/unix/ngx_writev_chain.c ++++ b/src/os/unix/ngx_writev_chain.c +@@ -9,6 +9,57 @@ + #include + #include + ++#if (NGX_HAVE_IO_URING) ++ngx_chain_t * ++ngx_async_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) ++{ ++ ngx_chain_t *cl; ++ ngx_event_t *wev; ++ ngx_iovec_t vec; ++ ++ wev = c->write; ++ ++ /* the maximum limit size is the maximum size_t value - the page size */ ++ ++ if (limit == 0 || limit > (off_t) (NGX_MAX_SIZE_T_VALUE - ngx_pagesize)) { ++ limit = NGX_MAX_SIZE_T_VALUE - ngx_pagesize; ++ } ++ ++ vec.iovs = c->iovs; ++ vec.nalloc = NGX_IOVS_PREALLOCATE; ++ ++ /* create the iovec and coalesce the neighbouring bufs */ ++ cl = ngx_output_chain_to_iovec(&vec, in, limit, c->log); ++ if (cl == NGX_CHAIN_ERROR) { ++ return NGX_CHAIN_ERROR; ++ } ++ ++ if (cl && cl->buf->in_file) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, 0, ++ "file buf in writev " ++ "t:%d r:%d f:%d %p %p-%p %p %O-%O", ++ cl->buf->temporary, ++ cl->buf->recycled, ++ cl->buf->in_file, ++ cl->buf->start, ++ cl->buf->pos, ++ cl->buf->last, ++ cl->buf->file, ++ cl->buf->file_pos, ++ cl->buf->file_last); ++ ++ ngx_debug_point(); ++ ++ return NGX_CHAIN_ERROR; ++ } ++ ++ if (ngx_add_event_with_iovec(wev, NGX_WRITE_EVENT, ++ vec.iovs, vec.count) != NGX_OK) { ++ return NGX_CHAIN_ERROR; ++ } ++ return NGX_CHAIN_EAGAIN; ++} ++#endif + + ngx_chain_t * + ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) +@@ -79,7 +130,15 @@ ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) + + send += vec.size; + ++#if (NGX_HAVE_IO_URING) ++ if (ngx_add_event_with_iovec(wev, NGX_WRITE_EVENT, ++ vec.iovs, vec.count) != NGX_OK) { ++ return NGX_CHAIN_ERROR; ++ } ++ return NGX_CHAIN_EAGAIN; ++#else + n = ngx_writev(c, &vec); ++#endif + + if (n == NGX_ERROR) { + return NGX_CHAIN_ERROR; +-- +2.31.1 + diff --git a/0007-io_uring-add-keepalive-support.patch b/0007-io_uring-add-keepalive-support.patch new file mode 100644 index 0000000..32dbc1c --- /dev/null +++ b/0007-io_uring-add-keepalive-support.patch @@ -0,0 +1,147 @@ +From 85c013f68f300a734bd61c43f5d0bca26b5e9721 Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Fri, 21 Aug 2020 11:12:50 +0800 +Subject: [PATCH 07/10] io_uring: add keepalive support + +Signed-off-by: Jiufei Xue +--- + src/http/ngx_http_request.c | 83 ++++++++++++++++++++++++++++++++++++- + 1 file changed, 81 insertions(+), 2 deletions(-) + +diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c +index b7e726846252..d17bf7baea5a 100644 +--- a/src/http/ngx_http_request.c ++++ b/src/http/ngx_http_request.c +@@ -51,7 +51,11 @@ static void ngx_http_writer_done(ngx_http_request_t *r); + static void ngx_http_request_finalizer(ngx_http_request_t *r); + + static void ngx_http_set_keepalive(ngx_http_request_t *r); ++#if (NGX_HAVE_IO_URING) ++static void ngx_http_keepalive_done_handler(ngx_event_t *ev); ++#else + static void ngx_http_keepalive_handler(ngx_event_t *ev); ++#endif + static void ngx_http_set_lingering_close(ngx_http_request_t *r); + static void ngx_http_lingering_close_handler(ngx_event_t *ev); + static ngx_int_t ngx_http_post_action(ngx_http_request_t *r); +@@ -3374,7 +3378,9 @@ ngx_http_set_keepalive(ngx_http_request_t *r) + + b = c->buffer; + +- if (ngx_pfree(c->pool, b->start) == NGX_OK) { ++ /* do not free the buffer when using io_uring since we will use it soon */ ++ if (!(ngx_event_flags & NGX_USE_IO_URING_EVENT) && ++ ngx_pfree(c->pool, b->start) == NGX_OK) { + + /* + * the special note for ngx_http_keepalive_handler() that +@@ -3423,7 +3429,11 @@ ngx_http_set_keepalive(ngx_http_request_t *r) + } + #endif + ++#if (NGX_HAVE_IO_URING) ++ rev->handler = ngx_http_keepalive_done_handler; ++#else + rev->handler = ngx_http_keepalive_handler; ++#endif + + if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) { + if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) { +@@ -3458,6 +3468,14 @@ ngx_http_set_keepalive(ngx_http_request_t *r) + r->http_state = NGX_HTTP_KEEPALIVE_STATE; + #endif + ++#if (NGX_HAVE_IO_URING) ++ if (ngx_handle_read_event_with_buf(rev, b->last, b->end - b->start) ++ != NGX_OK) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ngx_add_timer(rev, clcf->keepalive_timeout); ++#else + c->idle = 1; + ngx_reusable_connection(c, 1); + +@@ -3466,9 +3484,70 @@ ngx_http_set_keepalive(ngx_http_request_t *r) + if (rev->ready) { + ngx_post_event(rev, &ngx_posted_events); + } ++#endif ++} ++ ++#if (NGX_HAVE_IO_URING) ++static void ++ngx_http_keepalive_done_handler(ngx_event_t *rev) ++{ ++ ngx_connection_t *c = rev->data; ++ ssize_t n; ++ ngx_buf_t *b = c->buffer; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "http keepalive request handler"); ++ ++ if (rev->timedout || c->close) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ n = ngx_get_async_read_bytes(rev); ++ ++ if (n == NGX_AGAIN) { ++ if (ngx_handle_read_event_with_buf(rev, b->last, b->end - b->start) != NGX_OK) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ return; ++ } ++ if (n < 0) { ++ ngx_log_error(NGX_LOG_INFO, c->log, n, ++ "http read request failed"); ++ ngx_http_close_connection(c); ++ return; ++ } ++ c->log->handler = NULL; ++ ++ if (n == 0) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "client closed connection"); ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ b->last += n; ++ ++ c->log->handler = ngx_http_log_error; ++ c->log->action = "reading client request line"; ++ ++ c->data = ngx_http_create_request(c); ++ if (c->data == NULL) { ++ ngx_http_close_connection(c); ++ return; ++ } ++ ++ c->sent = 0; ++ c->destroyed = 0; ++ ++ ngx_del_timer(rev); ++ ++ rev->handler = ngx_http_process_request_line; ++ ngx_http_process_request_line(rev); + } + + ++#else + static void + ngx_http_keepalive_handler(ngx_event_t *rev) + { +@@ -3598,7 +3677,7 @@ ngx_http_keepalive_handler(ngx_event_t *rev) + rev->handler = ngx_http_process_request_line; + ngx_http_process_request_line(rev); + } +- ++#endif + + static void + ngx_http_set_lingering_close(ngx_http_request_t *r) +-- +2.31.1 + diff --git a/0008-io_uring-add-register-file-support.patch b/0008-io_uring-add-register-file-support.patch new file mode 100644 index 0000000..f505f7e --- /dev/null +++ b/0008-io_uring-add-register-file-support.patch @@ -0,0 +1,277 @@ +From 5abbab094f1311c4e9756a5a681529412e059f9e Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Fri, 21 Aug 2020 14:10:38 +0800 +Subject: [PATCH 08/10] io_uring: add register file support + +Signed-off-by: Jiufei Xue +--- + src/event/modules/ngx_io_uring_module.c | 61 +++++++++++++++++++++++++ + src/event/ngx_event.c | 13 +++++- + src/event/ngx_event_accept.c | 12 +++++ + src/os/unix/ngx_process_cycle.c | 12 +++++ + 4 files changed, 97 insertions(+), 1 deletion(-) + +diff --git a/src/event/modules/ngx_io_uring_module.c b/src/event/modules/ngx_io_uring_module.c +index 7a96b4188f4a..cd8c4e05f05d 100644 +--- a/src/event/modules/ngx_io_uring_module.c ++++ b/src/event/modules/ngx_io_uring_module.c +@@ -11,6 +11,7 @@ + + typedef struct { + ngx_uint_t entries; ++ ngx_flag_t register_file; + } ngx_io_uring_conf_t; + + +@@ -42,6 +43,8 @@ static struct io_uring_cqe **cqes; + static ngx_uint_t nevents; + + struct io_uring ngx_ring; ++int *registered_files; ++int register_file; + + static ngx_str_t io_uring_name = ngx_string("io_uring"); + +@@ -54,6 +57,14 @@ static ngx_command_t ngx_io_uring_commands[] = { + offsetof(ngx_io_uring_conf_t, entries), + NULL }, + ++ { ngx_string("io_uring_register_file"), ++ NGX_EVENT_CONF|NGX_CONF_TAKE1, ++ ngx_conf_set_flag_slot, ++ 0, ++ offsetof(ngx_io_uring_conf_t, register_file), ++ NULL }, ++ ++ + ngx_null_command + }; + +@@ -93,6 +104,27 @@ ngx_module_t ngx_io_uring_module = { + NGX_MODULE_V1_PADDING + }; + ++static int init_register_files(ngx_cycle_t *cycle) ++{ ++ int ret, i; ++ int max_register_files = 32768; ++ ++ registered_files = ngx_alloc(max_register_files * sizeof(int), cycle->log); ++ if (!registered_files) { ++ return NGX_ERROR; ++ } ++ for (i = 0; i < max_register_files; i++) ++ registered_files[i] = -1; ++ ++ ret = io_uring_register_files(&ngx_ring, registered_files, max_register_files); ++ if (ret < 0) { ++ ngx_free(registered_files); ++ registered_files = NULL; ++ return ret; ++ } ++ return 0; ++} ++ + static ngx_int_t + ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + { +@@ -120,6 +152,18 @@ ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + } + } + ++ register_file = urcf->register_file; ++ if (register_file) { ++ if (init_register_files(cycle)) { ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, ++ "init_register_files() failed"); ++ io_uring_queue_exit(&ngx_ring); ++ ngx_ring.ring_fd = 0; ++ ngx_free(cqes); ++ return NGX_ERROR; ++ } ++ } ++ + nevents = urcf->entries / 2; + + ngx_io = ngx_os_io; +@@ -131,6 +175,7 @@ ngx_io_uring_init(ngx_cycle_t *cycle, ngx_msec_t timer) + | NGX_USE_EPOLL_EVENT + | NGX_USE_IO_URING_EVENT; + ++ + return NGX_OK; + } + +@@ -139,6 +184,7 @@ ngx_io_uring_done(ngx_cycle_t *cycle) + { + io_uring_queue_exit(&ngx_ring); + ngx_ring.ring_fd = 0; ++ ngx_free(registered_files); + ngx_free(cqes); + } + +@@ -151,6 +197,9 @@ io_uring_add_poll(struct io_uring *ring, int fd, + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_poll_add(sqe, fd, event); ++ ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, data); + } + +@@ -162,6 +211,8 @@ io_uring_remove_poll(struct io_uring *ring, void *data) + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_poll_remove(sqe, data); ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, NULL); + } + +@@ -175,6 +226,8 @@ io_uring_add_recv(struct io_uring *ring, int fd, + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_recv(sqe, fd, buf, size, 0); ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, data); + } + +@@ -188,6 +241,8 @@ io_uring_add_send(struct io_uring *ring, int fd, + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_send(sqe, fd, buf, size, 0); ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, data); + } + +@@ -201,6 +256,8 @@ io_uring_add_readv(struct io_uring *ring, int fd, + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_readv(sqe, fd, iov, count, 0); ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, data); + } + +@@ -214,6 +271,8 @@ io_uring_add_writev(struct io_uring *ring, int fd, + /* TODO: !sqe ? */ + if (!sqe) ngx_abort(); + io_uring_prep_writev(sqe, fd, iov, count, 0); ++ if (register_file) ++ sqe->flags |= IOSQE_FIXED_FILE; + io_uring_sqe_set_data(sqe, data); + } + +@@ -571,6 +630,7 @@ ngx_io_uring_create_conf(ngx_cycle_t *cycle) + } + + urcf->entries = NGX_CONF_UNSET; ++ urcf->register_file = NGX_CONF_UNSET; + + return urcf; + } +@@ -581,6 +641,7 @@ ngx_io_uring_init_conf(ngx_cycle_t *cycle, void *conf) + ngx_io_uring_conf_t *urcf = conf; + + ngx_conf_init_uint_value(urcf->entries, 512); ++ ngx_conf_init_value(urcf->register_file, 0); + + return NGX_CONF_OK; + } +diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c +index 7f15280f1baf..43b786b02a29 100644 +--- a/src/event/ngx_event.c ++++ b/src/event/ngx_event.c +@@ -77,7 +77,10 @@ ngx_atomic_t *ngx_stat_waiting = &ngx_stat_waiting0; + + #endif + +- ++#if (NGX_HAVE_IO_URING) ++extern int register_file; ++extern struct io_uring ngx_ring; ++#endif + + static ngx_command_t ngx_events_commands[] = { + +@@ -831,6 +834,14 @@ ngx_event_process_init(ngx_cycle_t *cycle) + return NGX_ERROR; + } + ++#if (NGX_HAVE_IO_URING) ++ if (register_file) { ++ if (io_uring_register_files_update(&ngx_ring, ls[i].fd, &ls[i].fd, 1) < 0) { ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "register listen fd: %d failed", ls[i].fd); ++ } ++ } ++#endif + c->type = ls[i].type; + c->log = &ls[i].log; + +diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c +index 00221d050234..f7989b3e3f00 100644 +--- a/src/event/ngx_event_accept.c ++++ b/src/event/ngx_event_accept.c +@@ -9,6 +9,10 @@ + #include + #include + ++#if (NGX_HAVE_IO_URING) ++extern struct io_uring ngx_ring; ++extern int register_file; ++#endif + + static ngx_int_t ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all); + static void ngx_close_accepted_connection(ngx_connection_t *c); +@@ -309,6 +313,14 @@ ngx_event_accept(ngx_event_t *ev) + } + } + ++#if (NGX_HAVE_IO_URING) ++ if (register_file) { ++ if (io_uring_register_files_update(&ngx_ring, c->fd, &c->fd, 1) < 0) { ++ ngx_log_error(NGX_LOG_ALERT, log, 0, "register_files_update() failed"); ++ } ++ } ++#endif ++ + log->data = NULL; + log->handler = NULL; + +diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c +index f87e0092318a..199e429d7681 100644 +--- a/src/os/unix/ngx_process_cycle.c ++++ b/src/os/unix/ngx_process_cycle.c +@@ -69,6 +69,10 @@ static ngx_cycle_t ngx_exit_cycle; + static ngx_log_t ngx_exit_log; + static ngx_open_file_t ngx_exit_log_file; + ++#if (NGX_HAVE_IO_URING) ++extern struct io_uring ngx_ring; ++extern int register_file; ++#endif + + void + ngx_master_process_cycle(ngx_cycle_t *cycle) +@@ -954,6 +958,14 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker) + ngx_last_process = 0; + #endif + ++#if (NGX_HAVE_IO_URING) ++ if (register_file == 1) { ++ if (io_uring_register_files_update(&ngx_ring, ngx_channel, &ngx_channel, 1) < 1) { ++ ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "register_files_update() failed"); ++ } ++ } ++#endif ++ + if (ngx_add_channel_event(cycle, ngx_channel, NGX_READ_EVENT, + ngx_channel_handler) + == NGX_ERROR) +-- +2.31.1 + diff --git a/0009-conf-add-io_uring-configure-for-test.patch b/0009-conf-add-io_uring-configure-for-test.patch new file mode 100644 index 0000000..a357e75 --- /dev/null +++ b/0009-conf-add-io_uring-configure-for-test.patch @@ -0,0 +1,141 @@ +From 4d37b43ad0e194410ad40c04a7c05c3899192e80 Mon Sep 17 00:00:00 2001 +From: Jiufei Xue +Date: Fri, 21 Aug 2020 14:20:56 +0800 +Subject: [PATCH 09/10] conf: add io_uring configure for test + +Signed-off-by: Jiufei Xue +--- + conf/nginx.conf.uring | 121 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 121 insertions(+) + create mode 100644 conf/nginx.conf.uring + +diff --git a/conf/nginx.conf.uring b/conf/nginx.conf.uring +new file mode 100644 +index 000000000000..c2fe2edddbbd +--- /dev/null ++++ b/conf/nginx.conf.uring +@@ -0,0 +1,121 @@ ++ ++user root; ++worker_processes 1; ++worker_cpu_affinity 10000; ++ ++#error_log logs/error.log debug; ++ ++#pid logs/nginx.pid; ++ ++ ++events { ++ worker_connections 32768; ++ io_uring_entries 32768; ++ io_uring_register_file on; ++} ++ ++ ++http { ++ include mime.types; ++ default_type application/octet-stream; ++ ++ #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' ++ # '$status $body_bytes_sent "$http_referer" ' ++ # '"$http_user_agent" "$http_x_forwarded_for"'; ++ ++ access_log off; # logs/access.log main; ++ ++ lingering_close off; ++ sendfile off; ++ #aio on; ++ #tcp_nopush on; ++ ++ #keepalive_timeout 0; ++ keepalive_timeout 65; ++ ++ #gzip on; ++ ++ server { ++ listen 80 backlog=1024; ++ server_name localhost; ++ ++ #charset koi8-r; ++ ++ access_log off; #logs/host.access.log main; ++ ++ location / { ++ return 200; ++ root html; ++ index index.html index.htm; ++ } ++ ++ #error_page 404 /404.html; ++ ++ # redirect server error pages to the static page /50x.html ++ # ++ error_page 500 502 503 504 /50x.html; ++ location = /50x.html { ++ root html; ++ } ++ ++ # proxy the PHP scripts to Apache listening on 127.0.0.1:80 ++ # ++ #location ~ \.php$ { ++ # proxy_pass http://127.0.0.1; ++ #} ++ ++ # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 ++ # ++ #location ~ \.php$ { ++ # root html; ++ # fastcgi_pass 127.0.0.1:9000; ++ # fastcgi_index index.php; ++ # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; ++ # include fastcgi_params; ++ #} ++ ++ # deny access to .htaccess files, if Apache's document root ++ # concurs with nginx's one ++ # ++ #location ~ /\.ht { ++ # deny all; ++ #} ++ } ++ ++ ++ # another virtual host using mix of IP-, name-, and port-based configuration ++ # ++ #server { ++ # listen 8000; ++ # listen somename:8080; ++ # server_name somename alias another.alias; ++ ++ # location / { ++ # root html; ++ # index index.html index.htm; ++ # } ++ #} ++ ++ ++ # HTTPS server ++ # ++ #server { ++ # listen 443 ssl; ++ # server_name localhost; ++ ++ # ssl_certificate cert.pem; ++ # ssl_certificate_key cert.key; ++ ++ # ssl_session_cache shared:SSL:1m; ++ # ssl_session_timeout 5m; ++ ++ # ssl_ciphers HIGH:!aNULL:!MD5; ++ # ssl_prefer_server_ciphers on; ++ ++ # location / { ++ # root html; ++ # index index.html index.htm; ++ # } ++ #} ++ ++} +-- +2.31.1 + diff --git a/0010-config-set.patch b/0010-config-set.patch new file mode 100644 index 0000000..3301a1d --- /dev/null +++ b/0010-config-set.patch @@ -0,0 +1,72 @@ +From 0f8d2a385bbecb5694c6be5ab87fb045266c4cff Mon Sep 17 00:00:00 2001 +From: Zhihao Cheng +Date: Tue, 24 Aug 2021 03:44:01 +0800 +Subject: [PATCH 10/10] config set + +--- + conf/nginx.conf | 24 ++++++++++++++---------- + 1 file changed, 14 insertions(+), 10 deletions(-) + +diff --git a/conf/nginx.conf b/conf/nginx.conf +index 29bc085f28c9..8162bd151d91 100644 +--- a/conf/nginx.conf ++++ b/conf/nginx.conf +@@ -1,16 +1,17 @@ + +-#user nobody; +-worker_processes 1; ++user root; ++worker_processes 1; ++#worker_cpu_affinity 110000; + +-#error_log logs/error.log; +-#error_log logs/error.log notice; +-#error_log logs/error.log info; ++#error_log logs/error.log debug; + + #pid logs/nginx.pid; + ++worker_rlimit_nofile 81920; + + events { +- worker_connections 1024; ++ worker_connections 32768; ++ io_uring_entries 32768; + } + + +@@ -22,9 +23,11 @@ http { + # '$status $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + +- #access_log logs/access.log main; ++ access_log off; # logs/access.log main; + +- sendfile on; ++ lingering_close off; ++ sendfile off; ++ #aio on; + #tcp_nopush on; + + #keepalive_timeout 0; +@@ -33,14 +36,15 @@ http { + #gzip on; + + server { +- listen 80; ++ listen 81 reuseport backlog=1024; + server_name localhost; + + #charset koi8-r; + +- #access_log logs/host.access.log main; ++ access_log off; #logs/host.access.log main; + + location / { ++ return 200; + root html; + index index.html index.htm; + } +-- +2.31.1 + diff --git a/nginx.spec b/nginx.spec index b6b824d..1b55ad6 100644 --- a/nginx.spec +++ b/nginx.spec @@ -35,6 +35,16 @@ Patch0: nginx-auto-cc-gcc.patch Patch1: nginx-1.19.1-logs-perm.patch Patch2: nginx-fix-pidfile.patch Patch3: CVE-2021-23017.patch +Patch4: 0001-configure-add-io_uring-configuration.patch +Patch5: 0002-event-add-io_uring-module.patch +Patch6: 0003-event-add-new-add_event_with_-iovec-buf-interface.patch +Patch7: 0004-io_uring-add-io_uring-read-support.patch +Patch8: 0005-add-iovs-to-structure-ngx_connection_s.patch +Patch9: 0006-io_uring-add-io_uring-writev-support.patch +Patch10: 0007-io_uring-add-keepalive-support.patch +Patch11: 0008-io_uring-add-register-file-support.patch +Patch12: 0009-conf-add-io_uring-configure-for-test.patch +Patch13: 0010-config-set.patch BuildRequires: gcc openssl-devel pcre-devel zlib-devel systemd gperftools-devel Requires: nginx-filesystem = %{epoch}:%{version}-%{release} openssl pcre -- Gitee