From 62cc5a52dcb05cd28601e966a395f28c8365598b Mon Sep 17 00:00:00 2001 From: Yixing Liu Date: Wed, 14 Dec 2022 17:42:29 +0800 Subject: [PATCH] Support libhns stop sending db mechanism after reset Add an interface to the user space, which is used to receive the kernel reset state. After receiving the reset flag, the user space stops sending db. Signed-off-by: Yixing Liu (cherry picked from commit 8b56ab8b70262183cac5995dcb395ec099cc89a4) --- 0038-Update-kernel-headers.patch | 28 +++ ...libhns-Add-reset-stop-flow-mechanism.patch | 195 ++++++++++++++++++ rdma-core.spec | 10 +- 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 0038-Update-kernel-headers.patch create mode 100644 0039-libhns-Add-reset-stop-flow-mechanism.patch diff --git a/0038-Update-kernel-headers.patch b/0038-Update-kernel-headers.patch new file mode 100644 index 0000000..ba59cbd --- /dev/null +++ b/0038-Update-kernel-headers.patch @@ -0,0 +1,28 @@ +From 8a5429161e6932d4031ec705b695973d67729c71 Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Wed, 14 Dec 2022 16:37:26 +0800 +Subject: [PATCH rdma-core 1/2] Update kernel headers + + To commit ?? ("RDMA/hns: Kernel notify usr space to stop ring db"). + +Signed-off-by: Yixing Liu +Reviewed-by: Yangyang Li +--- + kernel-headers/rdma/hns-abi.h | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index 6950841..5988a62 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -127,6 +127,7 @@ struct hns_roce_ib_alloc_ucontext_resp { + __u32 dca_qps; + __u32 dca_mmap_size; + __aligned_u64 dca_mmap_key; ++ __aligned_u64 reset_mmap_key; + }; + + enum hns_roce_uctx_comp_mask { +-- +2.30.0 + diff --git a/0039-libhns-Add-reset-stop-flow-mechanism.patch b/0039-libhns-Add-reset-stop-flow-mechanism.patch new file mode 100644 index 0000000..4178c39 --- /dev/null +++ b/0039-libhns-Add-reset-stop-flow-mechanism.patch @@ -0,0 +1,195 @@ +From c3ee7375c80c7a8f0a943679566c87f17f87aa17 Mon Sep 17 00:00:00 2001 +From: Guofeng Yue +Date: Mon, 9 May 2022 16:03:38 +0800 +Subject: [PATCH rdma-core 2/2] libhns: Add reset stop flow mechanism + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I65WI7 + +------------------------------------------------------------------ + +Add an interface to the user space, which is used to receive +the kernel reset state. After receiving the reset flag, the +user space stops sending db. + +Signed-off-by: Yixing Liu +Signed-off-by: Guofeng Yue +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u.c | 30 ++++++++++++++++++++++++++++-- + providers/hns/hns_roce_u.h | 5 +++++ + providers/hns/hns_roce_u_db.h | 8 +++++++- + providers/hns/hns_roce_u_hw_v2.c | 19 ++++++++++++++----- + 4 files changed, 54 insertions(+), 8 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 0cf6d4b..3d29838 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -221,6 +221,24 @@ static void uninit_dca_context(struct hns_roce_context *ctx) + pthread_spin_destroy(&dca_ctx->lock); + } + ++static int init_reset_context(struct hns_roce_context *ctx, int cmd_fd, ++ struct hns_roce_alloc_ucontext_resp *resp, ++ int page_size) ++{ ++ uint64_t reset_mmap_key = resp->reset_mmap_key; ++ ++ /* The reset mmap key is 0, which means it is not supported. */ ++ if (reset_mmap_key == 0) ++ return 0; ++ ++ ctx->reset_state = mmap(NULL, page_size, PROT_READ, MAP_SHARED, ++ cmd_fd, reset_mmap_key); ++ if (ctx->reset_state == MAP_FAILED) ++ return -ENOMEM; ++ ++ return 0; ++} ++ + static int hns_roce_mmap(struct hns_roce_device *hr_dev, + struct hns_roce_context *context, int cmd_fd) + { +@@ -325,8 +343,11 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + &resp, ctx_attr, hr_dev->page_size)) + goto err_free; + ++ if (init_reset_context(context, cmd_fd, &resp, hr_dev->page_size)) ++ goto reset_free; ++ + if (hns_roce_mmap(hr_dev, context, cmd_fd)) +- goto dca_free; ++ goto uar_free; + + pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); + +@@ -335,7 +356,10 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + + return &context->ibv_ctx; + +-dca_free: ++uar_free: ++ if (context->reset_state) ++ munmap(context->reset_state, hr_dev->page_size); ++reset_free: + uninit_dca_context(context); + err_free: + verbs_uninit_context(&context->ibv_ctx); +@@ -349,6 +373,8 @@ static void hns_roce_free_context(struct ibv_context *ibctx) + struct hns_roce_context *context = to_hr_ctx(ibctx); + + munmap(context->uar, hr_dev->page_size); ++ if (context->reset_state) ++ munmap(context->reset_state, hr_dev->page_size); + uninit_dca_context(context); + verbs_uninit_context(&context->ibv_ctx); + free(context); +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 71c35c5..76c7adb 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -226,9 +226,14 @@ struct hns_roce_dca_ctx { + atomic_bitmap_t *sync_status; + }; + ++struct hns_roce_v2_reset_state { ++ uint32_t is_reset; ++}; ++ + struct hns_roce_context { + struct verbs_context ibv_ctx; + void *uar; ++ void *reset_state; + pthread_spinlock_t uar_lock; + + struct { +diff --git a/providers/hns/hns_roce_u_db.h b/providers/hns/hns_roce_u_db.h +index 8c47a53..de288de 100644 +--- a/providers/hns/hns_roce_u_db.h ++++ b/providers/hns/hns_roce_u_db.h +@@ -40,8 +40,14 @@ + + #define HNS_ROCE_WORD_NUM 2 + +-static inline void hns_roce_write64(void *dest, __le32 val[HNS_ROCE_WORD_NUM]) ++static inline void hns_roce_write64(struct hns_roce_context *ctx, void *dest, ++ __le32 val[HNS_ROCE_WORD_NUM]) + { ++ struct hns_roce_v2_reset_state *state = ctx->reset_state; ++ ++ if (state && state->is_reset) ++ return; ++ + mmio_write64_le(dest, *(__le64 *)val); + } + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 7661863..d0067d3 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -298,7 +298,8 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx, + hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB); + hr_reg_write(&rq_db, DB_PI, rq_head); + +- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&rq_db); ++ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, ++ (__le32 *)&rq_db); + } + + static void hns_roce_update_sq_db(struct hns_roce_context *ctx, +@@ -312,7 +313,7 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx, + hr_reg_write(&sq_db, DB_PI, qp->sq.head); + hr_reg_write(&sq_db, DB_SL, qp->sl); + +- hns_roce_write64(qp->sq.db_reg, (__le32 *)&sq_db); ++ hns_roce_write64(ctx, qp->sq.db_reg, (__le32 *)&sq_db); + } + + static void hns_roce_write512(uint64_t *dest, uint64_t *val) +@@ -323,6 +324,12 @@ static void hns_roce_write512(uint64_t *dest, uint64_t *val) + static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe) + { + struct hns_roce_rc_sq_wqe *rc_sq_wqe = wqe; ++ struct ibv_qp *ibvqp = &qp->verbs_qp.qp; ++ struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context); ++ struct hns_roce_v2_reset_state *state = ctx->reset_state; ++ ++ if (state && state->is_reset) ++ return; + + /* All kinds of DirectWQE have the same header field layout */ + hr_reg_enable(rc_sq_wqe, RCWQE_FLAG); +@@ -342,7 +349,8 @@ static void update_cq_db(struct hns_roce_context *ctx, struct hns_roce_cq *cq) + hr_reg_write(&cq_db, DB_CQ_CI, cq->cons_index); + hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1); + +- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db); ++ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, ++ (__le32 *)&cq_db); + } + + static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx, +@@ -857,7 +865,8 @@ static int hns_roce_u_v2_arm_cq(struct ibv_cq *ibvcq, int solicited) + hr_reg_write(&cq_db, DB_CQ_CMD_SN, cq->arm_sn); + hr_reg_write(&cq_db, DB_CQ_NOTIFY, solicited_flag); + +- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db); ++ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, ++ (__le32 *)&cq_db); + + return 0; + } +@@ -1934,7 +1943,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + + update_srq_db(&srq_db, srq); + +- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, ++ hns_roce_write64(ctx, ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, + (__le32 *)&srq_db); + } + +-- +2.30.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 971ad45..fc74128 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 7 +Release: 8 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -43,6 +43,8 @@ Patch33: 0034-libhns-Add-support-for-attaching-QP-s-WQE-buffer.patch Patch34: 0035-libhns-Use-shared-memory-to-sync-DCA-status.patch Patch35: 0036-libhns-Sync-DCA-status-by-shared-memory.patch Patch36: 0037-libhns-Add-direct-verbs-support-to-config-DCA.patch +Patch37: 0038-Update-kernel-headers.patch +Patch38: 0039-libhns-Add-reset-stop-flow-mechanism.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -290,6 +292,12 @@ fi %{_mandir}/* %changelog +* Wed Dec 14 2022 Yixing Liu - 41.0-6 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Support libhns reset stop ring db mechanism + * Wed Nov 30 2022 tangchengchang - 41.0-7 - Type: requirement - ID: NA -- Gitee