diff --git a/0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch b/0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch new file mode 100644 index 0000000000000000000000000000000000000000..13b0a9fc4b598ae320af1b30a90ac7ea9fd6b6c5 --- /dev/null +++ b/0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch @@ -0,0 +1,178 @@ +From fb07638be493fe0667e3fa0554ee434b7d6b3502 Mon Sep 17 00:00:00 2001 +From: Junxian Huang +Date: Fri, 9 Jun 2023 11:04:41 +0800 +Subject: [PATCH] libhns: Support user to choose using UD sl or pktype to adapt + MPI APP + +driver inclusion +category: bugfix +bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I7A5YM + +--------------------------------------------------------------- + +According to Annex17_RoCEv2 (A17.4.5.2), for RoCEv2 UD, a CQE should +carry a flag that indicates if the received frame is an IPv4, IPv6 or +RoCE packet. But currently, the values of the flag corresponding to +these packet types haven't been defined yet in WC. + +In UCX, 'sl' in ibv_wc for UD is used as the packet type flag, and the +packet type values have already been defined in the UCX patch of +https://github.com/openucx/ucx/commit/ed28845b88633e65d64fce8ec880060aa61bd59c + +Therefore, to adapt UCX, add a create flag to hnsdv_create_qp() to allow +users to choose whether they use 'sl' in ibv_wc as service level or +packet type for UD. For the latter, obtain and translate the packet type +from CQE and fill it to 'sl' in ibv_wc. + +Signed-off-by: Junxian Huang +Signed-off-by: Haoyue Xu +--- + providers/hns/hns_roce_u.h | 11 +++++++++++ + providers/hns/hns_roce_u_hw_v2.c | 32 +++++++++++++++++++++++++++++--- + providers/hns/hns_roce_u_verbs.c | 16 ++++++++++++++++ + providers/hns/hnsdv.h | 1 + + 4 files changed, 57 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 41e9599..e3012e1 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -195,6 +195,12 @@ enum hns_roce_tc_map_mode { + HNS_ROCE_TC_MAP_MODE_DSCP, + }; + ++enum hns_roce_pktype { ++ HNS_ROCE_PKTYPE_ROCE_V1, ++ HNS_ROCE_PKTYPE_ROCE_V2_IPV6, ++ HNS_ROCE_PKTYPE_ROCE_V2_IPV4, ++}; ++ + struct hns_roce_db_page { + struct hns_roce_db_page *prev, *next; + struct hns_roce_buf buf; +@@ -406,6 +412,11 @@ struct hns_roce_qp { + void *cur_wqe; + unsigned int rb_sq_head; /* roll back sq head */ + struct hns_roce_sge_info sge_info; ++ ++ /* Just for UD. If not enabled, 'sl' in ibv_wc ++ * will be filled with 'port_type' in cqe. ++ */ ++ bool enable_ud_sl; + }; + + struct hns_roce_av { +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 4b0ec5c..ee2fffe 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -522,9 +522,24 @@ static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe, + handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, wqe_buf); + } + +-static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc) ++static const uint8_t pktype_for_ud[] = { ++ HNS_ROCE_PKTYPE_ROCE_V1, ++ HNS_ROCE_PKTYPE_ROCE_V2_IPV4, ++ HNS_ROCE_PKTYPE_ROCE_V2_IPV6 ++}; ++ ++static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, ++ bool enable_ud_sl) + { +- wc->sl = hr_reg_read(cqe, CQE_SL); ++ uint8_t port_type; ++ ++ if (enable_ud_sl) { ++ wc->sl = hr_reg_read(cqe, CQE_SL); ++ } else { ++ port_type = hr_reg_read(cqe, CQE_PORT_TYPE); ++ wc->sl = pktype_for_ud[port_type]; ++ } ++ + wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN); + wc->slid = 0; + wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IBV_WC_GRH : 0; +@@ -554,7 +569,7 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + ++wq->tail; + + if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) +- parse_for_ud_qp(cqe, wc); ++ parse_for_ud_qp(cqe, wc, hr_qp->enable_ud_sl); + + if (hr_reg_read(cqe, CQE_CQE_INLINE)) + handle_recv_cqe_inl_from_rq(cqe, hr_qp); +@@ -2126,6 +2141,17 @@ static uint32_t wc_read_slid(struct ibv_cq_ex *current) + static uint8_t wc_read_sl(struct ibv_cq_ex *current) + { + struct hns_roce_cq *cq = to_hr_cq(ibv_cq_ex_to_cq(current)); ++ struct hns_roce_context *ctx = to_hr_ctx(current->context); ++ struct hns_roce_qp *hr_qp; ++ uint8_t port_type; ++ uint32_t qpn; ++ ++ qpn = hr_reg_read(cq->cqe, CQE_LCL_QPN); ++ hr_qp = hns_roce_v2_find_qp(ctx, qpn); ++ if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD && !hr_qp->enable_ud_sl) { ++ port_type = hr_reg_read(cq->cqe, CQE_PORT_TYPE); ++ return pktype_for_ud[port_type]; ++ } + + return (uint8_t)hr_reg_read(cq->cqe, CQE_SL); + } +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index fa27fc1..4b641ea 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1019,6 +1019,11 @@ static int check_qp_congest_type(struct hns_roce_context *ctx, + return 0; + } + ++enum { ++ HNSDV_QP_SUP_CREATE_FLAGS = HNSDV_QP_CREATE_ENABLE_DCA_MODE | ++ HNSDV_QP_CREATE_ENABLE_UD_SL, ++}; ++ + static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, + struct hnsdv_qp_init_attr *hns_attr, + struct hns_roce_cmd_flag *cmd_flag) +@@ -1034,6 +1039,14 @@ static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, + return -EINVAL; + } + ++ if (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS && ++ !check_comp_mask(hns_attr->create_flags, ++ HNSDV_QP_SUP_CREATE_FLAGS)) { ++ verbs_err(&ctx->ibv_ctx, "invalid create_flags 0x%x.\n", ++ hns_attr->create_flags); ++ return -EOPNOTSUPP; ++ } ++ + ret = check_qp_congest_type(ctx, hns_attr, cmd_flag); + if (ret) + return ret; +@@ -1685,6 +1698,9 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + + qp_setup_config(attr, qp, context); + ++ if (hns_attr && hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_UD_SL) ++ qp->enable_ud_sl = true; ++ + return &qp->verbs_qp.qp; + + err_dwqe: +diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h +index e15b428..365c314 100644 +--- a/providers/hns/hnsdv.h ++++ b/providers/hns/hnsdv.h +@@ -43,6 +43,7 @@ struct ibv_context *hnsdv_open_device(struct ibv_device *device, + + enum hnsdv_qp_create_flags { + HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0, ++ HNSDV_QP_CREATE_ENABLE_UD_SL = 1 << 1, + }; + + enum hnsdv_qp_congest_ctrl_type { +-- +2.25.1 + diff --git a/rdma-core.spec b/rdma-core.spec index e94c596d20ae88478ccb679583ce256e87d19097..07b220ae0f22f5acb7fe37eb0c8e89bad88478e5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 12 +Release: 13 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -55,6 +55,7 @@ Patch45: 0046-libhns-Disable-local-invalidate-operation.patch Patch46: 0047-libhns-Fix-the-owner-bit-error-of-sq-in-new-io.patch Patch47: 0048-libhns-Fix-incorrect-post-send-with-direct-wqe-of-wr.patch Patch48: 0049-libhns-Add-a-judgment-to-the-congestion-control-algo.patch +Patch49: 0050-libhns-Support-user-to-choose-using-UD-sl-or-pktype-.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -302,6 +303,12 @@ fi %{_mandir}/* %changelog +* Fri Jun 9 2023 Juan Zhou - 41.0-13 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: Support user to choose using UD sl or pktype to adapt MPI APP + * Fri Jun 2 2023 Juan Zhou - 41.0-12 - Type: bugfix - ID: NA