From 9f33bcde0c785026970f7040097f90d58db9ca58 Mon Sep 17 00:00:00 2001 From: Luoyouming Date: Mon, 31 Oct 2022 14:17:19 +0800 Subject: [PATCH 1/2] Bugfix for sge num and support inline feature Fix sge num bug, add compatibility for rq inline, support cqe inline Signed-off-by: Luoyouming --- ...constant-instead-of-sizeof-operation.patch | 45 +++ ...Fix-ext_sge-num-error-when-post-send.patch | 53 +++ 0003-Update-kernel-headers.patch | 46 +++ 0004-libhns-Fix-the-problem-of-sge-nums.patch | 254 +++++++++++++++ 0005-Update-kernel-headers.patch | 33 ++ ...compatibility-handling-for-rq-inline.patch | 46 +++ 0007-libhns-Refactor-rq-inline.patch | 301 ++++++++++++++++++ ...nline-support-wc_x_poll_cq-interface.patch | 46 +++ 0009-Update-kernel-headers.patch | 33 ++ 0010-libhns-Support-cqe-inline.patch | 263 +++++++++++++++ rdma-core.spec | 19 +- 11 files changed, 1138 insertions(+), 1 deletion(-) create mode 100644 0001-libhns-Use-a-constant-instead-of-sizeof-operation.patch create mode 100644 0002-libhns-Fix-ext_sge-num-error-when-post-send.patch create mode 100644 0003-Update-kernel-headers.patch create mode 100644 0004-libhns-Fix-the-problem-of-sge-nums.patch create mode 100644 0005-Update-kernel-headers.patch create mode 100644 0006-libhns-Add-compatibility-handling-for-rq-inline.patch create mode 100644 0007-libhns-Refactor-rq-inline.patch create mode 100644 0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch create mode 100644 0009-Update-kernel-headers.patch create mode 100644 0010-libhns-Support-cqe-inline.patch diff --git a/0001-libhns-Use-a-constant-instead-of-sizeof-operation.patch b/0001-libhns-Use-a-constant-instead-of-sizeof-operation.patch new file mode 100644 index 0000000..1c34c3b --- /dev/null +++ b/0001-libhns-Use-a-constant-instead-of-sizeof-operation.patch @@ -0,0 +1,45 @@ +From 0de1678211e710c2cd33e3aea98b1271cae9bd98 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Tue, 20 Sep 2022 11:47:45 +0800 +Subject: [PATCH v4 01/10] libhns: Use a constant instead of sizeof operation + +The sge size is known to be constant, so it's unnecessary to use sizeof to +calculate. + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u_hw_v2.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 0169250..d9ea18e 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -847,13 +847,12 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + uint32_t num_buf, + enum hns_roce_wr_buf_type buf_type) + { +- unsigned int sge_sz = sizeof(struct hns_roce_v2_wqe_data_seg); + unsigned int sge_mask = qp->ex_sge.sge_cnt - 1; + void *dst_addr, *src_addr, *tail_bound_addr; + uint32_t src_len, tail_len; + int i; + +- if (sge_info->total_len > qp->sq.max_gs * sge_sz) ++ if (sge_info->total_len > qp->sq.max_gs * HNS_ROCE_SGE_SIZE) + return EINVAL; + + dst_addr = get_send_sge_ex(qp, sge_info->start_idx & sge_mask); +@@ -880,7 +879,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + } + } + +- sge_info->valid_num = DIV_ROUND_UP(sge_info->total_len, sge_sz); ++ sge_info->valid_num = DIV_ROUND_UP(sge_info->total_len, HNS_ROCE_SGE_SIZE); + sge_info->start_idx += sge_info->valid_num; + + return 0; +-- +2.30.0 + diff --git a/0002-libhns-Fix-ext_sge-num-error-when-post-send.patch b/0002-libhns-Fix-ext_sge-num-error-when-post-send.patch new file mode 100644 index 0000000..a7ca2c8 --- /dev/null +++ b/0002-libhns-Fix-ext_sge-num-error-when-post-send.patch @@ -0,0 +1,53 @@ +From a57d5dfbc2701b9d0c47eb70a1bb82b16170a7d2 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Tue, 20 Sep 2022 11:53:18 +0800 +Subject: [PATCH v4 02/10] libhns: Fix ext_sge num error when post send + +The max_gs is the sum of extended sge and standard sge. In function +fill_ext_sge_inl_data, max_gs does not subtract the number of extended +sges, but is directly used to calculate the size of extended sges. + +Fixes:b7814b7b9715("libhns: Support inline data in extented sge space for RC") + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u_hw_v2.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index d9ea18e..bb4298f 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -841,6 +841,14 @@ static void get_src_buf_info(void **src_addr, uint32_t *src_len, + } + } + ++static unsigned int get_std_sge_num(struct hns_roce_qp *qp) ++{ ++ if (qp->verbs_qp.qp.qp_type == IBV_QPT_UD) ++ return 0; ++ ++ return HNS_ROCE_SGE_IN_WQE; ++} ++ + static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + struct hns_roce_sge_info *sge_info, + const void *buf_list, +@@ -850,9 +858,12 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + unsigned int sge_mask = qp->ex_sge.sge_cnt - 1; + void *dst_addr, *src_addr, *tail_bound_addr; + uint32_t src_len, tail_len; ++ unsigned int std_sge_num; + int i; + +- if (sge_info->total_len > qp->sq.max_gs * HNS_ROCE_SGE_SIZE) ++ std_sge_num = get_std_sge_num(qp); ++ if (sge_info->total_len > ++ (qp->sq.max_gs - std_sge_num) * HNS_ROCE_SGE_SIZE) + return EINVAL; + + dst_addr = get_send_sge_ex(qp, sge_info->start_idx & sge_mask); +-- +2.30.0 + diff --git a/0003-Update-kernel-headers.patch b/0003-Update-kernel-headers.patch new file mode 100644 index 0000000..616b58c --- /dev/null +++ b/0003-Update-kernel-headers.patch @@ -0,0 +1,46 @@ +From f20816984da80e2fe9a82b3b330f85150763243e Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Mon, 26 Sep 2022 11:14:05 +0800 +Subject: [PATCH v4 03/10] Update kernel headers + +To commit ?? ("RDMA/hns: Fix the problem of sge nums"). + +Signed-off-by: Luoyouming +--- + kernel-headers/rdma/hns-abi.h | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index f6fde06..fc83dfb 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -85,11 +85,26 @@ struct hns_roce_ib_create_qp_resp { + __aligned_u64 dwqe_mmap_key; + }; + ++enum { ++ HNS_ROCE_EXSGE_FLAGS = 1 << 0, ++}; ++ ++enum { ++ HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0, ++}; ++ + struct hns_roce_ib_alloc_ucontext_resp { + __u32 qp_tab_size; + __u32 cqe_size; + __u32 srq_tab_size; + __u32 reserved; ++ __u32 config; ++ __u32 max_inline_data; ++}; ++ ++struct hns_roce_ib_alloc_ucontext { ++ __u32 config; ++ __u32 reserved; + }; + + struct hns_roce_ib_alloc_pd_resp { +-- +2.30.0 + diff --git a/0004-libhns-Fix-the-problem-of-sge-nums.patch b/0004-libhns-Fix-the-problem-of-sge-nums.patch new file mode 100644 index 0000000..ada5774 --- /dev/null +++ b/0004-libhns-Fix-the-problem-of-sge-nums.patch @@ -0,0 +1,254 @@ +From 448d82b2c62f09f1dd9c8045d34623dedef1c111 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Fri, 19 Nov 2021 20:21:21 +0800 +Subject: [PATCH v4 04/10] libhns: Fix the problem of sge nums + +Currently, the driver only uses max_send_sge to initialize sge num +when creating_qp. So, in the sq inline scenario, the driver may not +has enough sge to send data. For example, if max_send_sge is 16 and +max_inline_data is 1024, the driver needs 1024/16=64 sge to send data. +Therefore, the calculation method of sge num is modified to take the +maximum value of max_send_sge and max_inline_data/16 to solve this +problem. + +Fixes:11c81d0e3a98("libhns: Refactor process of setting extended sge") +Fixes:b7814b7b9715("libhns: Support inline data in extented sge space for RC") + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u.c | 9 +++- + providers/hns/hns_roce_u.h | 3 ++ + providers/hns/hns_roce_u_abi.h | 2 +- + providers/hns/hns_roce_u_hw_v2.c | 13 +---- + providers/hns/hns_roce_u_verbs.c | 84 ++++++++++++++++++++++++-------- + 5 files changed, 77 insertions(+), 34 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index a46ceb9..1bd5bb1 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -103,9 +103,9 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + { + struct hns_roce_device *hr_dev = to_hr_dev(ibdev); + struct hns_roce_alloc_ucontext_resp resp = {}; ++ struct hns_roce_alloc_ucontext cmd = {}; + struct ibv_device_attr dev_attrs; + struct hns_roce_context *context; +- struct ibv_get_context cmd; + int i; + + context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, +@@ -113,7 +113,8 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + if (!context) + return NULL; + +- if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof(cmd), ++ cmd.config |= HNS_ROCE_EXSGE_FLAGS; ++ if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp))) + goto err_free; + +@@ -124,6 +125,10 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + else + context->cqe_size = HNS_ROCE_V3_CQE_SIZE; + ++ context->config = resp.config; ++ if (resp.config & HNS_ROCE_RSP_EXSGE_FLAGS) ++ context->max_inline_data = resp.max_inline_data; ++ + context->qp_table_shift = calc_table_shift(resp.qp_tab_size, + HNS_ROCE_QP_TABLE_BITS); + context->qp_table_mask = (1 << context->qp_table_shift) - 1; +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 5d90634..5388f9c 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -213,6 +213,8 @@ struct hns_roce_context { + unsigned int max_srq_sge; + int max_cqe; + unsigned int cqe_size; ++ uint32_t config; ++ unsigned int max_inline_data; + }; + + struct hns_roce_pd { +@@ -267,6 +269,7 @@ struct hns_roce_wq { + unsigned int head; + unsigned int tail; + unsigned int max_gs; ++ unsigned int ext_sge_cnt; + unsigned int rsv_sge; + unsigned int wqe_shift; + unsigned int shift; /* wq size is 2^shift */ +diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h +index 333f977..2753d30 100644 +--- a/providers/hns/hns_roce_u_abi.h ++++ b/providers/hns/hns_roce_u_abi.h +@@ -47,7 +47,7 @@ DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, + hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp); + + DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, +- empty, hns_roce_ib_alloc_ucontext_resp); ++ hns_roce_ib_alloc_ucontext, hns_roce_ib_alloc_ucontext_resp); + + DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP, + hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp); +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index bb4298f..ebe68bc 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -841,14 +841,6 @@ static void get_src_buf_info(void **src_addr, uint32_t *src_len, + } + } + +-static unsigned int get_std_sge_num(struct hns_roce_qp *qp) +-{ +- if (qp->verbs_qp.qp.qp_type == IBV_QPT_UD) +- return 0; +- +- return HNS_ROCE_SGE_IN_WQE; +-} +- + static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + struct hns_roce_sge_info *sge_info, + const void *buf_list, +@@ -858,12 +850,9 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, + unsigned int sge_mask = qp->ex_sge.sge_cnt - 1; + void *dst_addr, *src_addr, *tail_bound_addr; + uint32_t src_len, tail_len; +- unsigned int std_sge_num; + int i; + +- std_sge_num = get_std_sge_num(qp); +- if (sge_info->total_len > +- (qp->sq.max_gs - std_sge_num) * HNS_ROCE_SGE_SIZE) ++ if (sge_info->total_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) + return EINVAL; + + dst_addr = get_send_sge_ex(qp, sge_info->start_idx & sge_mask); +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index ba7f2ae..851b145 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -978,41 +978,88 @@ err_alloc: + return -ENOMEM; + } + +-static unsigned int get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) ++/** ++ * Calculated sge num according to attr's max_send_sge ++ */ ++static unsigned int get_sge_num_from_max_send_sge(bool is_ud, ++ uint32_t max_send_sge) + { +- if (qp->verbs_qp.qp.qp_type == IBV_QPT_UD) +- return qp->sq.max_gs; ++ unsigned int std_sge_num; ++ unsigned int min_sge; + +- if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) +- return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; ++ std_sge_num = is_ud ? 0 : HNS_ROCE_SGE_IN_WQE; ++ min_sge = is_ud ? 1 : 0; ++ return max_send_sge > std_sge_num ? (max_send_sge - std_sge_num) : ++ min_sge; ++} + +- return 0; ++/** ++ * Calculated sge num according to attr's max_inline_data ++ */ ++static unsigned int get_sge_num_from_max_inl_data(bool is_ud, ++ uint32_t max_inline_data) ++{ ++ unsigned int inline_sge = 0; ++ ++ inline_sge = max_inline_data / HNS_ROCE_SGE_SIZE; ++ /* ++ * if max_inline_data less than ++ * HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE, ++ * In addition to ud's mode, no need to extend sge. ++ */ ++ if (!is_ud && (inline_sge <= HNS_ROCE_SGE_IN_WQE)) ++ inline_sge = 0; ++ ++ return inline_sge; + } + +-static void set_ext_sge_param(struct hns_roce_device *hr_dev, ++static void set_ext_sge_param(struct hns_roce_context *ctx, + struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, unsigned int wr_cnt) + { ++ bool is_ud = (qp->verbs_qp.qp.qp_type == IBV_QPT_UD); ++ unsigned int ext_wqe_sge_cnt; ++ unsigned int inline_ext_sge; + unsigned int total_sge_cnt; +- unsigned int wqe_sge_cnt; ++ unsigned int std_sge_num; + + qp->ex_sge.sge_shift = HNS_ROCE_SGE_SHIFT; +- +- qp->sq.max_gs = attr->cap.max_send_sge; +- +- wqe_sge_cnt = get_wqe_ext_sge_cnt(qp); ++ std_sge_num = is_ud ? 0 : HNS_ROCE_SGE_IN_WQE; ++ ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud, ++ attr->cap.max_send_sge); ++ ++ if (ctx->config & HNS_ROCE_RSP_EXSGE_FLAGS) { ++ attr->cap.max_inline_data = min_t(uint32_t, roundup_pow_of_two( ++ attr->cap.max_inline_data), ++ ctx->max_inline_data); ++ ++ inline_ext_sge = max(ext_wqe_sge_cnt, ++ get_sge_num_from_max_inl_data(is_ud, ++ attr->cap.max_inline_data)); ++ qp->sq.ext_sge_cnt = inline_ext_sge ? ++ roundup_pow_of_two(inline_ext_sge) : 0; ++ qp->sq.max_gs = min((qp->sq.ext_sge_cnt + std_sge_num), ++ ctx->max_sge); ++ ++ ext_wqe_sge_cnt = qp->sq.ext_sge_cnt; ++ } else { ++ qp->sq.max_gs = max(1U, attr->cap.max_send_sge); ++ qp->sq.max_gs = min(qp->sq.max_gs, ctx->max_sge); ++ qp->sq.ext_sge_cnt = qp->sq.max_gs; ++ } + + /* If the number of extended sge is not zero, they MUST use the + * space of HNS_HW_PAGE_SIZE at least. + */ +- if (wqe_sge_cnt) { +- total_sge_cnt = roundup_pow_of_two(wr_cnt * wqe_sge_cnt); +- qp->ex_sge.sge_cnt = +- max(total_sge_cnt, +- (unsigned int)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); ++ if (ext_wqe_sge_cnt) { ++ total_sge_cnt = roundup_pow_of_two(wr_cnt * ext_wqe_sge_cnt); ++ qp->ex_sge.sge_cnt = max(total_sge_cnt, ++ (unsigned int)HNS_HW_PAGE_SIZE / ++ HNS_ROCE_SGE_SIZE); + } + } + ++ + static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, + struct hns_roce_context *ctx) +@@ -1044,10 +1091,9 @@ static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr, + qp->sq.wqe_cnt = cnt; + qp->sq.shift = hr_ilog32(cnt); + +- set_ext_sge_param(hr_dev, attr, qp, cnt); ++ set_ext_sge_param(ctx, attr, qp, cnt); + + qp->sq.max_post = min(ctx->max_qp_wr, cnt); +- qp->sq.max_gs = min(ctx->max_sge, qp->sq.max_gs); + + qp->sq_signal_bits = attr->sq_sig_all ? 0 : 1; + +-- +2.30.0 + diff --git a/0005-Update-kernel-headers.patch b/0005-Update-kernel-headers.patch new file mode 100644 index 0000000..7d68e63 --- /dev/null +++ b/0005-Update-kernel-headers.patch @@ -0,0 +1,33 @@ +From 542b54285dbaebbe0b5eb3279134b02484d7329d Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Mon, 10 Oct 2022 21:21:55 +0800 +Subject: [PATCH v4 05/10] Update kernel headers + +To commit ?? ("RDMA/hns: Remove enable rq inline in kernel and add +compatibility handling"). + +Signed-off-by: Luoyouming +--- + kernel-headers/rdma/hns-abi.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index fc83dfb..c70465d 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -87,10 +87,12 @@ struct hns_roce_ib_create_qp_resp { + + enum { + HNS_ROCE_EXSGE_FLAGS = 1 << 0, ++ HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, + }; + + enum { + HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0, ++ HNS_ROCE_RSP_RQ_INLINE_FLAGS = 1 << 1, + }; + + struct hns_roce_ib_alloc_ucontext_resp { +-- +2.30.0 + diff --git a/0006-libhns-Add-compatibility-handling-for-rq-inline.patch b/0006-libhns-Add-compatibility-handling-for-rq-inline.patch new file mode 100644 index 0000000..9854b87 --- /dev/null +++ b/0006-libhns-Add-compatibility-handling-for-rq-inline.patch @@ -0,0 +1,46 @@ +From 996bca51e2063dc790286cbc894e2c438f499441 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Mon, 10 Oct 2022 21:49:35 +0800 +Subject: [PATCH v4 06/10] libhns: Add compatibility handling for rq inline + +Add compatibility processing between different user space +and kernel space. + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u.c | 2 +- + providers/hns/hns_roce_u_verbs.c | 4 +++- + 2 files changed, 4 insertions(+), 2 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 1bd5bb1..6c9aefa 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -113,7 +113,7 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + if (!context) + return NULL; + +- cmd.config |= HNS_ROCE_EXSGE_FLAGS; ++ cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS; + if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp))) + goto err_free; +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 851b145..3e9a306 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1079,7 +1079,9 @@ static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr, + cnt = roundup_pow_of_two(attr->cap.max_recv_wr); + qp->rq.wqe_cnt = cnt; + qp->rq.shift = hr_ilog32(cnt); +- qp->rq_rinl_buf.wqe_cnt = cnt; ++ qp->rq_rinl_buf.wqe_cnt = 0; ++ if (ctx->config & HNS_ROCE_RSP_RQ_INLINE_FLAGS) ++ qp->rq_rinl_buf.wqe_cnt = cnt; + + attr->cap.max_recv_wr = qp->rq.wqe_cnt; + attr->cap.max_recv_sge = qp->rq.max_gs; +-- +2.30.0 + diff --git a/0007-libhns-Refactor-rq-inline.patch b/0007-libhns-Refactor-rq-inline.patch new file mode 100644 index 0000000..00de21f --- /dev/null +++ b/0007-libhns-Refactor-rq-inline.patch @@ -0,0 +1,301 @@ +From 9e5f5d39757a5479a1a4e1170978d2e09acb995b Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Fri, 9 Sep 2022 17:42:38 +0800 +Subject: [PATCH v4 07/10] libhns: Refactor rq inline + +The ibv_sge struct is enough, there is no need to customize the +hns_roce_rinl_sge struct. Refactored structures and functions +for reuse in cqe inline (rq, srq scenarios). + +Signed-off-by: Luoyouming +--- + providers/hns/hns_roce_u.h | 7 +-- + providers/hns/hns_roce_u_hw_v2.c | 103 +++++++++++++++---------------- + providers/hns/hns_roce_u_verbs.c | 46 +++++++------- + 3 files changed, 75 insertions(+), 81 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 5388f9c..57ebe55 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -290,13 +290,8 @@ struct hns_roce_sge_ex { + unsigned int sge_shift; + }; + +-struct hns_roce_rinl_sge { +- void *addr; +- unsigned int len; +-}; +- + struct hns_roce_rinl_wqe { +- struct hns_roce_rinl_sge *sg_list; ++ struct ibv_sge *sg_list; + unsigned int sge_cnt; + }; + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index ebe68bc..73acc9e 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -34,6 +34,7 @@ + #include + #include + #include ++#include + #include "hns_roce_u.h" + #include "hns_roce_u_db.h" + #include "hns_roce_u_hw_v2.h" +@@ -417,46 +418,42 @@ static void get_opcode_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + wc->opcode = wc_rcv_op_map[opcode]; + } + +-static int handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, +- struct hns_roce_qp **cur_qp, uint32_t opcode) ++static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe, ++ struct hns_roce_rinl_buf *rinl_buf, ++ uint32_t wr_cnt, uint8_t *buf) + { +- if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC) && +- (opcode == HNS_ROCE_RECV_OP_SEND || +- opcode == HNS_ROCE_RECV_OP_SEND_WITH_IMM || +- opcode == HNS_ROCE_RECV_OP_SEND_WITH_INV) && +- hr_reg_read(cqe, CQE_RQ_INLINE)) { +- struct hns_roce_rinl_sge *sge_list; +- uint32_t wr_num, wr_cnt, sge_num, data_len; +- uint8_t *wqe_buf; +- uint32_t sge_cnt, size; ++ struct ibv_sge *sge_list; ++ uint32_t sge_num, data_len; ++ uint32_t sge_cnt, size; + +- wr_num = hr_reg_read(cqe, CQE_WQE_IDX); +- wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1); ++ sge_list = rinl_buf->wqe_list[wr_cnt].sg_list; ++ sge_num = rinl_buf->wqe_list[wr_cnt].sge_cnt; + +- sge_list = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sg_list; +- sge_num = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sge_cnt; +- wqe_buf = (uint8_t *)get_recv_wqe_v2(*cur_qp, wr_cnt); ++ data_len = le32toh(cqe->byte_cnt); + +- data_len = wc->byte_len; ++ for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { ++ size = min(sge_list[sge_cnt].length, data_len); + +- for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); +- sge_cnt++) { +- size = sge_list[sge_cnt].len < data_len ? +- sge_list[sge_cnt].len : data_len; ++ memcpy((void *)(uintptr_t)sge_list[sge_cnt].addr, (void *)buf, size); ++ data_len -= size; ++ buf += size; ++ } + +- memcpy((void *)sge_list[sge_cnt].addr, +- (void *)wqe_buf, size); +- data_len -= size; +- wqe_buf += size; +- } ++ if (data_len) ++ hr_reg_write(cqe, CQE_STATUS, HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR); + +- if (data_len) { +- wc->status = IBV_WC_LOC_LEN_ERR; +- return V2_CQ_POLL_ERR; +- } +- } ++} + +- return V2_CQ_OK; ++static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe, ++ struct hns_roce_qp *cur_qp) ++{ ++ uint8_t *wqe_buf; ++ uint32_t wr_num; ++ ++ wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1); ++ ++ wqe_buf = (uint8_t *)get_recv_wqe_v2(cur_qp, wr_num); ++ handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, wqe_buf); + } + + static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc) +@@ -479,10 +476,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + } + + static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, +- struct hns_roce_qp *hr_qp, uint8_t opcode) ++ struct hns_roce_qp *hr_qp) + { + struct hns_roce_wq *wq; +- int ret; + + wq = &hr_qp->rq; + wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; +@@ -491,12 +487,8 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) + parse_for_ud_qp(cqe, wc); + +- ret = handle_recv_inl_wqe(cqe, wc, &hr_qp, opcode); +- if (ret) { +- verbs_err(verbs_get_ctx(hr_qp->verbs_qp.qp.context), +- PFX "failed to handle recv inline wqe!\n"); +- return ret; +- } ++ if (hr_reg_read(cqe, CQE_RQ_INLINE)) ++ handle_recv_rq_inl(cqe, hr_qp); + + return 0; + } +@@ -626,7 +618,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq + if (srq) + parse_cqe_for_srq(cqe, wc, srq); + else +- parse_cqe_for_resp(cqe, wc, cur_qp, opcode); ++ parse_cqe_for_resp(cqe, wc, cur_qp); + } + + return 0; +@@ -1355,26 +1347,31 @@ static void fill_recv_sge_to_wqe(struct ibv_recv_wr *wr, void *wqe, + } + } + ++static void fill_recv_inl_buf(struct hns_roce_rinl_buf *rinl_buf, ++ unsigned int wqe_idx, struct ibv_recv_wr *wr) ++{ ++ struct ibv_sge *sge_list; ++ unsigned int i; ++ ++ if (!rinl_buf->wqe_cnt) ++ return; ++ ++ sge_list = rinl_buf->wqe_list[wqe_idx].sg_list; ++ rinl_buf->wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge; ++ for (i = 0; i < wr->num_sge; i++) ++ memcpy((void *)&sge_list[i], (void *)&wr->sg_list[i], ++ sizeof(struct ibv_sge)); ++} ++ + static void fill_rq_wqe(struct hns_roce_qp *qp, struct ibv_recv_wr *wr, + unsigned int wqe_idx, unsigned int max_sge) + { +- struct hns_roce_rinl_sge *sge_list; +- unsigned int i; + void *wqe; + + wqe = get_recv_wqe_v2(qp, wqe_idx); + fill_recv_sge_to_wqe(wr, wqe, max_sge, qp->rq.rsv_sge); + +- if (!qp->rq_rinl_buf.wqe_cnt) +- return; +- +- /* QP support receive inline wqe */ +- sge_list = qp->rq_rinl_buf.wqe_list[wqe_idx].sg_list; +- qp->rq_rinl_buf.wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge; +- for (i = 0; i < wr->num_sge; i++) { +- sge_list[i].addr = (void *)(uintptr_t)wr->sg_list[i].addr; +- sge_list[i].len = wr->sg_list[i].length; +- } ++ fill_recv_inl_buf(&qp->rq_rinl_buf, wqe_idx, wr); + } + + static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 3e9a306..1d661dd 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -855,43 +855,45 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, + return verify_qp_create_cap(ctx, attr); + } + +-static int qp_alloc_recv_inl_buf(struct ibv_qp_cap *cap, +- struct hns_roce_qp *qp) ++static int alloc_recv_rinl_buf(uint32_t max_sge, ++ struct hns_roce_rinl_buf *rinl_buf) + { + unsigned int cnt; + int i; + +- cnt = qp->rq_rinl_buf.wqe_cnt; +- qp->rq_rinl_buf.wqe_list = calloc(cnt, +- sizeof(struct hns_roce_rinl_wqe)); +- if (!qp->rq_rinl_buf.wqe_list) ++ cnt = rinl_buf->wqe_cnt; ++ rinl_buf->wqe_list = calloc(cnt, ++ sizeof(struct hns_roce_rinl_wqe)); ++ if (!rinl_buf->wqe_list) + return ENOMEM; + +- qp->rq_rinl_buf.wqe_list[0].sg_list = calloc(cnt * cap->max_recv_sge, +- sizeof(struct hns_roce_rinl_sge)); +- if (!qp->rq_rinl_buf.wqe_list[0].sg_list) ++ rinl_buf->wqe_list[0].sg_list = calloc(cnt * max_sge, ++ sizeof(struct ibv_sge)); ++ if (!rinl_buf->wqe_list[0].sg_list) { ++ free(rinl_buf->wqe_list); + return ENOMEM; ++ } + + for (i = 0; i < cnt; i++) { +- int wqe_size = i * cap->max_recv_sge; ++ int wqe_size = i * max_sge; + +- qp->rq_rinl_buf.wqe_list[i].sg_list = +- &(qp->rq_rinl_buf.wqe_list[0].sg_list[wqe_size]); ++ rinl_buf->wqe_list[i].sg_list = ++ &(rinl_buf->wqe_list[0].sg_list[wqe_size]); + } + + return 0; + } + +-static void qp_free_recv_inl_buf(struct hns_roce_qp *qp) ++static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf) + { +- if (qp->rq_rinl_buf.wqe_list) { +- if (qp->rq_rinl_buf.wqe_list[0].sg_list) { +- free(qp->rq_rinl_buf.wqe_list[0].sg_list); +- qp->rq_rinl_buf.wqe_list[0].sg_list = NULL; ++ if (rinl_buf->wqe_list) { ++ if (rinl_buf->wqe_list[0].sg_list) { ++ free(rinl_buf->wqe_list[0].sg_list); ++ rinl_buf->wqe_list[0].sg_list = NULL; + } + +- free(qp->rq_rinl_buf.wqe_list); +- qp->rq_rinl_buf.wqe_list = NULL; ++ free(rinl_buf->wqe_list); ++ rinl_buf->wqe_list = NULL; + } + } + +@@ -930,7 +932,7 @@ static int calc_qp_buff_size(struct hns_roce_device *hr_dev, + + static void qp_free_wqe(struct hns_roce_qp *qp) + { +- qp_free_recv_inl_buf(qp); ++ free_recv_rinl_buf(&qp->rq_rinl_buf); + if (qp->sq.wqe_cnt) + free(qp->sq.wrid); + +@@ -958,7 +960,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp, + } + + if (qp->rq_rinl_buf.wqe_cnt) { +- if (qp_alloc_recv_inl_buf(cap, qp)) ++ if (alloc_recv_rinl_buf(cap->max_recv_sge, &qp->rq_rinl_buf)) + goto err_alloc; + } + +@@ -968,7 +970,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp, + return 0; + + err_alloc: +- qp_free_recv_inl_buf(qp); ++ free_recv_rinl_buf(&qp->rq_rinl_buf); + if (qp->rq.wrid) + free(qp->rq.wrid); + +-- +2.30.0 + diff --git a/0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch b/0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch new file mode 100644 index 0000000..aa9e57e --- /dev/null +++ b/0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch @@ -0,0 +1,46 @@ +From 22beeec9a0d8272fc4db60275ee4eee890068102 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Tue, 11 Oct 2022 10:21:24 +0800 +Subject: [PATCH v4 08/10] libhns: RQ inline support wc_x_poll_cq interface + +RQ inline support user use wc_x_poll_cq get data. + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u_hw_v2.c | 9 +++++++-- + 1 file changed, 7 insertions(+), 2 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index b37ea92..25d8861 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -561,10 +561,15 @@ static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx, + hns_roce_free_srq_wqe(srq, wqe_idx); + } + +-static void cqe_proc_rq(struct hns_roce_wq *wq, struct hns_roce_cq *cq) ++static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq) + { ++ struct hns_roce_wq *wq = &hr_qp->rq; ++ + cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; ++ ++ if (hr_reg_read(cq->cqe, CQE_RQ_INLINE)) ++ handle_recv_rq_inl(cq->cqe, hr_qp); + } + + static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp, +@@ -584,7 +589,7 @@ static int cqe_proc_wq(struct hns_roce_context *ctx, struct hns_roce_qp *qp, + if (srq) + cqe_proc_srq(srq, wqe_idx, cq); + else +- cqe_proc_rq(&qp->rq, cq); ++ cqe_proc_rq(qp, cq); + } + + return 0; +-- +2.30.0 + diff --git a/0009-Update-kernel-headers.patch b/0009-Update-kernel-headers.patch new file mode 100644 index 0000000..0f02221 --- /dev/null +++ b/0009-Update-kernel-headers.patch @@ -0,0 +1,33 @@ +From 14cee9bd8ab06104b9f9a0326b8d17a5bf8ee647 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Tue, 11 Oct 2022 10:50:36 +0800 +Subject: [PATCH v4 09/10] Update kernel headers + +To commit ?? ("RDMA/hns: Support cqe inline in user space"). + +Signed-off-by: Luoyouming +--- + kernel-headers/rdma/hns-abi.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index c70465d..41738b8 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -88,11 +88,13 @@ struct hns_roce_ib_create_qp_resp { + enum { + HNS_ROCE_EXSGE_FLAGS = 1 << 0, + HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, ++ HNS_ROCE_CQE_INLINE_FLAGS = 1 << 2, + }; + + enum { + HNS_ROCE_RSP_EXSGE_FLAGS = 1 << 0, + HNS_ROCE_RSP_RQ_INLINE_FLAGS = 1 << 1, ++ HNS_ROCE_RSP_CQE_INLINE_FLAGS = 1 << 2, + }; + + struct hns_roce_ib_alloc_ucontext_resp { +-- +2.30.0 + diff --git a/0010-libhns-Support-cqe-inline.patch b/0010-libhns-Support-cqe-inline.patch new file mode 100644 index 0000000..5698234 --- /dev/null +++ b/0010-libhns-Support-cqe-inline.patch @@ -0,0 +1,263 @@ +From 71eb90581a338242a26123790e5f24df90327465 Mon Sep 17 00:00:00 2001 +From: Luoyouming +Date: Thu, 11 Aug 2022 20:50:54 +0800 +Subject: [PATCH v4 10/10] libhns: Support cqe inline + +When rq or srq recv data less than or equal to 32 byte in size, roce driver +support get data from cqe. + +Signed-off-by: Luoyouming +Reviewed-by: Yangyang Li +--- + providers/hns/hns_roce_u.c | 3 ++- + providers/hns/hns_roce_u.h | 21 +++++++++-------- + providers/hns/hns_roce_u_hw_v2.c | 39 ++++++++++++++++++++++++++++++-- + providers/hns/hns_roce_u_hw_v2.h | 4 ++-- + providers/hns/hns_roce_u_verbs.c | 25 ++++++++++++++++++-- + 5 files changed, 75 insertions(+), 17 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 6c9aefa..266e73e 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -113,7 +113,8 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + if (!context) + return NULL; + +- cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS; ++ cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS | ++ HNS_ROCE_CQE_INLINE_FLAGS; + if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), + &resp.ibv_resp, sizeof(resp))) + goto err_free; +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 57ebe55..6b64cd0 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -246,10 +246,21 @@ struct hns_roce_idx_que { + unsigned int tail; + }; + ++struct hns_roce_rinl_wqe { ++ struct ibv_sge *sg_list; ++ unsigned int sge_cnt; ++}; ++ ++struct hns_roce_rinl_buf { ++ struct hns_roce_rinl_wqe *wqe_list; ++ unsigned int wqe_cnt; ++}; ++ + struct hns_roce_srq { + struct verbs_srq verbs_srq; + struct hns_roce_idx_que idx_que; + struct hns_roce_buf wqe_buf; ++ struct hns_roce_rinl_buf srq_rinl_buf; + pthread_spinlock_t lock; + unsigned long *wrid; + unsigned int srqn; +@@ -290,16 +301,6 @@ struct hns_roce_sge_ex { + unsigned int sge_shift; + }; + +-struct hns_roce_rinl_wqe { +- struct ibv_sge *sg_list; +- unsigned int sge_cnt; +-}; +- +-struct hns_roce_rinl_buf { +- struct hns_roce_rinl_wqe *wqe_list; +- unsigned int wqe_cnt; +-}; +- + struct hns_roce_qp { + struct verbs_qp verbs_qp; + struct hns_roce_buf buf; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 25d8861..7063b26 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -444,6 +444,28 @@ static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe, + + } + ++static void handle_recv_cqe_inl_from_rq(struct hns_roce_v2_cqe *cqe, ++ struct hns_roce_qp *cur_qp) ++{ ++ uint32_t wr_num; ++ ++ wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1); ++ ++ handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, ++ (uint8_t *)cqe->payload); ++} ++ ++static void handle_recv_cqe_inl_from_srq(struct hns_roce_v2_cqe *cqe, ++ struct hns_roce_srq *srq) ++{ ++ uint32_t wr_num; ++ ++ wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (srq->wqe_cnt - 1); ++ ++ handle_recv_inl_data(cqe, &(srq->srq_rinl_buf), wr_num, ++ (uint8_t *)cqe->payload); ++} ++ + static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp *cur_qp) + { +@@ -473,6 +495,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX); + wc->wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)]; + hns_roce_free_srq_wqe(srq, wqe_idx); ++ ++ if (hr_reg_read(cqe, CQE_CQE_INLINE)) ++ handle_recv_cqe_inl_from_srq(cqe, srq); + } + + static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, +@@ -487,7 +512,9 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc, + if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD) + parse_for_ud_qp(cqe, wc); + +- if (hr_reg_read(cqe, CQE_RQ_INLINE)) ++ if (hr_reg_read(cqe, CQE_CQE_INLINE)) ++ handle_recv_cqe_inl_from_rq(cqe, hr_qp); ++ else if (hr_reg_read(cqe, CQE_RQ_INLINE)) + handle_recv_rq_inl(cqe, hr_qp); + + return 0; +@@ -559,6 +586,9 @@ static void cqe_proc_srq(struct hns_roce_srq *srq, uint32_t wqe_idx, + { + cq->verbs_cq.cq_ex.wr_id = srq->wrid[wqe_idx & (srq->wqe_cnt - 1)]; + hns_roce_free_srq_wqe(srq, wqe_idx); ++ ++ if (hr_reg_read(cq->cqe, CQE_CQE_INLINE)) ++ handle_recv_cqe_inl_from_srq(cq->cqe, srq); + } + + static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq) +@@ -568,7 +598,9 @@ static void cqe_proc_rq(struct hns_roce_qp *hr_qp, struct hns_roce_cq *cq) + cq->verbs_cq.cq_ex.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; + ++wq->tail; + +- if (hr_reg_read(cq->cqe, CQE_RQ_INLINE)) ++ if (hr_reg_read(cq->cqe, CQE_CQE_INLINE)) ++ handle_recv_cqe_inl_from_rq(cq->cqe, hr_qp); ++ else if (hr_reg_read(cq->cqe, CQE_RQ_INLINE)) + handle_recv_rq_inl(cq->cqe, hr_qp); + } + +@@ -1725,6 +1757,9 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + + wqe = get_srq_wqe(srq, wqe_idx); + fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge); ++ ++ fill_recv_inl_buf(&srq->srq_rinl_buf, wqe_idx, wr); ++ + fill_wqe_idx(srq, wqe_idx); + + srq->wrid[wqe_idx] = wr->wr_id; +diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h +index 098dbdf..d71c695 100644 +--- a/providers/hns/hns_roce_u_hw_v2.h ++++ b/providers/hns/hns_roce_u_hw_v2.h +@@ -157,7 +157,7 @@ struct hns_roce_v2_cqe { + __le32 smac; + __le32 byte_28; + __le32 byte_32; +- __le32 rsv[8]; ++ __le32 payload[8]; + }; + + #define CQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cqe, h, l) +@@ -170,7 +170,7 @@ struct hns_roce_v2_cqe { + #define CQE_WQE_IDX CQE_FIELD_LOC(31, 16) + #define CQE_RKEY_IMMTDATA CQE_FIELD_LOC(63, 32) + #define CQE_XRC_SRQN CQE_FIELD_LOC(87, 64) +-#define CQE_RSV0 CQE_FIELD_LOC(95, 88) ++#define CQE_CQE_INLINE CQE_FIELD_LOC(89, 88) + #define CQE_LCL_QPN CQE_FIELD_LOC(119, 96) + #define CQE_SUB_STATUS CQE_FIELD_LOC(127, 120) + #define CQE_BYTE_CNT CQE_FIELD_LOC(159, 128) +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 1d661dd..cff9d1d 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -522,6 +522,8 @@ static int verify_srq_create_attr(struct hns_roce_context *context, + static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq, + struct ibv_srq_init_attr_ex *attr) + { ++ struct hns_roce_context *ctx = to_hr_ctx(context); ++ + if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2) + srq->rsv_sge = 1; + +@@ -531,6 +533,10 @@ static void set_srq_param(struct ibv_context *context, struct hns_roce_srq *srq, + srq->max_gs)); + attr->attr.max_sge = srq->max_gs; + attr->attr.srq_limit = 0; ++ ++ srq->srq_rinl_buf.wqe_cnt = 0; ++ if (ctx->config & HNS_ROCE_RSP_CQE_INLINE_FLAGS) ++ srq->srq_rinl_buf.wqe_cnt = srq->wqe_cnt; + } + + static int alloc_srq_idx_que(struct hns_roce_srq *srq) +@@ -570,6 +576,11 @@ static int alloc_srq_wqe_buf(struct hns_roce_srq *srq) + return hns_roce_alloc_buf(&srq->wqe_buf, buf_size, HNS_HW_PAGE_SIZE); + } + ++static int alloc_recv_rinl_buf(uint32_t max_sge, ++ struct hns_roce_rinl_buf *rinl_buf); ++ ++static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf); ++ + static int alloc_srq_buf(struct hns_roce_srq *srq) + { + int ret; +@@ -582,14 +593,22 @@ static int alloc_srq_buf(struct hns_roce_srq *srq) + if (ret) + goto err_idx_que; + ++ if (srq->srq_rinl_buf.wqe_cnt) { ++ ret = alloc_recv_rinl_buf(srq->max_gs, &srq->srq_rinl_buf); ++ if (ret) ++ goto err_wqe_buf; ++ } ++ + srq->wrid = calloc(srq->wqe_cnt, sizeof(*srq->wrid)); + if (!srq->wrid) { + ret = -ENOMEM; +- goto err_wqe_buf; ++ goto err_inl_buf; + } + + return 0; + ++err_inl_buf: ++ free_recv_rinl_buf(&srq->srq_rinl_buf); + err_wqe_buf: + hns_roce_free_buf(&srq->wqe_buf); + err_idx_que: +@@ -603,6 +622,7 @@ static void free_srq_buf(struct hns_roce_srq *srq) + { + free(srq->wrid); + hns_roce_free_buf(&srq->wqe_buf); ++ free_recv_rinl_buf(&srq->srq_rinl_buf); + hns_roce_free_buf(&srq->idx_que.buf); + free(srq->idx_que.bitmap); + } +@@ -1082,7 +1102,8 @@ static void hns_roce_set_qp_params(struct ibv_qp_init_attr_ex *attr, + qp->rq.wqe_cnt = cnt; + qp->rq.shift = hr_ilog32(cnt); + qp->rq_rinl_buf.wqe_cnt = 0; +- if (ctx->config & HNS_ROCE_RSP_RQ_INLINE_FLAGS) ++ if (ctx->config & (HNS_ROCE_RSP_RQ_INLINE_FLAGS | ++ HNS_ROCE_RSP_CQE_INLINE_FLAGS)) + qp->rq_rinl_buf.wqe_cnt = cnt; + + attr->cap.max_recv_wr = qp->rq.wqe_cnt; +-- +2.30.0 + diff --git a/rdma-core.spec b/rdma-core.spec index a1736e6..b9112a5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,11 +1,22 @@ Name: rdma-core Version: 41.0 -Release: 1 +Release: 2 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core Source: https://github.com/linux-rdma/rdma-core/releases/download/v%{version}/%{name}-%{version}.tar.gz +Patch0: 0001-libhns-Use-a-constant-instead-of-sizeof-operation.patch +Patch1: 0002-libhns-Fix-ext_sge-num-error-when-post-send.patch +Patch2: 0003-Update-kernel-headers.patch +Patch3: 0004-libhns-Fix-the-problem-of-sge-nums.patch +Patch4: 0005-Update-kernel-headers.patch +Patch5: 0006-libhns-Add-compatibility-handling-for-rq-inline.patch +Patch6: 0007-libhns-Refactor-rq-inline.patch +Patch7: 0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch +Patch8: 0009-Update-kernel-headers.patch +Patch9: 0010-libhns-Support-cqe-inline.patch + BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel BuildRequires: python3-devel python3-Cython python3 python3-docutils perl-generators @@ -249,6 +260,12 @@ fi %{_mandir}/* %changelog +* Sat Oct 08 2022 luoyouming - 41.0-2 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Support rq inline and cqe inline + * Tue Sep 27 2022 tangchengchang - 41.0-1 - Type: requirement - ID: NA -- Gitee From 0025a5cf22dfb89d5099ae90727fd863bc2af8c9 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Tue, 1 Nov 2022 20:52:54 +0800 Subject: [PATCH 2/2] Add support for hns DSCP Support DSCP for hns RoCE. Signed-off-by: Chengchang Tang --- 0011-Update-kernel-headers.patch | 38 ++++++++ 0012-libhns-Support-DSCP.patch | 150 +++++++++++++++++++++++++++++++ rdma-core.spec | 10 ++- 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 0011-Update-kernel-headers.patch create mode 100644 0012-libhns-Support-DSCP.patch diff --git a/0011-Update-kernel-headers.patch b/0011-Update-kernel-headers.patch new file mode 100644 index 0000000..7a86e19 --- /dev/null +++ b/0011-Update-kernel-headers.patch @@ -0,0 +1,38 @@ +From 12d2a17d404e3d5ba76863f64307ea52a7d15d15 Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Sat, 29 Oct 2022 10:44:17 +0800 +Subject: [PATCH 11/12] Update kernel headers + +To commit ?? ("RDMA/hns: Support DSCP of userspace"). + +Signed-off-by: Yixing Liu +--- + kernel-headers/rdma/hns-abi.h | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index 41738b8..542be5e 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -85,6 +85,18 @@ struct hns_roce_ib_create_qp_resp { + __aligned_u64 dwqe_mmap_key; + }; + ++struct hns_roce_ib_create_ah_resp { ++ __u8 priority; ++ __u8 tc_mode; ++ __u8 reserved[6]; ++}; ++ ++struct hns_roce_ib_modify_qp_resp { ++ __u8 tc_mode; ++ __u8 priority; ++ __u8 reserved[6]; ++}; ++ + enum { + HNS_ROCE_EXSGE_FLAGS = 1 << 0, + HNS_ROCE_RQ_INLINE_FLAGS = 1 << 1, +-- +2.30.0 + diff --git a/0012-libhns-Support-DSCP.patch b/0012-libhns-Support-DSCP.patch new file mode 100644 index 0000000..6433809 --- /dev/null +++ b/0012-libhns-Support-DSCP.patch @@ -0,0 +1,150 @@ +From b88e6ae3e144651092bce923123ca20361cdacab Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Tue, 27 Sep 2022 19:06:00 +0800 +Subject: [PATCH 12/12] libhns: Support DSCP + +This patch adds user mode DSCP function through +the mapping of dscp-tc configured in kernel mode. + +Signed-off-by: Yixing Liu +--- + providers/hns/hns_roce_u.h | 7 +++++++ + providers/hns/hns_roce_u_abi.h | 6 ++++++ + providers/hns/hns_roce_u_hw_v2.c | 19 +++++++++++++++---- + providers/hns/hns_roce_u_verbs.c | 7 +++++-- + 4 files changed, 33 insertions(+), 6 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 6b64cd0..8c1cb1e 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -175,6 +175,11 @@ enum hns_roce_db_type { + HNS_ROCE_DB_TYPE_NUM + }; + ++enum hns_roce_tc_map_mode { ++ HNS_ROCE_TC_MAP_MODE_PRIO, ++ HNS_ROCE_TC_MAP_MODE_DSCP, ++}; ++ + struct hns_roce_db_page { + struct hns_roce_db_page *prev, *next; + struct hns_roce_buf buf; +@@ -315,6 +320,8 @@ struct hns_roce_qp { + unsigned int next_sge; + int port_num; + uint8_t sl; ++ uint8_t tc_mode; ++ uint8_t priority; + unsigned int qkey; + enum ibv_mtu path_mtu; + +diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h +index 2753d30..0519ac7 100644 +--- a/providers/hns/hns_roce_u_abi.h ++++ b/providers/hns/hns_roce_u_abi.h +@@ -49,6 +49,9 @@ DECLARE_DRV_CMD(hns_roce_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, + DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, + hns_roce_ib_alloc_ucontext, hns_roce_ib_alloc_ucontext_resp); + ++DECLARE_DRV_CMD(hns_roce_create_ah, IB_USER_VERBS_CMD_CREATE_AH, empty, ++ hns_roce_ib_create_ah_resp); ++ + DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP, + hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp); + +@@ -61,4 +64,7 @@ DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ, + DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ, + hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp); + ++DECLARE_DRV_CMD(hns_roce_modify_qp_ex, IB_USER_VERBS_EX_CMD_MODIFY_QP, ++ empty, hns_roce_ib_modify_qp_resp); ++ + #endif /* _HNS_ROCE_U_ABI_H */ +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index a30d461..c652eea 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1543,10 +1543,11 @@ static void record_qp_attr(struct ibv_qp *qp, struct ibv_qp_attr *attr, + static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask) + { +- int ret; +- struct ibv_modify_qp cmd; ++ struct hns_roce_modify_qp_ex_resp resp_ex = {}; ++ struct hns_roce_modify_qp_ex cmd_ex = {}; + struct hns_roce_qp *hr_qp = to_hr_qp(qp); + bool flag = false; /* modify qp to error */ ++ int ret; + + if ((attr_mask & IBV_QP_STATE) && (attr->qp_state == IBV_QPS_ERR)) { + pthread_spin_lock(&hr_qp->sq.lock); +@@ -1554,7 +1555,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + flag = true; + } + +- ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd)); ++ ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp_ex.ibv_resp, ++ sizeof(resp_ex)); + + if (flag) { + pthread_spin_unlock(&hr_qp->rq.lock); +@@ -1564,8 +1567,13 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + if (ret) + return ret; + +- if (attr_mask & IBV_QP_STATE) ++ if (attr_mask & IBV_QP_STATE) { + qp->state = attr->qp_state; ++ if (attr->qp_state == IBV_QPS_RTR) { ++ hr_qp->tc_mode = resp_ex.drv_payload.tc_mode; ++ hr_qp->priority = resp_ex.drv_payload.priority; ++ } ++ } + + if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) { + if (qp->recv_cq) +@@ -1579,6 +1587,9 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + hns_roce_init_qp_indices(to_hr_qp(qp)); + } + ++ if (hr_qp->tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) ++ hr_qp->sl = hr_qp->priority; ++ + record_qp_attr(qp, attr, attr_mask); + + return ret; +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index cff9d1d..3b7a67d 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1449,7 +1449,7 @@ static int get_tclass(struct ibv_context *context, struct ibv_ah_attr *attr, + struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) + { + struct hns_roce_device *hr_dev = to_hr_dev(pd->context->device); +- struct ib_uverbs_create_ah_resp resp = {}; ++ struct hns_roce_create_ah_resp resp = {}; + struct hns_roce_ah *ah; + + /* HIP08 don't support create ah */ +@@ -1477,12 +1477,15 @@ struct ibv_ah *hns_roce_u_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) + memcpy(ah->av.dgid, attr->grh.dgid.raw, ARRAY_SIZE(ah->av.dgid)); + } + +- if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp, sizeof(resp))) ++ if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp.ibv_resp, sizeof(resp))) + goto err; + + if (ibv_resolve_eth_l2_from_gid(pd->context, attr, ah->av.mac, NULL)) + goto err; + ++ if (resp.tc_mode == HNS_ROCE_TC_MAP_MODE_DSCP) ++ ah->av.sl = resp.priority; ++ + ah->av.udp_sport = get_ah_udp_sport(attr); + + return &ah->ibv_ah; +-- +2.30.0 + diff --git a/rdma-core.spec b/rdma-core.spec index b9112a5..9f1c3f5 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 41.0 -Release: 2 +Release: 3 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core @@ -16,6 +16,8 @@ Patch6: 0007-libhns-Refactor-rq-inline.patch Patch7: 0008-libhns-RQ-inline-support-wc_x_poll_cq-interface.patch Patch8: 0009-Update-kernel-headers.patch Patch9: 0010-libhns-Support-cqe-inline.patch +Patch10: 0011-Update-kernel-headers.patch +Patch11: 0012-libhns-Support-DSCP.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -260,6 +262,12 @@ fi %{_mandir}/* %changelog +* Sat Oct 29 2022 tangchengchang - 41.0-3 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Support rq inline and cqe inline + * Sat Oct 08 2022 luoyouming - 41.0-2 - Type: requirement - ID: NA -- Gitee