diff --git a/0001-Update-kernel-headers.patch b/0001-Update-kernel-headers.patch new file mode 100644 index 0000000000000000000000000000000000000000..8a96d70a8fcd4636b0d88dce6d2972929c71ea39 --- /dev/null +++ b/0001-Update-kernel-headers.patch @@ -0,0 +1,41 @@ +From 693d55e80976217215844258e5b78bc115382689 Mon Sep 17 00:00:00 2001 +From: Guofeng Yue +Date: Mon, 10 Jan 2022 10:44:23 +0800 +Subject: [PATCH 1/8] Update kernel headers + +To commit 62c4d8878d13 ("RDMA/hns: Remove support for HIP06"). + +Signed-off-by: Guofeng Yue +--- + kernel-headers/rdma/hns-abi.h | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index 42b17765..abfd36e2 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -77,17 +77,19 @@ enum hns_roce_qp_cap_flags { + HNS_ROCE_QP_CAP_RQ_RECORD_DB = 1 << 0, + HNS_ROCE_QP_CAP_SQ_RECORD_DB = 1 << 1, + HNS_ROCE_QP_CAP_OWNER_DB = 1 << 2, ++ HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, + }; + + struct hns_roce_ib_create_qp_resp { + __aligned_u64 cap_flags; ++ __aligned_u64 dwqe_mmap_key; + }; + + struct hns_roce_ib_alloc_ucontext_resp { + __u32 qp_tab_size; + __u32 cqe_size; +- __u32 srq_tab_size; +- __u32 reserved; ++ __u32 srq_tab_size; ++ __u32 reserved; + }; + + struct hns_roce_ib_alloc_pd_resp { +-- +2.33.0 + diff --git a/0002-libhns-Fix-the-ownership-of-the-head-tail-pointer-of.patch b/0002-libhns-Fix-the-ownership-of-the-head-tail-pointer-of.patch new file mode 100644 index 0000000000000000000000000000000000000000..7dcc9b88e9113c24a20889e1f8de7449bce436de --- /dev/null +++ b/0002-libhns-Fix-the-ownership-of-the-head-tail-pointer-of.patch @@ -0,0 +1,120 @@ +From 08ec3c43bf9710fdf3ca664f7cd63436e67339d7 Mon Sep 17 00:00:00 2001 +From: Wenpeng Liang +Date: Tue, 11 May 2021 19:06:34 +0800 +Subject: [PATCH 2/8] libhns: Fix the ownership of the head/tail pointer of SRQ + WQE + +The CQE of SRQ is not generated in the order of wqe, so the wqe_idx +corresponding to the idle WQE should be placed in a FIFO, then the hardware +will be instructed to obtain the corresponding WQE. Therefore, the WQ +of SRQ has no concept of head pointer and tail pointer, but the queue of +wqe_idx does. + +Signed-off-by: Wenpeng Liang +Signed-off-by: Weihang Li +--- + providers/hns/hns_roce_u.h | 4 ++-- + providers/hns/hns_roce_u_hw_v2.c | 12 ++++++------ + providers/hns/hns_roce_u_verbs.c | 6 +++--- + 3 files changed, 11 insertions(+), 11 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 8f805dd1..b3f48113 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -205,6 +205,8 @@ struct hns_roce_idx_que { + int entry_shift; + unsigned long *bitmap; + int bitmap_cnt; ++ unsigned int head; ++ unsigned int tail; + }; + + struct hns_roce_srq { +@@ -217,8 +219,6 @@ struct hns_roce_srq { + unsigned int max_gs; + unsigned int rsv_sge; + unsigned int wqe_shift; +- int head; +- int tail; + unsigned int *db; + unsigned short counter; + struct hns_roce_idx_que idx_que; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 4988943a..f947dbd7 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -262,7 +262,7 @@ static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, uint16_t ind) + bitmap_num = ind / BIT_CNT_PER_LONG; + bit_num = ind % BIT_CNT_PER_LONG; + srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num); +- srq->tail++; ++ srq->idx_que.tail++; + + pthread_spin_unlock(&srq->lock); + } +@@ -1564,7 +1564,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + pthread_spin_lock(&srq->lock); + + /* current idx of srqwq */ +- ind = srq->head & (srq->wqe_cnt - 1); ++ ind = srq->idx_que.head & (srq->wqe_cnt - 1); + + max_sge = srq->max_gs - srq->rsv_sge; + for (nreq = 0; wr; ++nreq, wr = wr->next) { +@@ -1574,7 +1574,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + break; + } + +- if (srq->head == srq->tail) { ++ if (srq->idx_que.head == srq->idx_que.tail) { + ret = -ENOMEM; + *bad_wr = wr; + break; +@@ -1607,7 +1607,7 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + } + + if (nreq) { +- srq->head += nreq; ++ srq->idx_que.head += nreq; + + /* + * Make sure that descriptors are written before +@@ -1617,8 +1617,8 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + + srq_db.byte_4 = htole32(HNS_ROCE_V2_SRQ_DB << DB_BYTE_4_CMD_S | + srq->srqn); +- srq_db.parameter = +- htole32(srq->head & DB_PARAM_SRQ_PRODUCER_COUNTER_M); ++ srq_db.parameter = htole32(srq->idx_que.head & ++ DB_PARAM_SRQ_PRODUCER_COUNTER_M); + + hns_roce_write64((uint32_t *)&srq_db, ctx, + ROCEE_VF_DB_CFG0_OFFSET); +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 30ab072a..9b4934b9 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -491,6 +491,9 @@ static int hns_roce_create_idx_que(struct hns_roce_srq *srq) + for (i = 0; i < idx_que->bitmap_cnt; ++i) + idx_que->bitmap[i] = ~(0UL); + ++ idx_que->head = 0; ++ idx_que->tail = srq->wqe_cnt - 1; ++ + return 0; + } + +@@ -512,9 +515,6 @@ static int hns_roce_alloc_srq_buf(struct hns_roce_srq *srq) + return ENOMEM; + } + +- srq->head = 0; +- srq->tail = srq->wqe_cnt - 1; +- + return 0; + } + +-- +2.33.0 + diff --git a/0003-libhns-Fix-wrong-data-type-when-writing-doorbell.patch b/0003-libhns-Fix-wrong-data-type-when-writing-doorbell.patch new file mode 100644 index 0000000000000000000000000000000000000000..6385c36b0ee9c9b764cf3d9791dd0530cbecc104 --- /dev/null +++ b/0003-libhns-Fix-wrong-data-type-when-writing-doorbell.patch @@ -0,0 +1,180 @@ +From 9cc4c4b8d31b35428859ef626d4428fc393aace4 Mon Sep 17 00:00:00 2001 +From: Lang Cheng +Date: Thu, 11 Nov 2021 21:08:35 +0800 +Subject: [PATCH 3/8] libhns: Fix wrong data type when writing doorbell + +The DB data is a __le32[] value instead of uint32_t[], and the DB register +should be written with a little-endian data instead of uint64_t. + +Fixes: 1523fbb1ea8e ("libhns: Add verbs of cq support") +Signed-off-by: Lang Cheng +Signed-off-by: Yixing Liu +Signed-off-by: Wenpeng Liang +--- + providers/hns/hns_roce_u_db.h | 14 ++++---------- + providers/hns/hns_roce_u_hw_v1.c | 17 +++++++++-------- + providers/hns/hns_roce_u_hw_v2.c | 23 ++++++++++++----------- + 3 files changed, 25 insertions(+), 29 deletions(-) + +diff --git a/providers/hns/hns_roce_u_db.h b/providers/hns/hns_roce_u_db.h +index b44e64d4..13df9b52 100644 +--- a/providers/hns/hns_roce_u_db.h ++++ b/providers/hns/hns_roce_u_db.h +@@ -32,23 +32,17 @@ + + #include + ++#include + #include "hns_roce_u.h" + + #ifndef _HNS_ROCE_U_DB_H + #define _HNS_ROCE_U_DB_H + +-#if __BYTE_ORDER == __LITTLE_ENDIAN +-#define HNS_ROCE_PAIR_TO_64(val) ((uint64_t) val[1] << 32 | val[0]) +-#elif __BYTE_ORDER == __BIG_ENDIAN +-#define HNS_ROCE_PAIR_TO_64(val) ((uint64_t) val[0] << 32 | val[1]) +-#else +-#error __BYTE_ORDER not defined +-#endif ++#define HNS_ROCE_WORD_NUM 2 + +-static inline void hns_roce_write64(uint32_t val[2], +- struct hns_roce_context *ctx, int offset) ++static inline void hns_roce_write64(void *dest, __le32 val[HNS_ROCE_WORD_NUM]) + { +- *(volatile uint64_t *) (ctx->uar + offset) = HNS_ROCE_PAIR_TO_64(val); ++ mmio_write64_le(dest, *(__le64 *)val); + } + + void *hns_roce_alloc_db(struct hns_roce_context *ctx, +diff --git a/providers/hns/hns_roce_u_hw_v1.c b/providers/hns/hns_roce_u_hw_v1.c +index 8f0a71aa..14ee4817 100644 +--- a/providers/hns/hns_roce_u_hw_v1.c ++++ b/providers/hns/hns_roce_u_hw_v1.c +@@ -65,7 +65,7 @@ static void hns_roce_update_rq_head(struct hns_roce_context *ctx, + + udma_to_device_barrier(); + +- hns_roce_write64((uint32_t *)&rq_db, ctx, ROCEE_DB_OTHERS_L_0_REG); ++ hns_roce_write64(ctx->uar + ROCEE_DB_OTHERS_L_0_REG, (__le32 *)&rq_db); + } + + static void hns_roce_update_sq_head(struct hns_roce_context *ctx, +@@ -84,7 +84,7 @@ static void hns_roce_update_sq_head(struct hns_roce_context *ctx, + + udma_to_device_barrier(); + +- hns_roce_write64((uint32_t *)&sq_db, ctx, ROCEE_DB_SQ_L_0_REG); ++ hns_roce_write64(ctx->uar + ROCEE_DB_SQ_L_0_REG, (__le32 *)&sq_db); + } + + static void hns_roce_update_cq_cons_index(struct hns_roce_context *ctx, +@@ -102,7 +102,7 @@ static void hns_roce_update_cq_cons_index(struct hns_roce_context *ctx, + CQ_DB_U32_4_CONS_IDX_S, + cq->cons_index & ((cq->cq_depth << 1) - 1)); + +- hns_roce_write64((uint32_t *)&cq_db, ctx, ROCEE_DB_OTHERS_L_0_REG); ++ hns_roce_write64(ctx->uar + ROCEE_DB_OTHERS_L_0_REG, (__le32 *)&cq_db); + } + + static void hns_roce_handle_error_cqe(struct hns_roce_cqe *cqe, +@@ -422,10 +422,11 @@ static int hns_roce_u_v1_poll_cq(struct ibv_cq *ibvcq, int ne, + */ + static int hns_roce_u_v1_arm_cq(struct ibv_cq *ibvcq, int solicited) + { +- uint32_t ci; +- uint32_t solicited_flag; +- struct hns_roce_cq_db cq_db = {}; ++ struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context); + struct hns_roce_cq *cq = to_hr_cq(ibvcq); ++ struct hns_roce_cq_db cq_db = {}; ++ uint32_t solicited_flag; ++ uint32_t ci; + + ci = cq->cons_index & ((cq->cq_depth << 1) - 1); + solicited_flag = solicited ? HNS_ROCE_CQ_DB_REQ_SOL : +@@ -441,8 +442,8 @@ static int hns_roce_u_v1_arm_cq(struct ibv_cq *ibvcq, int solicited) + roce_set_field(cq_db.u32_4, CQ_DB_U32_4_CONS_IDX_M, + CQ_DB_U32_4_CONS_IDX_S, ci); + +- hns_roce_write64((uint32_t *)&cq_db, to_hr_ctx(ibvcq->context), +- ROCEE_DB_OTHERS_L_0_REG); ++ hns_roce_write64(ctx->uar + ROCEE_DB_OTHERS_L_0_REG, (__le32 *)&cq_db); ++ + return 0; + } + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index f947dbd7..efd949f4 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -293,7 +293,7 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx, + HNS_ROCE_V2_RQ_DB); + rq_db.parameter = htole32(rq_head); + +- hns_roce_write64((uint32_t *)&rq_db, ctx, ROCEE_VF_DB_CFG0_OFFSET); ++ hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&rq_db); + } + + static void hns_roce_update_sq_db(struct hns_roce_context *ctx, +@@ -308,7 +308,7 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx, + sq_db.parameter = htole32(sq_head); + roce_set_field(sq_db.parameter, DB_PARAM_SL_M, DB_PARAM_SL_S, sl); + +- hns_roce_write64((uint32_t *)&sq_db, ctx, ROCEE_VF_DB_CFG0_OFFSET); ++ hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&sq_db); + } + + static void hns_roce_v2_update_cq_cons_index(struct hns_roce_context *ctx, +@@ -325,7 +325,7 @@ static void hns_roce_v2_update_cq_cons_index(struct hns_roce_context *ctx, + roce_set_field(cq_db.parameter, DB_PARAM_CQ_CMD_SN_M, + DB_PARAM_CQ_CMD_SN_S, 1); + +- hns_roce_write64((uint32_t *)&cq_db, ctx, ROCEE_VF_DB_CFG0_OFFSET); ++ hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db); + } + + static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx, +@@ -659,11 +659,12 @@ static int hns_roce_u_v2_poll_cq(struct ibv_cq *ibvcq, int ne, + + static int hns_roce_u_v2_arm_cq(struct ibv_cq *ibvcq, int solicited) + { +- uint32_t ci; +- uint32_t cmd_sn; +- uint32_t solicited_flag; +- struct hns_roce_db cq_db = {}; ++ struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context); + struct hns_roce_cq *cq = to_hr_cq(ibvcq); ++ struct hns_roce_db cq_db = {}; ++ uint32_t solicited_flag; ++ uint32_t cmd_sn; ++ uint32_t ci; + + ci = cq->cons_index & ((cq->cq_depth << 1) - 1); + cmd_sn = cq->arm_sn & HNS_ROCE_CMDSN_MASK; +@@ -681,8 +682,8 @@ static int hns_roce_u_v2_arm_cq(struct ibv_cq *ibvcq, int solicited) + DB_PARAM_CQ_CMD_SN_S, cmd_sn); + roce_set_bit(cq_db.parameter, DB_PARAM_CQ_NOTIFY_S, solicited_flag); + +- hns_roce_write64((uint32_t *)&cq_db, to_hr_ctx(ibvcq->context), +- ROCEE_VF_DB_CFG0_OFFSET); ++ hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&cq_db); ++ + return 0; + } + +@@ -1620,8 +1621,8 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + srq_db.parameter = htole32(srq->idx_que.head & + DB_PARAM_SRQ_PRODUCER_COUNTER_M); + +- hns_roce_write64((uint32_t *)&srq_db, ctx, +- ROCEE_VF_DB_CFG0_OFFSET); ++ hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, ++ (__le32 *)&srq_db); + } + + pthread_spin_unlock(&srq->lock); +-- +2.33.0 + diff --git a/0004-libhns-Remove-unsupported-QP-type.patch b/0004-libhns-Remove-unsupported-QP-type.patch new file mode 100644 index 0000000000000000000000000000000000000000..e144b050b6b376db6e91723db6a091c0572066b9 --- /dev/null +++ b/0004-libhns-Remove-unsupported-QP-type.patch @@ -0,0 +1,43 @@ +From 60d82566fc94b11280be26733bc306e6af3d2697 Mon Sep 17 00:00:00 2001 +From: Wenpeng Liang +Date: Tue, 9 Nov 2021 20:40:58 +0800 +Subject: [PATCH 4/8] libhns: Remove unsupported QP type + +Currently, user space does not support UC type QP. + +Signed-off-by: Wenpeng Liang +Signed-off-by: Leon Romanovsky +--- + providers/hns/hns_roce_u_hw_v1.c | 1 - + providers/hns/hns_roce_u_hw_v2.c | 3 +-- + 2 files changed, 1 insertion(+), 3 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v1.c b/providers/hns/hns_roce_u_hw_v1.c +index 14ee4817..279c9b0f 100644 +--- a/providers/hns/hns_roce_u_hw_v1.c ++++ b/providers/hns/hns_roce_u_hw_v1.c +@@ -532,7 +532,6 @@ static int hns_roce_u_v1_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, + ctrl->flag |= htole32(ps_opcode); + wqe += sizeof(struct hns_roce_wqe_raddr_seg); + break; +- case IBV_QPT_UC: + case IBV_QPT_UD: + default: + break; +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index efd949f4..c62f74b5 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -460,8 +460,7 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, + struct hns_roce_qp **cur_qp, + struct ibv_wc *wc, uint32_t opcode) + { +- if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC || +- (*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_UC) && ++ if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC) && + (opcode == HNS_ROCE_RECV_OP_SEND || + opcode == HNS_ROCE_RECV_OP_SEND_WITH_IMM || + opcode == HNS_ROCE_RECV_OP_SEND_WITH_INV) && +-- +2.33.0 + diff --git a/0005-libhns-Avoid-using-WQE-indexes-that-exceed-the-SRQ-s.patch b/0005-libhns-Avoid-using-WQE-indexes-that-exceed-the-SRQ-s.patch new file mode 100644 index 0000000000000000000000000000000000000000..4ae71d1c8622acffb3e68d1d96d91cb478991e0b --- /dev/null +++ b/0005-libhns-Avoid-using-WQE-indexes-that-exceed-the-SRQ-s.patch @@ -0,0 +1,67 @@ +From e460a4208d1821b1477e621ad5a7b72068e844f9 Mon Sep 17 00:00:00 2001 +From: Wenpeng Liang +Date: Tue, 11 May 2021 19:06:32 +0800 +Subject: [PATCH 5/8] libhns: Avoid using WQE indexes that exceed the SRQ size + +The index of SRQ WQE got from bitmap may be greater than the capability, +so a check for that should be added. + +Signed-off-by: Wenpeng Liang +Signed-off-by: Weihang Li +--- + providers/hns/hns_roce_u_hw_v2.c | 20 ++++++++++++++------ + 1 file changed, 14 insertions(+), 6 deletions(-) + +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index c62f74b5..1169b64b 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -1527,8 +1527,9 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + return ret; + } + +-static int find_empty_entry(struct hns_roce_idx_que *idx_que) ++static int get_wqe_idx(struct hns_roce_srq *srq, int *wqe_idx) + { ++ struct hns_roce_idx_que *idx_que = &srq->idx_que; + int bit_num; + int i; + +@@ -1536,12 +1537,20 @@ static int find_empty_entry(struct hns_roce_idx_que *idx_que) + for (i = 0; i < idx_que->bitmap_cnt && idx_que->bitmap[i] == 0; ++i) + ; + if (i == idx_que->bitmap_cnt) +- return ENOMEM; ++ return -ENOMEM; + + bit_num = ffsl(idx_que->bitmap[i]); + idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1)); + +- return i * BIT_CNT_PER_LONG + (bit_num - 1); ++ *wqe_idx = i * BIT_CNT_PER_LONG + (bit_num - 1); ++ ++ /* If wqe_cnt is less than BIT_CNT_PER_LONG, wqe_idx may be greater ++ * than wqe_cnt. ++ */ ++ if (*wqe_idx >= srq->wqe_cnt) ++ return -ENOMEM; ++ ++ return 0; + } + + static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, +@@ -1580,9 +1589,8 @@ static int hns_roce_u_v2_post_srq_recv(struct ibv_srq *ib_srq, + break; + } + +- wqe_idx = find_empty_entry(&srq->idx_que); +- if (wqe_idx < 0 || wqe_idx >= srq->wqe_cnt) { +- ret = -ENOMEM; ++ ret = get_wqe_idx(srq, &wqe_idx); ++ if (ret) { + *bad_wr = wr; + break; + } +-- +2.33.0 + diff --git a/0006-libhns-Don-t-create-RQ-for-a-QP-that-associated-with.patch b/0006-libhns-Don-t-create-RQ-for-a-QP-that-associated-with.patch new file mode 100644 index 0000000000000000000000000000000000000000..9105a5d969a62354e1a57df3153e84c0c494bbc4 --- /dev/null +++ b/0006-libhns-Don-t-create-RQ-for-a-QP-that-associated-with.patch @@ -0,0 +1,33 @@ +From 91034654bdb2fd6e1fce81b4c1aea41bb4b6bf98 Mon Sep 17 00:00:00 2001 +From: Wenpeng Liang +Date: Tue, 11 May 2021 19:06:33 +0800 +Subject: [PATCH 6/8] libhns: Don't create RQ for a QP that associated with a + SRQ + +If a QP is associated with a SRQ, it's RQ should not be created. + +Signed-off-by: Wenpeng Liang +Signed-off-by: Weihang Li +--- + providers/hns/hns_roce_u_verbs.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 9b4934b9..125858d2 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -760,6 +760,11 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + cap->max_recv_sge > ctx->max_sge) + return -EINVAL; + ++ if (attr->srq) { ++ cap->max_recv_wr = 0; ++ cap->max_recv_sge = 0; ++ } ++ + min_wqe_num = hr_dev->hw_version == HNS_ROCE_HW_VER1 ? + HNS_ROCE_V1_MIN_WQE_NUM : HNS_ROCE_V2_MIN_WQE_NUM; + +-- +2.33.0 + diff --git a/0007-libhns-Add-support-for-direct-wqe.patch b/0007-libhns-Add-support-for-direct-wqe.patch new file mode 100644 index 0000000000000000000000000000000000000000..e310d4922dbc6f9f22bc11b2bed828745b767033 --- /dev/null +++ b/0007-libhns-Add-support-for-direct-wqe.patch @@ -0,0 +1,368 @@ +From 64c66455fef1c908cc8f06a2b71aa2fd71806218 Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Wed, 15 Dec 2021 16:42:30 +0800 +Subject: [PATCH 7/8] libhns: Add support for direct wqe + +The current write wqe mechanism is to write to DDR first, and then notify +the hardware through doorbell to read the data. Direct wqe is a mechanism +to fill wqe directly into the hardware. In the case of light load, the wqe +will be filled into pcie bar space of the hardware, this will reduce one +memory access operation and therefore reduce the latency. SIMD instructions +allows cpu to write the 512 bits at one time to device memory, thus it can +be used for posting direct wqe. + +The process of post send of HIP08/09: + + +-----------+ + | post send | + +-----+-----+ + | + +-----+-----+ + | write WQE | + +-----+-----+ + | + | udma_to_device_barrier() + | + +-----+-----+ Y +-----------+ N + | HIP09 ? +------+ multi WR ?+-------------+ + +-----+-----+ +-----+-----+ | + | N | Y | + +-----+-----+ +-----+-----+ +--------+--------+ + | ring DB | | ring DB | |direct WQE (ST4) | + +-----------+ +-----------+ +-----------------+ + +Signed-off-by: Yixing Liu +Signed-off-by: Lang Cheng +Signed-off-by: Wenpeng Liang +--- + providers/hns/hns_roce_u.h | 5 +++- + providers/hns/hns_roce_u_hw_v2.c | 43 ++++++++++++++++++++++++++------ + providers/hns/hns_roce_u_hw_v2.h | 31 +++++++++++++---------- + providers/hns/hns_roce_u_verbs.c | 26 +++++++++++++++++-- + util/mmio.h | 27 +++++++++++++++++++- + 5 files changed, 107 insertions(+), 25 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index b3f48113..37711363 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -80,6 +80,8 @@ + + #define INVALID_SGE_LENGTH 0x80000000 + ++#define HNS_ROCE_DWQE_PAGE_SIZE 65536 ++ + #define HNS_ROCE_ADDRESS_MASK 0xFFFFFFFF + #define HNS_ROCE_ADDRESS_SHIFT 32 + +@@ -279,13 +281,14 @@ struct hns_roce_qp { + struct hns_roce_sge_ex ex_sge; + unsigned int next_sge; + int port_num; +- int sl; ++ uint8_t sl; + unsigned int qkey; + enum ibv_mtu path_mtu; + + struct hns_roce_rinl_buf rq_rinl_buf; + unsigned long flags; + int refcnt; /* specially used for XRC */ ++ void *dwqe_page; + }; + + struct hns_roce_av { +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index 1169b64b..f102fd61 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -33,6 +33,7 @@ + #define _GNU_SOURCE + #include + #include ++#include + #include "hns_roce_u.h" + #include "hns_roce_u_db.h" + #include "hns_roce_u_hw_v2.h" +@@ -297,20 +298,40 @@ static void hns_roce_update_rq_db(struct hns_roce_context *ctx, + } + + static void hns_roce_update_sq_db(struct hns_roce_context *ctx, +- unsigned int qpn, unsigned int sl, +- unsigned int sq_head) ++ struct hns_roce_qp *qp) + { + struct hns_roce_db sq_db = {}; + +- sq_db.byte_4 = htole32(qpn); ++ sq_db.byte_4 = htole32(qp->verbs_qp.qp.qp_num); + roce_set_field(sq_db.byte_4, DB_BYTE_4_CMD_M, DB_BYTE_4_CMD_S, + HNS_ROCE_V2_SQ_DB); +- sq_db.parameter = htole32(sq_head); +- roce_set_field(sq_db.parameter, DB_PARAM_SL_M, DB_PARAM_SL_S, sl); + ++ sq_db.parameter = htole32(qp->sq.head); ++ roce_set_field(sq_db.parameter, DB_PARAM_SL_M, DB_PARAM_SL_S, qp->sl); + hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&sq_db); + } + ++static void hns_roce_write512(uint64_t *dest, uint64_t *val) ++{ ++ mmio_memcpy_x64(dest, val, sizeof(struct hns_roce_rc_sq_wqe)); ++} ++ ++static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe) ++{ ++ struct hns_roce_rc_sq_wqe *rc_sq_wqe = wqe; ++ ++ /* All kinds of DirectWQE have the same header field layout */ ++ roce_set_bit(rc_sq_wqe->byte_4, RC_SQ_WQE_BYTE_4_FLAG_S, 1); ++ roce_set_field(rc_sq_wqe->byte_4, RC_SQ_WQE_BYTE_4_DB_SL_L_M, ++ RC_SQ_WQE_BYTE_4_DB_SL_L_S, qp->sl); ++ roce_set_field(rc_sq_wqe->byte_4, RC_SQ_WQE_BYTE_4_DB_SL_H_M, ++ RC_SQ_WQE_BYTE_4_DB_SL_H_S, qp->sl >> HNS_ROCE_SL_SHIFT); ++ roce_set_field(rc_sq_wqe->byte_4, RC_SQ_WQE_BYTE_4_WQE_INDEX_M, ++ RC_SQ_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head); ++ ++ hns_roce_write512(qp->dwqe_page, wqe); ++} ++ + static void hns_roce_v2_update_cq_cons_index(struct hns_roce_context *ctx, + struct hns_roce_cq *cq) + { +@@ -339,8 +360,7 @@ static struct hns_roce_qp *hns_roce_v2_find_qp(struct hns_roce_context *ctx, + return NULL; + } + +-static void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, +- struct hns_roce_qp *qp) ++void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp) + { + uint32_t qpn = qp->verbs_qp.qp.qp_num; + uint32_t tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift; +@@ -1196,6 +1216,7 @@ int hns_roce_u_v2_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, + break; + case IBV_QPT_UD: + ret = set_ud_wqe(wqe, qp, wr, nreq, &sge_info); ++ qp->sl = to_hr_ah(wr->wr.ud.ah)->av.sl; + break; + default: + ret = EINVAL; +@@ -1214,7 +1235,10 @@ out: + + udma_to_device_barrier(); + +- hns_roce_update_sq_db(ctx, ibvqp->qp_num, qp->sl, qp->sq.head); ++ if (nreq == 1 && (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) ++ hns_roce_write_dwqe(qp, wqe); ++ else ++ hns_roce_update_sq_db(ctx, qp); + + if (qp->flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) + *(qp->sdb) = qp->sq.head & 0xffff; +@@ -1506,6 +1530,9 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp) + if (ret) + return ret; + ++ if (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE) ++ munmap(qp->dwqe_page, HNS_ROCE_DWQE_PAGE_SIZE); ++ + hns_roce_v2_clear_qp(ctx, qp); + + hns_roce_lock_cqs(ibqp); +diff --git a/providers/hns/hns_roce_u_hw_v2.h b/providers/hns/hns_roce_u_hw_v2.h +index c13d82e3..af72cd70 100644 +--- a/providers/hns/hns_roce_u_hw_v2.h ++++ b/providers/hns/hns_roce_u_hw_v2.h +@@ -40,6 +40,8 @@ + + #define HNS_ROCE_CMDSN_MASK 0x3 + ++#define HNS_ROCE_SL_SHIFT 2 ++ + /* V2 REG DEFINITION */ + #define ROCEE_VF_DB_CFG0_OFFSET 0x0230 + +@@ -133,6 +135,8 @@ struct hns_roce_db { + #define DB_BYTE_4_CMD_S 24 + #define DB_BYTE_4_CMD_M GENMASK(27, 24) + ++#define DB_BYTE_4_FLAG_S 31 ++ + #define DB_PARAM_SRQ_PRODUCER_COUNTER_S 0 + #define DB_PARAM_SRQ_PRODUCER_COUNTER_M GENMASK(15, 0) + +@@ -216,8 +220,16 @@ struct hns_roce_rc_sq_wqe { + }; + + #define RC_SQ_WQE_BYTE_4_OPCODE_S 0 +-#define RC_SQ_WQE_BYTE_4_OPCODE_M \ +- (((1UL << 5) - 1) << RC_SQ_WQE_BYTE_4_OPCODE_S) ++#define RC_SQ_WQE_BYTE_4_OPCODE_M GENMASK(4, 0) ++ ++#define RC_SQ_WQE_BYTE_4_DB_SL_L_S 5 ++#define RC_SQ_WQE_BYTE_4_DB_SL_L_M GENMASK(6, 5) ++ ++#define RC_SQ_WQE_BYTE_4_DB_SL_H_S 13 ++#define RC_SQ_WQE_BYTE_4_DB_SL_H_M GENMASK(14, 13) ++ ++#define RC_SQ_WQE_BYTE_4_WQE_INDEX_S 15 ++#define RC_SQ_WQE_BYTE_4_WQE_INDEX_M GENMASK(30, 15) + + #define RC_SQ_WQE_BYTE_4_OWNER_S 7 + +@@ -239,6 +251,8 @@ struct hns_roce_rc_sq_wqe { + + #define RC_SQ_WQE_BYTE_4_RDMA_WRITE_S 22 + ++#define RC_SQ_WQE_BYTE_4_FLAG_S 31 ++ + #define RC_SQ_WQE_BYTE_16_XRC_SRQN_S 0 + #define RC_SQ_WQE_BYTE_16_XRC_SRQN_M \ + (((1UL << 24) - 1) << RC_SQ_WQE_BYTE_16_XRC_SRQN_S) +@@ -311,23 +325,12 @@ struct hns_roce_ud_sq_wqe { + #define UD_SQ_WQE_OPCODE_S 0 + #define UD_SQ_WQE_OPCODE_M GENMASK(4, 0) + +-#define UD_SQ_WQE_DB_SL_L_S 5 +-#define UD_SQ_WQE_DB_SL_L_M GENMASK(6, 5) +- +-#define UD_SQ_WQE_DB_SL_H_S 13 +-#define UD_SQ_WQE_DB_SL_H_M GENMASK(14, 13) +- +-#define UD_SQ_WQE_INDEX_S 15 +-#define UD_SQ_WQE_INDEX_M GENMASK(30, 15) +- + #define UD_SQ_WQE_OWNER_S 7 + + #define UD_SQ_WQE_CQE_S 8 + + #define UD_SQ_WQE_SE_S 11 + +-#define UD_SQ_WQE_FLAG_S 31 +- + #define UD_SQ_WQE_PD_S 0 + #define UD_SQ_WQE_PD_M GENMASK(23, 0) + +@@ -376,4 +379,6 @@ struct hns_roce_ud_sq_wqe { + + #define MAX_SERVICE_LEVEL 0x7 + ++void hns_roce_v2_clear_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp); ++ + #endif /* _HNS_ROCE_U_HW_V2_H */ +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 125858d2..fc902815 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1076,7 +1076,8 @@ static int hns_roce_store_qp(struct hns_roce_context *ctx, + + static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, +- struct hns_roce_context *ctx) ++ struct hns_roce_context *ctx, ++ uint64_t *dwqe_mmap_key) + { + struct hns_roce_create_qp_ex_resp resp_ex = {}; + struct hns_roce_create_qp_ex cmd_ex = {}; +@@ -1093,6 +1094,7 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + &resp_ex.ibv_resp, sizeof(resp_ex)); + + qp->flags = resp_ex.drv_payload.cap_flags; ++ *dwqe_mmap_key = resp_ex.drv_payload.dwqe_mmap_key; + + return ret; + } +@@ -1144,11 +1146,23 @@ static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr, + return ret; + } + ++static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp, ++ uint64_t dwqe_mmap_key) ++{ ++ qp->dwqe_page = mmap(NULL, HNS_ROCE_DWQE_PAGE_SIZE, PROT_WRITE, ++ MAP_SHARED, ibv_ctx->cmd_fd, dwqe_mmap_key); ++ if (qp->dwqe_page == MAP_FAILED) ++ return -EINVAL; ++ ++ return 0; ++} ++ + static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + struct ibv_qp_init_attr_ex *attr) + { + struct hns_roce_context *context = to_hr_ctx(ibv_ctx); + struct hns_roce_qp *qp; ++ uint64_t dwqe_mmap_key; + int ret; + + ret = verify_qp_create_attr(context, attr); +@@ -1167,7 +1181,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + if (ret) + goto err_buf; + +- ret = qp_exec_create_cmd(attr, qp, context); ++ ret = qp_exec_create_cmd(attr, qp, context, &dwqe_mmap_key); + if (ret) + goto err_cmd; + +@@ -1175,10 +1189,18 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + if (ret) + goto err_store; + ++ if (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE) { ++ ret = mmap_dwqe(ibv_ctx, qp, dwqe_mmap_key); ++ if (ret) ++ goto err_dwqe; ++ } ++ + qp_setup_config(attr, qp, context); + + return &qp->verbs_qp.qp; + ++err_dwqe: ++ hns_roce_v2_clear_qp(context, qp); + err_store: + ibv_cmd_destroy_qp(&qp->verbs_qp.qp); + err_cmd: +diff --git a/util/mmio.h b/util/mmio.h +index 101af9dd..01d1455e 100644 +--- a/util/mmio.h ++++ b/util/mmio.h +@@ -210,8 +210,33 @@ static inline void mmio_memcpy_x64(void *dest, const void *src, size_t bytecnt) + { + s390_mmio_write(dest, src, bytecnt); + } +-#else + ++#elif defined(__aarch64__) || defined(__arm__) ++#include ++ ++static inline void _mmio_memcpy_x64_64b(void *dest, const void *src) ++{ ++ vst4q_u64(dest, vld4q_u64(src)); ++} ++ ++static inline void _mmio_memcpy_x64(void *dest, const void *src, size_t bytecnt) ++{ ++ do { ++ _mmio_memcpy_x64_64b(dest, src); ++ bytecnt -= sizeof(uint64x2x4_t); ++ src += sizeof(uint64x2x4_t); ++ } while (bytecnt > 0); ++} ++ ++#define mmio_memcpy_x64(dest, src, bytecount) \ ++ ({ \ ++ if (__builtin_constant_p((bytecount) == 64)) \ ++ _mmio_memcpy_x64_64b((dest), (src)); \ ++ else \ ++ _mmio_memcpy_x64((dest), (src), (bytecount)); \ ++ }) ++ ++#else + /* Transfer is some multiple of 64 bytes */ + static inline void mmio_memcpy_x64(void *dest, const void *src, size_t bytecnt) + { +-- +2.33.0 + diff --git a/0008-libhns-Use-new-SQ-doorbell-register-for-HIP09.patch b/0008-libhns-Use-new-SQ-doorbell-register-for-HIP09.patch new file mode 100644 index 0000000000000000000000000000000000000000..b19d4c1b7662a37dc60dd3abf78645bbf9374e67 --- /dev/null +++ b/0008-libhns-Use-new-SQ-doorbell-register-for-HIP09.patch @@ -0,0 +1,70 @@ +From 608c142e7cbac2a6c02071022fe87b081a6ddc4f Mon Sep 17 00:00:00 2001 +From: Yixing Liu +Date: Tue, 21 Dec 2021 21:38:08 +0800 +Subject: [PATCH 8/8] libhns: Use new SQ doorbell register for HIP09 + +HIP09 set a new BAR space for SQ doorbell. Each SQ doorbell has an +independent BAR space and the size is 64KB. SQ doorbell share +the same BAR space with direct WQE. + +Signed-off-by: Yixing Liu +Signed-off-by: Wenpeng Liang +--- + providers/hns/hns_roce_u.h | 1 + + providers/hns/hns_roce_u_hw_v2.c | 4 ++-- + providers/hns/hns_roce_u_verbs.c | 5 +++++ + 3 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 37711363..460363b7 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -238,6 +238,7 @@ struct hns_roce_wq { + unsigned int wqe_shift; + unsigned int shift; /* wq size is 2^shift */ + int offset; ++ void *db_reg; + }; + + /* record the result of sge process */ +diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c +index f102fd61..9cbc0aac 100644 +--- a/providers/hns/hns_roce_u_hw_v2.c ++++ b/providers/hns/hns_roce_u_hw_v2.c +@@ -308,7 +308,7 @@ static void hns_roce_update_sq_db(struct hns_roce_context *ctx, + + sq_db.parameter = htole32(qp->sq.head); + roce_set_field(sq_db.parameter, DB_PARAM_SL_M, DB_PARAM_SL_S, qp->sl); +- hns_roce_write64(ctx->uar + ROCEE_VF_DB_CFG0_OFFSET, (__le32 *)&sq_db); ++ hns_roce_write64(qp->sq.db_reg, (__le32 *)&sq_db); + } + + static void hns_roce_write512(uint64_t *dest, uint64_t *val) +@@ -329,7 +329,7 @@ static void hns_roce_write_dwqe(struct hns_roce_qp *qp, void *wqe) + roce_set_field(rc_sq_wqe->byte_4, RC_SQ_WQE_BYTE_4_WQE_INDEX_M, + RC_SQ_WQE_BYTE_4_WQE_INDEX_S, qp->sq.head); + +- hns_roce_write512(qp->dwqe_page, wqe); ++ hns_roce_write512(qp->sq.db_reg, wqe); + } + + static void hns_roce_v2_update_cq_cons_index(struct hns_roce_context *ctx, +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index fc902815..c5022c83 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -1117,6 +1117,11 @@ static void qp_setup_config(struct ibv_qp_init_attr_ex *attr, + } + + qp->max_inline_data = attr->cap.max_inline_data; ++ ++ if (qp->flags & HNS_ROCE_QP_CAP_DIRECT_WQE) ++ qp->sq.db_reg = qp->dwqe_page; ++ else ++ qp->sq.db_reg = ctx->uar + ROCEE_VF_DB_CFG0_OFFSET; + } + + void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx) +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index 1f809cc5f5a6cf24be6f50639a90cb8f236c753f..467159a0cadb47036cad9d9a70634d20b9470fed 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,11 +1,20 @@ Name: rdma-core Version: 35.1 -Release: 1 +Release: 2 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core Source: https://github.com/linux-rdma/rdma-core/releases/download/v%{version}/%{name}-%{version}.tar.gz + Patch0: backport-fixbug-increase-maximum-number-of-cpus-rdma.patch +Patch1: 0001-Update-kernel-headers.patch +Patch2: 0002-libhns-Fix-the-ownership-of-the-head-tail-pointer-of.patch +Patch3: 0003-libhns-Fix-wrong-data-type-when-writing-doorbell.patch +Patch4: 0004-libhns-Remove-unsupported-QP-type.patch +Patch5: 0005-libhns-Avoid-using-WQE-indexes-that-exceed-the-SRQ-s.patch +Patch6: 0006-libhns-Don-t-create-RQ-for-a-QP-that-associated-with.patch +Patch7: 0007-libhns-Add-support-for-direct-wqe.patch +Patch8: 0008-libhns-Use-new-SQ-doorbell-register-for-HIP09.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -250,6 +259,12 @@ fi %{_mandir}/* %changelog +* Thu Jan 10 2022 tangchengchang - 35.1-2 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Add support for hns DWQE + * Thu Dec 09 2021 gaihuiying - 35.1-1 - Type: requirement - ID: NA