From 9e5f5d39757a5479a1a4e1170978d2e09acb995b Mon Sep 17 00:00:00 2001
From: Luoyouming <luoyouming@huawei.com>
Date: Fri, 9 Sep 2022 17:42:38 +0800
Subject: [PATCH v4 07/10] libhns: Refactor rq inline

The ibv_sge struct is enough, there is no need to customize the
hns_roce_rinl_sge struct. Refactored structures and functions
for reuse in cqe inline (rq, srq scenarios).

Signed-off-by: Luoyouming <luoyouming@huawei.com>
---
 providers/hns/hns_roce_u.h       |   7 +--
 providers/hns/hns_roce_u_hw_v2.c | 103 +++++++++++++++----------------
 providers/hns/hns_roce_u_verbs.c |  46 +++++++-------
 3 files changed, 75 insertions(+), 81 deletions(-)

diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 5388f9c..57ebe55 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -290,13 +290,8 @@ struct hns_roce_sge_ex {
 	unsigned int			sge_shift;
 };
 
-struct hns_roce_rinl_sge {
-	void				*addr;
-	unsigned int			len;
-};
-
 struct hns_roce_rinl_wqe {
-	struct hns_roce_rinl_sge	*sg_list;
+	struct ibv_sge			*sg_list;
 	unsigned int			sge_cnt;
 };
 
diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index ebe68bc..73acc9e 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -34,6 +34,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/mman.h>
+#include <ccan/minmax.h>
 #include "hns_roce_u.h"
 #include "hns_roce_u_db.h"
 #include "hns_roce_u_hw_v2.h"
@@ -417,46 +418,42 @@ static void get_opcode_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
 	wc->opcode = wc_rcv_op_map[opcode];
 }
 
-static int handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
-			       struct hns_roce_qp **cur_qp, uint32_t opcode)
+static void handle_recv_inl_data(struct hns_roce_v2_cqe *cqe,
+				 struct hns_roce_rinl_buf *rinl_buf,
+				 uint32_t wr_cnt, uint8_t *buf)
 {
-	if (((*cur_qp)->verbs_qp.qp.qp_type == IBV_QPT_RC) &&
-	    (opcode == HNS_ROCE_RECV_OP_SEND ||
-	     opcode == HNS_ROCE_RECV_OP_SEND_WITH_IMM ||
-	     opcode == HNS_ROCE_RECV_OP_SEND_WITH_INV) &&
-	     hr_reg_read(cqe, CQE_RQ_INLINE)) {
-		struct hns_roce_rinl_sge *sge_list;
-		uint32_t wr_num, wr_cnt, sge_num, data_len;
-		uint8_t *wqe_buf;
-		uint32_t sge_cnt, size;
+	struct ibv_sge *sge_list;
+	uint32_t sge_num, data_len;
+	uint32_t sge_cnt, size;
 
-		wr_num = hr_reg_read(cqe, CQE_WQE_IDX);
-		wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
+	sge_list = rinl_buf->wqe_list[wr_cnt].sg_list;
+	sge_num = rinl_buf->wqe_list[wr_cnt].sge_cnt;
 
-		sge_list = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sg_list;
-		sge_num = (*cur_qp)->rq_rinl_buf.wqe_list[wr_cnt].sge_cnt;
-		wqe_buf = (uint8_t *)get_recv_wqe_v2(*cur_qp, wr_cnt);
+	data_len = le32toh(cqe->byte_cnt);
 
-		data_len = wc->byte_len;
+	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
+		size = min(sge_list[sge_cnt].length, data_len);
 
-		for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len);
-		     sge_cnt++) {
-			size = sge_list[sge_cnt].len < data_len ?
-			       sge_list[sge_cnt].len : data_len;
+		memcpy((void *)(uintptr_t)sge_list[sge_cnt].addr, (void *)buf, size);
+		data_len -= size;
+		buf += size;
+	}
 
-			memcpy((void *)sge_list[sge_cnt].addr,
-				(void *)wqe_buf, size);
-			data_len -= size;
-			wqe_buf += size;
-		}
+	if (data_len)
+		hr_reg_write(cqe, CQE_STATUS, HNS_ROCE_V2_CQE_LOCAL_LENGTH_ERR);
 
-		if (data_len) {
-			wc->status = IBV_WC_LOC_LEN_ERR;
-			return V2_CQ_POLL_ERR;
-		}
-	}
+}
 
-	return V2_CQ_OK;
+static void handle_recv_rq_inl(struct hns_roce_v2_cqe *cqe,
+			       struct hns_roce_qp *cur_qp)
+{
+	uint8_t *wqe_buf;
+	uint32_t wr_num;
+
+	wr_num = hr_reg_read(cqe, CQE_WQE_IDX) & (cur_qp->rq.wqe_cnt - 1);
+
+	wqe_buf = (uint8_t *)get_recv_wqe_v2(cur_qp, wr_num);
+	handle_recv_inl_data(cqe, &(cur_qp->rq_rinl_buf), wr_num, wqe_buf);
 }
 
 static void parse_for_ud_qp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc)
@@ -479,10 +476,9 @@ static void parse_cqe_for_srq(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
 }
 
 static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
-			       struct hns_roce_qp *hr_qp, uint8_t opcode)
+			       struct hns_roce_qp *hr_qp)
 {
 	struct hns_roce_wq *wq;
-	int ret;
 
 	wq = &hr_qp->rq;
 	wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
@@ -491,12 +487,8 @@ static int parse_cqe_for_resp(struct hns_roce_v2_cqe *cqe, struct ibv_wc *wc,
 	if (hr_qp->verbs_qp.qp.qp_type == IBV_QPT_UD)
 		parse_for_ud_qp(cqe, wc);
 
-	ret = handle_recv_inl_wqe(cqe, wc, &hr_qp, opcode);
-	if (ret) {
-		verbs_err(verbs_get_ctx(hr_qp->verbs_qp.qp.context),
-			  PFX "failed to handle recv inline wqe!\n");
-		return ret;
-	}
+	if (hr_reg_read(cqe, CQE_RQ_INLINE))
+		handle_recv_rq_inl(cqe, hr_qp);
 
 	return 0;
 }
@@ -626,7 +618,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq
 		if (srq)
 			parse_cqe_for_srq(cqe, wc, srq);
 		else
-			parse_cqe_for_resp(cqe, wc, cur_qp, opcode);
+			parse_cqe_for_resp(cqe, wc, cur_qp);
 	}
 
 	return 0;
@@ -1355,26 +1347,31 @@ static void fill_recv_sge_to_wqe(struct ibv_recv_wr *wr, void *wqe,
 	}
 }
 
+static void fill_recv_inl_buf(struct hns_roce_rinl_buf *rinl_buf,
+			     unsigned int wqe_idx, struct ibv_recv_wr *wr)
+{
+	struct ibv_sge *sge_list;
+	unsigned int i;
+
+	if (!rinl_buf->wqe_cnt)
+		return;
+
+	sge_list = rinl_buf->wqe_list[wqe_idx].sg_list;
+	rinl_buf->wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge;
+	for (i = 0; i < wr->num_sge; i++)
+		memcpy((void *)&sge_list[i], (void *)&wr->sg_list[i],
+			sizeof(struct ibv_sge));
+}
+
 static void fill_rq_wqe(struct hns_roce_qp *qp, struct ibv_recv_wr *wr,
 			unsigned int wqe_idx, unsigned int max_sge)
 {
-	struct hns_roce_rinl_sge *sge_list;
-	unsigned int i;
 	void *wqe;
 
 	wqe = get_recv_wqe_v2(qp, wqe_idx);
 	fill_recv_sge_to_wqe(wr, wqe, max_sge, qp->rq.rsv_sge);
 
-	if (!qp->rq_rinl_buf.wqe_cnt)
-		return;
-
-	/* QP support receive inline wqe */
-	sge_list = qp->rq_rinl_buf.wqe_list[wqe_idx].sg_list;
-	qp->rq_rinl_buf.wqe_list[wqe_idx].sge_cnt = (unsigned int)wr->num_sge;
-	for (i = 0; i < wr->num_sge; i++) {
-		sge_list[i].addr = (void *)(uintptr_t)wr->sg_list[i].addr;
-		sge_list[i].len = wr->sg_list[i].length;
-	}
+	fill_recv_inl_buf(&qp->rq_rinl_buf, wqe_idx, wr);
 }
 
 static int hns_roce_u_v2_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 3e9a306..1d661dd 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -855,43 +855,45 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx,
 	return verify_qp_create_cap(ctx, attr);
 }
 
-static int qp_alloc_recv_inl_buf(struct ibv_qp_cap *cap,
-				 struct hns_roce_qp *qp)
+static int alloc_recv_rinl_buf(uint32_t max_sge,
+				 struct hns_roce_rinl_buf *rinl_buf)
 {
 	unsigned int cnt;
 	int i;
 
-	cnt = qp->rq_rinl_buf.wqe_cnt;
-	qp->rq_rinl_buf.wqe_list = calloc(cnt,
-					  sizeof(struct hns_roce_rinl_wqe));
-	if (!qp->rq_rinl_buf.wqe_list)
+	cnt = rinl_buf->wqe_cnt;
+	rinl_buf->wqe_list = calloc(cnt,
+				   sizeof(struct hns_roce_rinl_wqe));
+	if (!rinl_buf->wqe_list)
 		return ENOMEM;
 
-	qp->rq_rinl_buf.wqe_list[0].sg_list = calloc(cnt * cap->max_recv_sge,
-					sizeof(struct hns_roce_rinl_sge));
-	if (!qp->rq_rinl_buf.wqe_list[0].sg_list)
+	rinl_buf->wqe_list[0].sg_list = calloc(cnt * max_sge,
+					      sizeof(struct ibv_sge));
+	if (!rinl_buf->wqe_list[0].sg_list) {
+		free(rinl_buf->wqe_list);
 		return ENOMEM;
+	}
 
 	for (i = 0; i < cnt; i++) {
-		int wqe_size = i * cap->max_recv_sge;
+		int wqe_size = i * max_sge;
 
-		qp->rq_rinl_buf.wqe_list[i].sg_list =
-			  &(qp->rq_rinl_buf.wqe_list[0].sg_list[wqe_size]);
+		rinl_buf->wqe_list[i].sg_list =
+			  &(rinl_buf->wqe_list[0].sg_list[wqe_size]);
 	}
 
 	return 0;
 }
 
-static void qp_free_recv_inl_buf(struct hns_roce_qp *qp)
+static void free_recv_rinl_buf(struct hns_roce_rinl_buf *rinl_buf)
 {
-	if (qp->rq_rinl_buf.wqe_list) {
-		if (qp->rq_rinl_buf.wqe_list[0].sg_list) {
-			free(qp->rq_rinl_buf.wqe_list[0].sg_list);
-			qp->rq_rinl_buf.wqe_list[0].sg_list = NULL;
+	if (rinl_buf->wqe_list) {
+		if (rinl_buf->wqe_list[0].sg_list) {
+			free(rinl_buf->wqe_list[0].sg_list);
+			rinl_buf->wqe_list[0].sg_list = NULL;
 		}
 
-		free(qp->rq_rinl_buf.wqe_list);
-		qp->rq_rinl_buf.wqe_list = NULL;
+		free(rinl_buf->wqe_list);
+		rinl_buf->wqe_list = NULL;
 	}
 }
 
@@ -930,7 +932,7 @@ static int calc_qp_buff_size(struct hns_roce_device *hr_dev,
 
 static void qp_free_wqe(struct hns_roce_qp *qp)
 {
-	qp_free_recv_inl_buf(qp);
+	free_recv_rinl_buf(&qp->rq_rinl_buf);
 	if (qp->sq.wqe_cnt)
 		free(qp->sq.wrid);
 
@@ -958,7 +960,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp,
 	}
 
 	if (qp->rq_rinl_buf.wqe_cnt) {
-		if (qp_alloc_recv_inl_buf(cap, qp))
+		if (alloc_recv_rinl_buf(cap->max_recv_sge, &qp->rq_rinl_buf))
 			goto err_alloc;
 	}
 
@@ -968,7 +970,7 @@ static int qp_alloc_wqe(struct ibv_qp_cap *cap, struct hns_roce_qp *qp,
 	return 0;
 
 err_alloc:
-	qp_free_recv_inl_buf(qp);
+	free_recv_rinl_buf(&qp->rq_rinl_buf);
 	if (qp->rq.wrid)
 		free(qp->rq.wrid);
 
-- 
2.30.0