diff --git a/0003-Update-kernel-headers.patch b/0003-Update-kernel-headers.patch new file mode 100644 index 0000000000000000000000000000000000000000..2bae99845031c7ffefd9b8b3c3b795c763c50c24 --- /dev/null +++ b/0003-Update-kernel-headers.patch @@ -0,0 +1,57 @@ +From 75fdc5c9a0cce5cc6859189eeb91ce1846edc4b9 Mon Sep 17 00:00:00 2001 +From: Junxian Huang +Date: Tue, 5 Mar 2024 13:57:20 +0800 +Subject: [PATCH 3/7] Update kernel headers + +To commit: 6ec429d5887a ("RDMA/hns: Support userspace configuring +congestion control algorithm with QP granularity"). + +Signed-off-by: Junxian Huang +--- + kernel-headers/rdma/hns-abi.h | 16 ++++++++++++++++ + 1 file changed, 16 insertions(+) + +diff --git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h +index f77697c..39ed8a4 100644 +--- a/kernel-headers/rdma/hns-abi.h ++++ b/kernel-headers/rdma/hns-abi.h +@@ -73,6 +73,17 @@ struct hns_roce_ib_create_srq_resp { + __u32 cap_flags; /* Use enum hns_roce_srq_cap_flags */ + }; + ++enum hns_roce_congest_type_flags { ++ HNS_ROCE_CREATE_QP_FLAGS_DCQCN, ++ HNS_ROCE_CREATE_QP_FLAGS_LDCP, ++ HNS_ROCE_CREATE_QP_FLAGS_HC3, ++ HNS_ROCE_CREATE_QP_FLAGS_DIP, ++}; ++ ++enum hns_roce_create_qp_comp_mask { ++ HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 0, ++}; ++ + struct hns_roce_ib_create_qp { + __aligned_u64 buf_addr; + __aligned_u64 db_addr; +@@ -81,6 +92,9 @@ struct hns_roce_ib_create_qp { + __u8 sq_no_prefetch; + __u8 reserved[5]; + __aligned_u64 sdb_addr; ++ __aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */ ++ __aligned_u64 create_flags; ++ __aligned_u64 cong_type_flags; + }; + + enum hns_roce_qp_cap_flags { +@@ -120,6 +134,8 @@ struct hns_roce_ib_alloc_ucontext_resp { + __u32 reserved; + __u32 config; + __u32 max_inline_data; ++ __u8 congest_type; ++ __u8 reserved0[7]; + }; + + struct hns_roce_ib_alloc_ucontext { +-- +2.33.0 + diff --git a/0004-libhns-Introduce-hns-direct-verbs.patch b/0004-libhns-Introduce-hns-direct-verbs.patch new file mode 100644 index 0000000000000000000000000000000000000000..0a044920491f73b487f56555c2a68ab9bb5c6e2f --- /dev/null +++ b/0004-libhns-Introduce-hns-direct-verbs.patch @@ -0,0 +1,466 @@ +From 4822f5d7166996c1a619f7c51d156a029e85dd53 Mon Sep 17 00:00:00 2001 +From: Junxian Huang +Date: Tue, 5 Mar 2024 13:57:22 +0800 +Subject: [PATCH 4/7] libhns: Introduce hns direct verbs + +driver inclusion +category: feature +bugzilla: https://gitee.com/openeuler/kernel/issues/I95UWO + +------------------------------------------------------------------ +Introduce the frame of hns direct verbs, including hnsdv_is_supported(), +hnsdv_create_qp() and hnsdv_query_device(). + +Signed-off-by: Junxian Huang +Signed-off-by: Ran Zhou +--- + debian/ibverbs-providers.install | 1 + + debian/ibverbs-providers.lintian-overrides | 4 +- + debian/ibverbs-providers.symbols | 6 ++ + debian/libibverbs-dev.install | 4 ++ + providers/hns/CMakeLists.txt | 9 ++- + providers/hns/hns_roce_u.c | 13 ++++ + providers/hns/hns_roce_u.h | 2 + + providers/hns/hns_roce_u_abi.h | 1 + + providers/hns/hns_roce_u_verbs.c | 69 ++++++++++++++++++++-- + providers/hns/hnsdv.h | 37 ++++++++++++ + providers/hns/libhns.map | 9 +++ + redhat/rdma-core.spec | 5 +- + suse/rdma-core.spec | 21 ++++++- + 13 files changed, 171 insertions(+), 10 deletions(-) + create mode 100644 providers/hns/hnsdv.h + create mode 100644 providers/hns/libhns.map + +diff --git a/debian/ibverbs-providers.install b/debian/ibverbs-providers.install +index a003a30..fea15e0 100644 +--- a/debian/ibverbs-providers.install ++++ b/debian/ibverbs-providers.install +@@ -1,6 +1,7 @@ + etc/libibverbs.d/ + usr/lib/*/libefa.so.* + usr/lib/*/libibverbs/lib*-rdmav*.so ++usr/lib/*/libhns.so.* + usr/lib/*/libmana.so.* + usr/lib/*/libmlx4.so.* + usr/lib/*/libmlx5.so.* +diff --git a/debian/ibverbs-providers.lintian-overrides b/debian/ibverbs-providers.lintian-overrides +index 5815058..fd73a76 100644 +--- a/debian/ibverbs-providers.lintian-overrides ++++ b/debian/ibverbs-providers.lintian-overrides +@@ -1,2 +1,2 @@ +-# libefa, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions. +-ibverbs-providers: package-name-doesnt-match-sonames libefa1 libmana1 libmlx4-1 libmlx5-1 ++# libefa, libhns, libmana, libmlx4 and libmlx5 are ibverbs provider that provides more functions. ++ibverbs-providers: package-name-doesnt-match-sonames libefa1 libhns-1 libmana1 libmlx4-1 libmlx5-1 +diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols +index 72361bd..d2c0989 100644 +--- a/debian/ibverbs-providers.symbols ++++ b/debian/ibverbs-providers.symbols +@@ -174,6 +174,12 @@ libefa.so.1 ibverbs-providers #MINVER# + efadv_cq_from_ibv_cq_ex@EFA_1.2 43 + efadv_create_cq@EFA_1.2 43 + efadv_query_mr@EFA_1.3 50 ++libhns.so.1 ibverbs-providers #MINVER# ++* Build-Depends-Package: libibverbs-dev ++ HNS_1.0@HNS_1.0 51 ++ hnsdv_is_supported@HNS_1.0 51 ++ hnsdv_create_qp@HNS_1.0 51 ++ hnsdv_query_device@HNS_1.0 51 + libmana.so.1 ibverbs-providers #MINVER# + * Build-Depends-Package: libibverbs-dev + MANA_1.0@MANA_1.0 41 +diff --git a/debian/libibverbs-dev.install b/debian/libibverbs-dev.install +index 5f2ffd5..ef5b9a4 100644 +--- a/debian/libibverbs-dev.install ++++ b/debian/libibverbs-dev.install +@@ -1,5 +1,6 @@ + usr/include/infiniband/arch.h + usr/include/infiniband/efadv.h ++usr/include/infiniband/hnsdv.h + usr/include/infiniband/ib_user_ioctl_verbs.h + usr/include/infiniband/manadv.h + usr/include/infiniband/mlx4dv.h +@@ -15,6 +16,8 @@ usr/include/infiniband/verbs_api.h + usr/lib/*/lib*-rdmav*.a + usr/lib/*/libefa.a + usr/lib/*/libefa.so ++usr/lib/*/libhns.a ++usr/lib/*/libhns.so + usr/lib/*/libibverbs*.so + usr/lib/*/libibverbs.a + usr/lib/*/libmana.a +@@ -24,6 +27,7 @@ usr/lib/*/libmlx4.so + usr/lib/*/libmlx5.a + usr/lib/*/libmlx5.so + usr/lib/*/pkgconfig/libefa.pc ++usr/lib/*/pkgconfig/libhns.pc + usr/lib/*/pkgconfig/libibverbs.pc + usr/lib/*/pkgconfig/libmana.pc + usr/lib/*/pkgconfig/libmlx4.pc +diff --git a/providers/hns/CMakeLists.txt b/providers/hns/CMakeLists.txt +index 7aaca75..58139ae 100644 +--- a/providers/hns/CMakeLists.txt ++++ b/providers/hns/CMakeLists.txt +@@ -1,7 +1,14 @@ +-rdma_provider(hns ++rdma_shared_provider(hns libhns.map ++ 1 1.0.${PACKAGE_VERSION} + hns_roce_u.c + hns_roce_u_buf.c + hns_roce_u_db.c + hns_roce_u_hw_v2.c + hns_roce_u_verbs.c + ) ++ ++publish_headers(infiniband ++ hnsdv.h ++) ++ ++rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 266e73e..0b254fb 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -216,4 +216,17 @@ static const struct verbs_device_ops hns_roce_dev_ops = { + .uninit_device = hns_uninit_device, + .alloc_context = hns_roce_alloc_context, + }; ++ ++bool is_hns_dev(struct ibv_device *device) ++{ ++ struct verbs_device *verbs_device = verbs_get_device(device); ++ ++ return verbs_device->ops == &hns_roce_dev_ops; ++} ++ ++bool hnsdv_is_supported(struct ibv_device *device) ++{ ++ return is_hns_dev(device); ++} ++ + PROVIDER_DRIVER(hns, hns_roce_dev_ops); +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 5ec2734..99fa23f 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -501,6 +501,8 @@ void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx); + + void hns_roce_init_qp_indices(struct hns_roce_qp *qp); + ++bool is_hns_dev(struct ibv_device *device); ++ + extern const struct hns_roce_u_hw hns_roce_u_hw_v2; + + #endif /* _HNS_ROCE_U_H */ +diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h +index ec47c4b..7e9bbc1 100644 +--- a/providers/hns/hns_roce_u_abi.h ++++ b/providers/hns/hns_roce_u_abi.h +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include "hnsdv.h" + + DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, + empty, hns_roce_ib_alloc_pd_resp); +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index d081bb3..997b7e0 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -785,6 +785,25 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) + return 0; + } + ++enum { ++ HNSDV_QP_SUP_COMP_MASK = 0, ++}; ++ ++static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, ++ struct hnsdv_qp_init_attr *hns_attr) ++{ ++ if (!hns_attr) ++ return 0; ++ ++ if (!check_comp_mask(hns_attr->comp_mask, HNSDV_QP_SUP_COMP_MASK)) { ++ verbs_err(&ctx->ibv_ctx, "invalid hnsdv comp_mask 0x%x.\n", ++ hns_attr->comp_mask); ++ return EINVAL; ++ } ++ ++ return 0; ++} ++ + enum { + CREATE_QP_SUP_COMP_MASK = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_XRCD | + IBV_QP_INIT_ATTR_SEND_OPS_FLAGS, +@@ -866,7 +885,8 @@ static int verify_qp_create_cap(struct hns_roce_context *ctx, + } + + static int verify_qp_create_attr(struct hns_roce_context *ctx, +- struct ibv_qp_init_attr_ex *attr) ++ struct ibv_qp_init_attr_ex *attr, ++ struct hnsdv_qp_init_attr *hns_attr) + { + int ret; + +@@ -874,6 +894,10 @@ static int verify_qp_create_attr(struct hns_roce_context *ctx, + if (ret) + return ret; + ++ ret = check_hnsdv_qp_attr(ctx, hns_attr); ++ if (ret) ++ return ret; ++ + return verify_qp_create_cap(ctx, attr); + } + +@@ -1274,14 +1298,15 @@ static int mmap_dwqe(struct ibv_context *ibv_ctx, struct hns_roce_qp *qp, + } + + static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, +- struct ibv_qp_init_attr_ex *attr) ++ struct ibv_qp_init_attr_ex *attr, ++ struct hnsdv_qp_init_attr *hns_attr) + { + struct hns_roce_context *context = to_hr_ctx(ibv_ctx); + struct hns_roce_qp *qp; + uint64_t dwqe_mmap_key; + int ret; + +- ret = verify_qp_create_attr(context, attr); ++ ret = verify_qp_create_attr(context, attr, hns_attr); + if (ret) + goto err; + +@@ -1345,7 +1370,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd, + attrx.comp_mask = IBV_QP_INIT_ATTR_PD; + attrx.pd = pd; + +- qp = create_qp(pd->context, &attrx); ++ qp = create_qp(pd->context, &attrx, NULL); + if (qp) + memcpy(attr, &attrx, sizeof(*attr)); + +@@ -1355,7 +1380,41 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd, + struct ibv_qp *hns_roce_u_create_qp_ex(struct ibv_context *context, + struct ibv_qp_init_attr_ex *attr) + { +- return create_qp(context, attr); ++ return create_qp(context, attr, NULL); ++} ++ ++struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, ++ struct ibv_qp_init_attr_ex *qp_attr, ++ struct hnsdv_qp_init_attr *hns_attr) ++{ ++ if (!context || !qp_attr) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (!is_hns_dev(context->device)) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ++ return create_qp(context, qp_attr, hns_attr); ++} ++ ++int hnsdv_query_device(struct ibv_context *context, ++ struct hnsdv_context *attrs_out) ++{ ++ struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; ++ ++ if (!ctx || !attrs_out) ++ return EINVAL; ++ ++ if (!is_hns_dev(context->device)) { ++ verbs_err(verbs_get_ctx(context), "not a HNS RoCE device!\n"); ++ return EOPNOTSUPP; ++ } ++ memset(attrs_out, 0, sizeof(*attrs_out)); ++ ++ return 0; + } + + struct ibv_qp *hns_roce_u_open_qp(struct ibv_context *context, +diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h +new file mode 100644 +index 0000000..49ba08a +--- /dev/null ++++ b/providers/hns/hnsdv.h +@@ -0,0 +1,37 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ ++/* ++ * Copyright (c) 2024 Hisilicon Limited. ++ */ ++ ++#ifndef __HNSDV_H__ ++#define __HNSDV_H__ ++ ++#include ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++struct hnsdv_qp_init_attr { ++ uint64_t comp_mask; ++}; ++ ++struct hnsdv_context { ++ uint64_t comp_mask; ++}; ++ ++bool hnsdv_is_supported(struct ibv_device *device); ++int hnsdv_query_device(struct ibv_context *ctx_in, ++ struct hnsdv_context *attrs_out); ++struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, ++ struct ibv_qp_init_attr_ex *qp_attr, ++ struct hnsdv_qp_init_attr *hns_qp_attr); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __HNSDV_H__ */ +diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map +new file mode 100644 +index 0000000..e9bf417 +--- /dev/null ++++ b/providers/hns/libhns.map +@@ -0,0 +1,9 @@ ++/* Export symbols should be added below according to ++ Documentation/versioning.md document. */ ++HNS_1.0 { ++ global: ++ hnsdv_is_supported; ++ hnsdv_create_qp; ++ hnsdv_query_device; ++ local: *; ++}; +diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec +index c6ddcfd..c347195 100644 +--- a/redhat/rdma-core.spec ++++ b/redhat/rdma-core.spec +@@ -158,6 +158,8 @@ Provides: liberdma = %{version}-%{release} + Obsoletes: liberdma < %{version}-%{release} + Provides: libhfi1 = %{version}-%{release} + Obsoletes: libhfi1 < %{version}-%{release} ++Provides: libhns = %{version}-%{release} ++Obsoletes: libhns < %{version}-%{release} + Provides: libipathverbs = %{version}-%{release} + Obsoletes: libipathverbs < %{version}-%{release} + Provides: libirdma = %{version}-%{release} +@@ -188,7 +190,7 @@ Device-specific plug-in ibverbs userspace drivers are included: + - libefa: Amazon Elastic Fabric Adapter + - liberdma: Alibaba Elastic RDMA (iWarp) Adapter + - libhfi1: Intel Omni-Path HFI +-- libhns: HiSilicon Hip06 SoC ++- libhns: HiSilicon Hip08+ SoC + - libipathverbs: QLogic InfiniPath HCA + - libirdma: Intel Ethernet Connection RDMA + - libmana: Microsoft Azure Network Adapter +@@ -575,6 +577,7 @@ fi + %dir %{_sysconfdir}/libibverbs.d + %dir %{_libdir}/libibverbs + %{_libdir}/libefa.so.* ++%{_libdir}/libhns.so.* + %{_libdir}/libibverbs*.so.* + %{_libdir}/libibverbs/*.so + %{_libdir}/libmana.so.* +diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec +index d534dbc..f2bd0c0 100644 +--- a/suse/rdma-core.spec ++++ b/suse/rdma-core.spec +@@ -35,6 +35,7 @@ License: BSD-2-Clause OR GPL-2.0-only + Group: Productivity/Networking/Other + + %define efa_so_major 1 ++%define hns_so_major 1 + %define verbs_so_major 1 + %define rdmacm_so_major 1 + %define umad_so_major 3 +@@ -45,6 +46,7 @@ Group: Productivity/Networking/Other + %define mad_major 5 + + %define efa_lname libefa%{efa_so_major} ++%define hns_lname libhns%{hns_so_major} + %define verbs_lname libibverbs%{verbs_so_major} + %define rdmacm_lname librdmacm%{rdmacm_so_major} + %define umad_lname libibumad%{umad_so_major} +@@ -159,6 +161,7 @@ Requires: %{umad_lname} = %{version}-%{release} + Requires: %{verbs_lname} = %{version}-%{release} + %if 0%{?dma_coherent} + Requires: %{efa_lname} = %{version}-%{release} ++Requires: %{hns_lname} = %{version}-%{release} + Requires: %{mana_lname} = %{version}-%{release} + Requires: %{mlx4_lname} = %{version}-%{release} + Requires: %{mlx5_lname} = %{version}-%{release} +@@ -200,6 +203,7 @@ Requires: %{name}%{?_isa} = %{version}-%{release} + Obsoletes: libcxgb4-rdmav2 < %{version}-%{release} + Obsoletes: libefa-rdmav2 < %{version}-%{release} + Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release} ++Obsoletes: libhns-rdmav2 < %{version}-%{release} + Obsoletes: libipathverbs-rdmav2 < %{version}-%{release} + Obsoletes: libmana-rdmav2 < %{version}-%{release} + Obsoletes: libmlx4-rdmav2 < %{version}-%{release} +@@ -209,6 +213,7 @@ Obsoletes: libocrdma-rdmav2 < %{version}-%{release} + Obsoletes: librxe-rdmav2 < %{version}-%{release} + %if 0%{?dma_coherent} + Requires: %{efa_lname} = %{version}-%{release} ++Requires: %{hns_lname} = %{version}-%{release} + Requires: %{mana_lname} = %{version}-%{release} + Requires: %{mlx4_lname} = %{version}-%{release} + Requires: %{mlx5_lname} = %{version}-%{release} +@@ -228,7 +233,7 @@ Device-specific plug-in ibverbs userspace drivers are included: + - libcxgb4: Chelsio T4 iWARP HCA + - libefa: Amazon Elastic Fabric Adapter + - libhfi1: Intel Omni-Path HFI +-- libhns: HiSilicon Hip06 SoC ++- libhns: HiSilicon Hip08+ SoC + - libipathverbs: QLogic InfiniPath HCA + - libirdma: Intel Ethernet Connection RDMA + - libmana: Microsoft Azure Network Adapter +@@ -256,6 +261,13 @@ Group: System/Libraries + %description -n %efa_lname + This package contains the efa runtime library. + ++%package -n %hns_lname ++Summary: HNS runtime library ++Group: System/Libraries ++ ++%description -n %hns_lname ++This package contains the hns runtime library. ++ + %package -n %mana_lname + Summary: MANA runtime library + Group: System/Libraries +@@ -508,6 +520,9 @@ rm -rf %{buildroot}/%{_sbindir}/srp_daemon.sh + %post -n %efa_lname -p /sbin/ldconfig + %postun -n %efa_lname -p /sbin/ldconfig + ++%post -n %hns_lname -p /sbin/ldconfig ++%postun -n %hns_lname -p /sbin/ldconfig ++ + %post -n %mana_lname -p /sbin/ldconfig + %postun -n %mana_lname -p /sbin/ldconfig + +@@ -700,6 +715,10 @@ done + %files -n %efa_lname + %{_libdir}/libefa*.so.* + ++%files -n %hns_lname ++%defattr(-,root,root) ++%{_libdir}/libhns*.so.* ++ + %files -n %mana_lname + %{_libdir}/libmana*.so.* + +-- +2.33.0 + diff --git a/0005-libhns-Encapsulate-context-attribute-setting-into-a-.patch b/0005-libhns-Encapsulate-context-attribute-setting-into-a-.patch new file mode 100644 index 0000000000000000000000000000000000000000..e53cc220522ddbe0c12cb46d6316090dc880d307 --- /dev/null +++ b/0005-libhns-Encapsulate-context-attribute-setting-into-a-.patch @@ -0,0 +1,139 @@ +From da7f5d66f410f226f1cc0437bb4fc3124fcbb3f3 Mon Sep 17 00:00:00 2001 +From: Junxian Huang +Date: Tue, 5 Mar 2024 13:57:23 +0800 +Subject: [PATCH 6/7] libhns: Encapsulate context attribute setting into a + single function + +driver inclusion +category: feature +bugzilla: https://gitee.com/openeuler/kernel/issues/I95UWO + +------------------------------------------------------------------ +This patch doesn't involve functional changes. Just encapsulate context +attribute setting into a single function set_context_attr() to make +hns_roce_alloc_context() more readable. + +Signed-off-by: Junxian Huang +Signed-off-by: Ran Zhou dd +--- + providers/hns/hns_roce_u.c | 69 ++++++++++++++++++++++---------------- + 1 file changed, 40 insertions(+), 29 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 0b254fb..69f7d3f 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -97,50 +97,33 @@ static uint32_t calc_table_shift(uint32_t entry_count, uint32_t size_shift) + return count_shift > size_shift ? count_shift - size_shift : 0; + } + +-static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, +- int cmd_fd, +- void *private_data) ++static int set_context_attr(struct hns_roce_device *hr_dev, ++ struct hns_roce_context *context, ++ struct hns_roce_alloc_ucontext_resp *resp) + { +- struct hns_roce_device *hr_dev = to_hr_dev(ibdev); +- struct hns_roce_alloc_ucontext_resp resp = {}; +- struct hns_roce_alloc_ucontext cmd = {}; + struct ibv_device_attr dev_attrs; +- struct hns_roce_context *context; + int i; + +- context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, +- RDMA_DRIVER_HNS); +- if (!context) +- return NULL; +- +- cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS | +- HNS_ROCE_CQE_INLINE_FLAGS; +- if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp))) +- goto err_free; +- +- if (!resp.cqe_size) ++ if (!resp->cqe_size) + context->cqe_size = HNS_ROCE_CQE_SIZE; +- else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE) +- context->cqe_size = resp.cqe_size; ++ else if (resp->cqe_size <= HNS_ROCE_V3_CQE_SIZE) ++ context->cqe_size = resp->cqe_size; + else + context->cqe_size = HNS_ROCE_V3_CQE_SIZE; + +- context->config = resp.config; +- if (resp.config & HNS_ROCE_RSP_EXSGE_FLAGS) +- context->max_inline_data = resp.max_inline_data; ++ context->config = resp->config; ++ if (resp->config & HNS_ROCE_RSP_EXSGE_FLAGS) ++ context->max_inline_data = resp->max_inline_data; + +- context->qp_table_shift = calc_table_shift(resp.qp_tab_size, ++ context->qp_table_shift = calc_table_shift(resp->qp_tab_size, + HNS_ROCE_QP_TABLE_BITS); + context->qp_table_mask = (1 << context->qp_table_shift) - 1; +- pthread_mutex_init(&context->qp_table_mutex, NULL); + for (i = 0; i < HNS_ROCE_QP_TABLE_SIZE; ++i) + context->qp_table[i].refcnt = 0; + +- context->srq_table_shift = calc_table_shift(resp.srq_tab_size, ++ context->srq_table_shift = calc_table_shift(resp->srq_tab_size, + HNS_ROCE_SRQ_TABLE_BITS); + context->srq_table_mask = (1 << context->srq_table_shift) - 1; +- pthread_mutex_init(&context->srq_table_mutex, NULL); + for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i) + context->srq_table[i].refcnt = 0; + +@@ -149,7 +132,7 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + struct ibv_device_attr_ex, + orig_attr), + sizeof(dev_attrs))) +- goto err_free; ++ return EIO; + + hr_dev->hw_version = dev_attrs.hw_ver; + context->max_qp_wr = dev_attrs.max_qp_wr; +@@ -158,11 +141,39 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, + context->max_srq_wr = dev_attrs.max_srq_wr; + context->max_srq_sge = dev_attrs.max_srq_sge; + ++ return 0; ++} ++ ++static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, ++ int cmd_fd, ++ void *private_data) ++{ ++ struct hns_roce_device *hr_dev = to_hr_dev(ibdev); ++ struct hns_roce_alloc_ucontext_resp resp = {}; ++ struct hns_roce_alloc_ucontext cmd = {}; ++ struct hns_roce_context *context; ++ ++ context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, ++ RDMA_DRIVER_HNS); ++ if (!context) ++ return NULL; ++ ++ cmd.config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS | ++ HNS_ROCE_CQE_INLINE_FLAGS; ++ if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd), ++ &resp.ibv_resp, sizeof(resp))) ++ goto err_free; ++ ++ if (set_context_attr(hr_dev, context, &resp)) ++ goto err_free; ++ + context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, cmd_fd, 0); + if (context->uar == MAP_FAILED) + goto err_free; + ++ pthread_mutex_init(&context->qp_table_mutex, NULL); ++ pthread_mutex_init(&context->srq_table_mutex, NULL); + pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); + + verbs_set_ops(&context->ibv_ctx, &hns_common_ops); +-- +2.33.0 + diff --git a/0006-libhns-Support-congestion-control-algorithm-configur.patch b/0006-libhns-Support-congestion-control-algorithm-configur.patch new file mode 100644 index 0000000000000000000000000000000000000000..7af97c46d6d1eea5833dfa6d0fb1e453e12ebf61 --- /dev/null +++ b/0006-libhns-Support-congestion-control-algorithm-configur.patch @@ -0,0 +1,185 @@ +From 4cc15f4ef3dadb3219719376822cf427df338f2a Mon Sep 17 00:00:00 2001 +From: Junxian Huang +Date: Tue, 5 Mar 2024 13:57:24 +0800 +Subject: [PATCH 7/7] libhns: Support congestion control algorithm + configuration with direct verbs + +driver inclusion +category: feature +bugzilla: https://gitee.com/openeuler/kernel/issues/I95UWO + +------------------------------------------------------------------ +Add support for configuration of congestion control algorithms in QP +granularity with direct verbs hnsdv_create_qp(). + +Signed-off-by: Junxian Huang +Signed-off-by: Ran Zhou +--- + providers/hns/hns_roce_u.c | 1 + + providers/hns/hns_roce_u.h | 1 + + providers/hns/hns_roce_u_verbs.c | 45 ++++++++++++++++++++++++++++---- + providers/hns/hnsdv.h | 25 ++++++++++++++++-- + 4 files changed, 65 insertions(+), 7 deletions(-) + +diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c +index 69f7d3f..90f250e 100644 +--- a/providers/hns/hns_roce_u.c ++++ b/providers/hns/hns_roce_u.c +@@ -135,6 +135,7 @@ static int set_context_attr(struct hns_roce_device *hr_dev, + return EIO; + + hr_dev->hw_version = dev_attrs.hw_ver; ++ hr_dev->congest_cap = resp->congest_type; + context->max_qp_wr = dev_attrs.max_qp_wr; + context->max_sge = dev_attrs.max_sge; + context->max_cqe = dev_attrs.max_cqe; +diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h +index 99fa23f..c73e5c0 100644 +--- a/providers/hns/hns_roce_u.h ++++ b/providers/hns/hns_roce_u.h +@@ -158,6 +158,7 @@ struct hns_roce_device { + int page_size; + const struct hns_roce_u_hw *u_hw; + int hw_version; ++ uint8_t congest_cap; + }; + + struct hns_roce_buf { +diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c +index 997b7e0..dcdc722 100644 +--- a/providers/hns/hns_roce_u_verbs.c ++++ b/providers/hns/hns_roce_u_verbs.c +@@ -786,7 +786,7 @@ int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq) + } + + enum { +- HNSDV_QP_SUP_COMP_MASK = 0, ++ HNSDV_QP_SUP_COMP_MASK = HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE, + }; + + static int check_hnsdv_qp_attr(struct hns_roce_context *ctx, +@@ -1209,10 +1209,33 @@ static int hns_roce_store_qp(struct hns_roce_context *ctx, + return 0; + } + ++static int to_cmd_cong_type(uint8_t cong_type, __u64 *cmd_cong_type) ++{ ++ switch (cong_type) { ++ case HNSDV_QP_CREATE_ENABLE_DCQCN: ++ *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_DCQCN; ++ break; ++ case HNSDV_QP_CREATE_ENABLE_LDCP: ++ *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_LDCP; ++ break; ++ case HNSDV_QP_CREATE_ENABLE_HC3: ++ *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_HC3; ++ break; ++ case HNSDV_QP_CREATE_ENABLE_DIP: ++ *cmd_cong_type = HNS_ROCE_CREATE_QP_FLAGS_DIP; ++ break; ++ default: ++ return EINVAL; ++ } ++ ++ return 0; ++} ++ + static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + struct hns_roce_qp *qp, + struct hns_roce_context *ctx, +- uint64_t *dwqe_mmap_key) ++ uint64_t *dwqe_mmap_key, ++ struct hnsdv_qp_init_attr *hns_attr) + { + struct hns_roce_create_qp_ex_resp resp_ex = {}; + struct hns_roce_create_qp_ex cmd_ex = {}; +@@ -1224,6 +1247,15 @@ static int qp_exec_create_cmd(struct ibv_qp_init_attr_ex *attr, + cmd_ex.log_sq_stride = qp->sq.wqe_shift; + cmd_ex.log_sq_bb_count = hr_ilog32(qp->sq.wqe_cnt); + ++ if (hns_attr && ++ hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE) { ++ ret = to_cmd_cong_type(hns_attr->congest_type, ++ &cmd_ex.cong_type_flags); ++ if (ret) ++ return ret; ++ cmd_ex.comp_mask |= HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE; ++ } ++ + ret = ibv_cmd_create_qp_ex2(&ctx->ibv_ctx.context, &qp->verbs_qp, attr, + &cmd_ex.ibv_cmd, sizeof(cmd_ex), + &resp_ex.ibv_resp, sizeof(resp_ex)); +@@ -1322,7 +1354,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + if (ret) + goto err_buf; + +- ret = qp_exec_create_cmd(attr, qp, context, &dwqe_mmap_key); ++ ret = qp_exec_create_cmd(attr, qp, context, &dwqe_mmap_key, hns_attr); + if (ret) + goto err_cmd; + +@@ -1403,9 +1435,9 @@ struct ibv_qp *hnsdv_create_qp(struct ibv_context *context, + int hnsdv_query_device(struct ibv_context *context, + struct hnsdv_context *attrs_out) + { +- struct hns_roce_context *ctx = context ? to_hr_ctx(context) : NULL; ++ struct hns_roce_device *hr_dev = to_hr_dev(context->device); + +- if (!ctx || !attrs_out) ++ if (!hr_dev || !attrs_out) + return EINVAL; + + if (!is_hns_dev(context->device)) { +@@ -1414,6 +1446,9 @@ int hnsdv_query_device(struct ibv_context *context, + } + memset(attrs_out, 0, sizeof(*attrs_out)); + ++ attrs_out->comp_mask |= HNSDV_CONTEXT_MASK_CONGEST_TYPE; ++ attrs_out->congest_type = hr_dev->congest_cap; ++ + return 0; + } + +diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h +index 49ba08a..451b26e 100644 +--- a/providers/hns/hnsdv.h ++++ b/providers/hns/hnsdv.h +@@ -15,12 +15,33 @@ + extern "C" { + #endif + ++enum hnsdv_qp_congest_ctrl_type { ++ HNSDV_QP_CREATE_ENABLE_DCQCN = 1 << 0, ++ HNSDV_QP_CREATE_ENABLE_LDCP = 1 << 1, ++ HNSDV_QP_CREATE_ENABLE_HC3 = 1 << 2, ++ HNSDV_QP_CREATE_ENABLE_DIP = 1 << 3, ++}; ++ ++enum hnsdv_qp_init_attr_mask { ++ HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE = 1 << 1, ++}; ++ + struct hnsdv_qp_init_attr { +- uint64_t comp_mask; ++ uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */ ++ uint32_t create_flags; ++ uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ ++ uint8_t reserved[3]; ++}; ++ ++enum hnsdv_query_context_comp_mask { ++ HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0, + }; + + struct hnsdv_context { +- uint64_t comp_mask; ++ uint64_t comp_mask; /* Use enum hnsdv_query_context_comp_mask */ ++ uint64_t flags; ++ uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */ ++ uint8_t reserved[7]; + }; + + bool hnsdv_is_supported(struct ibv_device *device); +-- +2.33.0 + diff --git a/rdma-core.spec b/rdma-core.spec index d11610b779ece2a84d42c25fc76a117dd8d235df..01b45ffdcfbd43d8f5dd1a53c96cb9f4d4efb25d 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,13 +1,17 @@ Name: rdma-core Version: 50.0 -Release: 2 +Release: 3 Summary: RDMA core userspace libraries and daemons License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core Source: https://github.com/linux-rdma/rdma-core/releases/download/v%{version}/%{name}-%{version}.tar.gz -Patch1: 0001-Update-kernel-headers.patch -Patch2: 0002-libhns-Support-DSCP.patch +Patch1: 0001-Update-kernel-headers.patch +Patch2: 0002-libhns-Support-DSCP.patch +patch3: 0003-Update-kernel-headers.patch +patch4: 0004-libhns-Introduce-hns-direct-verbs.patch +patch5: 0005-libhns-Encapsulate-context-attribute-setting-into-a-.patch +patch6: 0006-libhns-Support-congestion-control-algorithm-configur.patch BuildRequires: binutils cmake >= 2.8.11 gcc libudev-devel pkgconfig pkgconfig(libnl-3.0) BuildRequires: pkgconfig(libnl-route-3.0) valgrind-devel systemd systemd-devel @@ -480,6 +484,7 @@ fi %dir %{_sysconfdir}/libibverbs.d %dir %{_libdir}/libibverbs %{_libdir}/libefa.so.* +%{_libdir}/libhns.so.* %{_libdir}/libibverbs*.so.* %{_libdir}/libibverbs/*.so %{_libdir}/libmana.so.* @@ -585,6 +590,12 @@ fi %{_mandir}/* %changelog +* Tue Mar 12 2024 Ran Zhou - 50.0-3 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Backport congestion control from mainline + * Thu Feb 22 2024 Ran Zhou - 50.0-2 - Type: requirement - ID: NA