diff --git a/0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch b/0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch new file mode 100644 index 0000000000000000000000000000000000000000..73a04357e8b8712504c924c1fa975a3a45afca05 --- /dev/null +++ b/0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch @@ -0,0 +1,75 @@ +From 950e7e2da1b50e5aab20b2efb450890df1ef0ed1 Mon Sep 17 00:00:00 2001 +From: Chengchang Tang +Date: Mon, 7 Nov 2022 21:09:08 +0800 +Subject: Perftest: Fix verification of max_inline_data for *_create_qp_ex() + +Currently, attr.cap.max_inline_data is used for validation in +*_create_qp() and *_create_qp_ex(). But actually, when entering +the create_qp_ex path, the variable attr is not used. So the current +check of the *_create_qp_ex() branch is meaningless. + +The attr_ex.cap.max_inline_data is used to check the max_inline_data +in *_create_qp_ex() path. And related printing error has also been +fixed. + +Fixes: 13f71777e6f0 ("Added new post_send API usage for RC,UC,UD,XRC") +Signed-off-by: Chengchang Tang +--- + src/perftest_resources.c | 24 ++++++++++++++---------- + 1 file changed, 14 insertions(+), 10 deletions(-) + +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index b6f1cb8..5065181 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -2364,22 +2364,21 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, + int dc_num_of_qps = user_param->num_of_qps / 2; + + int is_dc_server_side = 0; ++ struct ibv_qp_init_attr attr; ++ memset(&attr, 0, sizeof(struct ibv_qp_init_attr)); ++ struct ibv_qp_cap *qp_cap = &attr.cap; ++ + #ifdef HAVE_IBV_WR_API + enum ibv_wr_opcode opcode; +- struct ibv_qp_init_attr attr; + struct ibv_qp_init_attr_ex attr_ex; ++ memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex)); + #ifdef HAVE_MLX5DV + struct mlx5dv_qp_init_attr attr_dv; + memset(&attr_dv, 0, sizeof(attr_dv)); + #endif +- memset(&attr, 0, sizeof(struct ibv_qp_init_attr)); +- memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex)); + #ifdef HAVE_SRD + struct efadv_qp_init_attr efa_attr = {}; + #endif +- #else +- struct ibv_qp_init_attr attr; +- memset(&attr, 0, sizeof(struct ibv_qp_init_attr)); + #endif + + attr.send_cq = ctx->send_cq; +@@ -2554,10 +2553,15 @@ struct ibv_qp* ctx_qp_create(struct pingpong_context *ctx, + fprintf(stderr, "Current TX depth is %d and inline size is %d .\n", user_param->tx_depth, user_param->inline_size); + } + +- if (user_param->inline_size > attr.cap.max_inline_data) { +- user_param->inline_size = attr.cap.max_inline_data; +- printf(" Actual inline-size(%d) > requested inline-size(%d)\n", +- attr.cap.max_inline_data, user_param->inline_size); ++ #ifdef HAVE_IBV_WR_API ++ if (!user_param->use_old_post_send) ++ qp_cap = &attr_ex.cap; ++ #endif ++ ++ if (user_param->inline_size > qp_cap->max_inline_data) { ++ printf(" Actual inline-size(%d) < requested inline-size(%d)\n", ++ qp_cap->max_inline_data, user_param->inline_size); ++ user_param->inline_size = qp_cap->max_inline_data; + } + + return qp; +-- +2.30.0 + diff --git a/0002-Perftest-Add-support-for-HNS.patch b/0002-Perftest-Add-support-for-HNS.patch new file mode 100644 index 0000000000000000000000000000000000000000..7e44ae33996be2051b5d8c82baef1b179b0cbb86 --- /dev/null +++ b/0002-Perftest-Add-support-for-HNS.patch @@ -0,0 +1,72 @@ +From da0dc664ee473f6e35abfadcf121fd4e4081ee9b Mon Sep 17 00:00:00 2001 +From: Chengchang Tang +Date: Fri, 16 Sep 2022 11:28:29 +0800 +Subject: Perftest: Add support for HNS + +Add support for HNS device by making it recognized by perftest. +Make the perftest allow testing new post send method for hns roce. +And a suitable default inline data size is applied. + +Signed-off-by: Chengchang Tang +--- + src/perftest_parameters.c | 9 +++++++++ + src/perftest_parameters.h | 1 + + src/perftest_resources.c | 3 ++- + 3 files changed, 12 insertions(+), 1 deletion(-) + +diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c +index 5fd9f7d..73a8df5 100755 +--- a/src/perftest_parameters.c ++++ b/src/perftest_parameters.c +@@ -1851,6 +1851,13 @@ enum ctx_device ib_dev_name(struct ibv_context *context) + case 55300 : dev_fname = NETXTREME; break; + case 61344 : dev_fname = EFA; break; + case 61345 : dev_fname = EFA; break; ++ case 41506 : dev_fname = HNS; break; ++ case 41507 : dev_fname = HNS; break; ++ case 41508 : dev_fname = HNS; break; ++ case 41509 : dev_fname = HNS; break; ++ case 41510 : dev_fname = HNS; break; ++ case 41512 : dev_fname = HNS; break; ++ case 41519 : dev_fname = HNS; break; + default : dev_fname = UNKNOWN; + } + } +@@ -2054,6 +2061,8 @@ static void ctx_set_max_inline(struct ibv_context *context,struct perftest_param + user_param->inline_size = 32; + else if (current_dev == QLOGIC_E4) + user_param->inline_size = 128; ++ else if (current_dev == HNS) ++ user_param->inline_size = 32; + + } else { + user_param->inline_size = 0; +diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h +index bad587c..717a154 100755 +--- a/src/perftest_parameters.h ++++ b/src/perftest_parameters.h +@@ -380,6 +380,7 @@ enum ctx_device { + CONNECTX7 = 26, + QLOGIC_AHP = 27, + BLUEFIELD3 = 28, ++ HNS = 29, + }; + + /* Units for rate limiter */ +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index 5065181..dc19cc3 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -1979,7 +1979,8 @@ int verify_params_with_device_context(struct ibv_context *context, + current_dev != BLUEFIELD && + current_dev != BLUEFIELD2 && + current_dev != BLUEFIELD3 && +- current_dev != EFA) ++ current_dev != EFA && ++ current_dev != HNS) + { + if (!user_param->use_old_post_send) + { +-- +2.30.0 + diff --git a/0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch b/0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch new file mode 100644 index 0000000000000000000000000000000000000000..6f8e1c581005dcd95b0057759c02324b6d4d0f6d --- /dev/null +++ b/0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch @@ -0,0 +1,29 @@ +From 1c7d57d794609887f3084408630814c8f02fad1f Mon Sep 17 00:00:00 2001 +From: Chengchang Tang +Date: Mon, 7 Nov 2022 16:09:42 +0800 +Subject: Perftest: Add new HNS roce device (ROH) to support new_io + +Add ROH device id. + +Signed-off-by: Chengchang Tang +--- + src/perftest_parameters.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c +index 73a8df5..a048a5a 100755 +--- a/src/perftest_parameters.c ++++ b/src/perftest_parameters.c +@@ -1858,6 +1858,9 @@ enum ctx_device ib_dev_name(struct ibv_context *context) + case 41510 : dev_fname = HNS; break; + case 41512 : dev_fname = HNS; break; + case 41519 : dev_fname = HNS; break; ++ case 41511 : dev_fname = HNS; break; ++ case 41516 : dev_fname = HNS; break; ++ case 41517 : dev_fname = HNS; break; + default : dev_fname = UNKNOWN; + } + } +-- +2.30.0 + diff --git a/0004-Perftest-Increase-max-inline-size-to-support-larger-.patch b/0004-Perftest-Increase-max-inline-size-to-support-larger-.patch new file mode 100644 index 0000000000000000000000000000000000000000..46e02eceab1ff3f86b16f250f05fa29aa256f89e --- /dev/null +++ b/0004-Perftest-Increase-max-inline-size-to-support-larger-.patch @@ -0,0 +1,33 @@ +From 18152fd789f6be3eaf82684206a3ce86149180fe Mon Sep 17 00:00:00 2001 +From: Chengchang Tang +Date: Tue, 8 Nov 2022 11:48:35 +0800 +Subject: Perftest: Increase max inline size to support larger inline tests + +For some hns roce devices, a maximum of 1024 bytes of inline is +supported. + +So, increate the MAX_INLINE and MAX_INLINE_UD to support related test. + +Signed-off-by: Chengchang Tang +--- + src/perftest_parameters.h | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/perftest_parameters.h b/src/perftest_parameters.h +index 717a154..25be929 100755 +--- a/src/perftest_parameters.h ++++ b/src/perftest_parameters.h +@@ -178,8 +178,8 @@ + #define UC_MAX_RX (16000) + #define MIN_CQ_MOD (1) + #define MAX_CQ_MOD (1024) +-#define MAX_INLINE (912) +-#define MAX_INLINE_UD (884) ++#define MAX_INLINE (1024) ++#define MAX_INLINE_UD (1024) + #define MIN_EQ_NUM (0) + #define MAX_EQ_NUM (2048) + +-- +2.30.0 + diff --git a/0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch b/0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch new file mode 100644 index 0000000000000000000000000000000000000000..7d041d23efd4d51784d7be5e1e811873df80ada6 --- /dev/null +++ b/0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch @@ -0,0 +1,144 @@ +From cfe0f71c2382f95e809f7edefa052e7ac9a41f8e Mon Sep 17 00:00:00 2001 +From: Chengchang Tang +Date: Thu, 28 Apr 2022 15:59:42 +0800 +Subject: Perftest: replace rand() with getrandom() during MR buffer + initialization + +rand() has very poor performance in some OS. + +ib_send_bw will spend a lot of time during MR initialization when +testing large packects in above scenario. + +test has been done: +""" +int main(int argc, char *argv[]) +{ + char *a = malloc(HUGE_MR_SIZE * sizeof(char)); + unsigned int i; + char *tmp = a; + int ret; + + srand(time(NULL)); + if (a == NULL) + exit(1); + + if (argc <= 1) + goto fall_back; + + for (i = HUGE_MR_SIZE; i > 0;) { + ret = getrandom(tmp, i, 0); + if (ret < 0) + goto fall_back; + tmp += ret; + i -= ret; + } + goto out; + +fall_back: + for(i = 0; i < HUGE_MR_SIZE; i++) + a[i] = (char)rand(); +out: + free(a); + return 0; +} + +time ./a.out +real 5m35.033s +user 5m33.546s +sys 0m0.918s + +time ./a.out 1 + +real 0m6.454s +user 0m0.000s +sys 0m6.449s +""" + +As shown in the test above, getrandom() has a much better performance, +so replace rand() with it. + +Signed-off-by: Chengchang Tang +--- + configure.ac | 1 + + src/perftest_resources.c | 31 ++++++++++++++++++++++++++----- + 2 files changed, 27 insertions(+), 5 deletions(-) + +diff --git a/configure.ac b/configure.ac +index 0c310f9..007957b 100755 +--- a/configure.ac ++++ b/configure.ac +@@ -60,6 +60,7 @@ AC_PROG_LIBTOOL + AC_PROG_RANLIB + AC_HEADER_STDC + AC_CHECK_HEADERS([infiniband/verbs.h],,[AC_MSG_ERROR([ibverbs header files not found])]) ++AC_CHECK_HEADERS([sys/random.h],,[AC_MSG_ERROR([getrandom available])]) + AC_CHECK_LIB([ibverbs], [ibv_get_device_list], [], [AC_MSG_ERROR([libibverbs not found])]) + AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-devel not found])) + AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found])) +diff --git a/src/perftest_resources.c b/src/perftest_resources.c +index dc19cc3..334bede 100755 +--- a/src/perftest_resources.c ++++ b/src/perftest_resources.c +@@ -22,6 +22,9 @@ + #ifdef HAVE_CONFIG_H + #include + #endif ++#ifdef HAVE_SYS_RANDOM_H ++#include ++#endif + #ifdef HAVE_SRD + #include + #endif +@@ -1716,12 +1719,33 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa + return ret; + } + ++static void random_data(char *buf, int buff_size) ++{ ++ int i; ++#ifdef HAVE_SYS_RANDOM_H ++ char *tmp = buf; ++ int ret; ++ ++ for(i = buff_size; i > 0;) { ++ ret = getrandom(tmp, i, 0); ++ if(ret < 0) ++ goto fall_back; ++ tmp += ret; ++ i -= ret; ++ } ++ return; ++fall_back: ++#endif ++ srand(time(NULL)); ++ for (i = 0; i < buff_size; i++) ++ buf[i] = (char)rand(); ++} ++ + /****************************************************************************** + * + ******************************************************************************/ + int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param, int qp_index) + { +- int i; + int flags = IBV_ACCESS_LOCAL_WRITE; + + +@@ -1860,13 +1884,10 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u + #ifdef HAVE_CUDA + if (!user_param->use_cuda) { + #endif +- srand(time(NULL)); + if (user_param->verb == WRITE && user_param->tst == LAT) { + memset(ctx->buf[qp_index], 0, ctx->buff_size); + } else { +- for (i = 0; i < ctx->buff_size; i++) { +- ((char*)ctx->buf[qp_index])[i] = (char)rand(); +- } ++ random_data(ctx->buf[qp_index], ctx->buff_size); + } + #ifdef HAVE_CUDA + } +-- +2.30.0 + diff --git a/perftest.spec b/perftest.spec index 318fffa29ceff0708fbb693e494faedb24241038..f221f2de1c32e9265763a813152f3e369ebb48a9 100644 --- a/perftest.spec +++ b/perftest.spec @@ -1,12 +1,18 @@ Name: perftest Version: 4.5 -Release: 1 +Release: 2 License: GPLv2 or BSD Summary: RDMA Performance Testing Tools Url: https://github.com/linux-rdma/perftest Source: https://github.com/linux-rdma/perftest/releases/download/v4.5-0.12/perftest-4.5-0.12.ge93c538.tar.gz -BuildRequires: gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2 +Patch1: 0001-Perftest-Fix-verification-of-max_inline_data-for-_cr.patch +Patch2: 0002-Perftest-Add-support-for-HNS.patch +Patch3: 0003-Perftest-Add-new-HNS-roce-device-ROH-to-support-new_.patch +Patch4: 0004-Perftest-Increase-max-inline-size-to-support-larger-.patch +Patch5: 0005-Perftest-replace-rand-with-getrandom-during-MR-buffe.patch + +BuildRequires: automake gcc libibverbs-devel >= 1.2.0 librdmacm-devel >= 1.0.21 libibumad-devel >= 1.3.10.2 BuildRequires: pciutils-devel Obsoletes: openib-perftest < 1.3 @@ -30,6 +36,12 @@ done %_bindir/* %changelog +* Mon Nov 07 2022 tangchengchang - 4.5-2 +- Type: requirement +- ID: NA +- SUG: NA +- DESC: Add hns support and fixes some bug in perftest + * Tue Jan 18 2022 SimpleUpdate Robot - 4.5-1 - Upgrade to version 4.5