From 18fdd6e9fbaeddb7f50f7dc1d7c87b89680ec96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?why=E6=B6=9B?= <1396718917@qq.com> Date: Tue, 14 Jan 2025 11:28:16 +0000 Subject: [PATCH 01/11] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=20app/zh/blog?= =?UTF-8?q?s/RDMA/RDMA.md=20=E4=B8=BA=20app/zh/blogs/RDMA/RDMA=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E6=8C=87=E5=AF=BC=5F2023.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/zh/blogs/RDMA/RDMA.md => "app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" (100%) diff --git a/app/zh/blogs/RDMA/RDMA.md "b/app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" similarity index 100% rename from app/zh/blogs/RDMA/RDMA.md rename to "app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" -- Gitee From 83d7d841bcb74b098893ed9aa983c1fca20138b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?why=E6=B6=9B?= <1396718917@qq.com> Date: Wed, 15 Jan 2025 01:18:33 +0000 Subject: [PATCH 02/11] =?UTF-8?q?update=20app/zh/blogs/RDMA/RDMA=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E6=8C=87=E5=AF=BC=5F2023.md.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: why涛 <1396718917@qq.com> --- ...DMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git "a/app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" "b/app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" index 4baff774..39573178 100644 --- "a/app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" +++ "b/app/zh/blogs/RDMA/RDMA\347\275\221\347\273\234\346\214\207\345\257\274_2023.md" @@ -1,5 +1,5 @@ --- -title: "RDMA网络指导" +title: "RDMA网络指导_2023" date: '2023-11-07' category: 'blog' tags: ['openGauss'] -- Gitee From 4d9363e9aee08bd054a0ceafb9907f45a4178f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?why=E6=B6=9B?= <1396718917@qq.com> Date: Wed, 15 Jan 2025 01:19:02 +0000 Subject: [PATCH 03/11] add app/zh/blogs/RDMA. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: why涛 <1396718917@qq.com> --- .../RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" diff --git "a/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" "b/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" new file mode 100644 index 00000000..e69de29b -- Gitee From c2f238beb3c8e2ce669120667a470f7f88c7532a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?why=E6=B6=9B?= <1396718917@qq.com> Date: Wed, 15 Jan 2025 01:19:36 +0000 Subject: [PATCH 04/11] =?UTF-8?q?update=20app/zh/blogs/RDMA/RMDA=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E6=8C=87=E5=AF=BC=5F2024.md.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: why涛 <1396718917@qq.com> --- ...7\273\234\346\214\207\345\257\274_2024.md" | 497 ++++++++++++++++++ 1 file changed, 497 insertions(+) diff --git "a/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" "b/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" index e69de29b..88af1114 100644 --- "a/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" +++ "b/app/zh/blogs/RDMA/RMDA\347\275\221\347\273\234\346\214\207\345\257\274_2024.md" @@ -0,0 +1,497 @@ +# 识别CX4/CX5网卡 + +执行以下命令。 + +``` +lspci |grep Mellanox +``` + +回显如下: + +``` +81:00.0 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5] +81:00.1 Ethernet controller: Mellanox Technologies MT27800 Family [ConnectX-5] +``` + +# 安装MLNX驱动 + +1. 选择下载与OS匹配的驱动包,地址:[https://network.nvidia.com/products/infiniband-drivers/linux/mlnx\_ofed/](https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/)。 + +  + +2. 新建目录,将OS镜像文件挂载至新建目录。OS镜像名称请根据实际情况进行修改。 + + ``` + mkdir -p /mnt/iso + mount openEuler-22.03-LTS-aarch64-dvd.iso /mnt/iso + ``` + +3. 安装过程中需要部分依赖,请配置好OS镜像源,例如本地镜像。 + 1. 打开镜像源文件。 + + ``` + vim /etc/yum.repos.d/openEuler.repo + ``` + + 2. 按“i”进入编辑模式,只保留以下内容。 + + ``` + [OS] + name=OS + baseurl=file:///mnt/iso + enabled=1 + gpgcheck=0 + ``` + + 3. 按“Esc”键,输入**:wq!**,按“Enter”保存并退出编辑。 + 4. 缓存软件包。 + + ``` + yum makecache + ``` + +4. 上传驱动包至服务器并解压。驱动包名称请根据实际情况进行修改。 + + ``` + tar -zxvf MLNX_OFED_LINUX-5.8-1.1.2.1-openeuler22.03-aarch64.tgz + ``` + +5. 进入解压后的驱动包目录,执行以下命令安装驱动。 + + ``` + ./mlnxofedinstall --without-depcheck --without-fw-update --force + ``` + + 如果提示内核不匹配,则执行以下命令。 + + ``` + ./mlnxofedinstall --add-kernel-support + ``` + +6. 设置重启后自动拉起驱动。 + + ``` + chkconfig --add openibd + /etc/init.d/openibd start + chkconfig openibd on + ``` + +7. 安装完成后重启服务器。 + +# 安装检查 + +1. 检查驱动RoCE LAG功能。 + 1. 查看RoCE LAG功能是否启用。 + + ``` + find /sys/ -name roce_lag_enable | xargs cat + ``` + + - 输出为1,则表示开启。 + - 输出为0或无输出,则表示关闭。 + - 预期要关闭才可以,如果没有关闭,请执行[1.b](#li519083722516)。 + + 2. 关闭RoCE LAG功能。 + + ``` + sed '/load_module mlx5_core/a\ files=`find /sys -name roce_lag_enable`;for file in $files;do echo 0 > $file;done' -i /etc/init.d/openibd + ``` + + 3. 执行后重启节点使其生效,重启后再执行[1.a](#li389014811257)查看是否生效。 + + ``` + reboot + ``` + +2. 检查驱动版本。 + + ``` + ofed_info -s + ``` + + 回显驱动版本与[安装MLNX驱动](安装MLNX驱动.md)版本一致则版本无误。 + +3. 加载MST工具。 + + ``` + mst start + ``` + + 回显如下表示加载成功。 + + ``` + Starting MST (Mellanox Software Tools) driver set + Loading MST PCI module - Success + Loading MST PCI configuration module - Success + Create devices + Unloading MST PCI module (unused) - Success + ``` + +4. 查询设备路径和网络端口。 + 1. 查询设备上RoCE和IB卡的设备路径。 + + ``` + mst status + ``` + + 回显如下: + + ``` + MST modules: + ------------ + MST PCI module is not loaded + MST PCI configuration module loaded + + MST devices: + ------------ + /dev/mst/mt4119_pciconf0 - PCI configuration cycles access. + domain:bus:dev.fn=0000:81:00.0 addr.reg=88 data.reg=92 cr_bar.gw_offset=-1 + Chip revision is: 00 + ``` + + 其中,MST devices字段枚举的设备路径/dev/mst/mst\_typeN(N取值:0,1,2,……)代表一张CX卡,mst\_type和CX网卡型号映射关系参考[表1](#table93856218287)。 + + **表 1** mst\_type和CX网卡型号映射关系 + + +
+ | ++ | +
---|---|
+ | ++ | +
+ | ++ | +
+ | ++ | +
+ | ++ | +
gbfHuA`0cqTyiAn`A1C Xt+7T|9+mO{@sMR!pfPoK;=Eoljk#J2GzDO_GZ~u)T+yVX&+Ahmklq
zH;~DzF7het82g`QlYVXjo~?gO9BkfeCTCuLBy7`g>?}=wvN$Nk&SfHpKIf7jcc1lR
z+-lFB@SVe{GiNz};R4s6JtI2i4aa^RPkR^Ro5+v&@$Xz?5C>=5wqyC#&wP4%k?VJF
zvtj%+Rvy)fgEKghR7geHO@j2ie!X1trtVDn{T+oh#l+tX#m}?$o0a)@t0^u_;LgP}
zL}cV?Hk&OiB`+p|Gpqb)YNO#)Ee?Lca%n*p(XY<3bf7C1#t%03E%2Sao%83 Z1`s{)U44Ns<<;a~J3z#kF}RH#
z&&*ZFNGizT*0y={Qp6mrrljO4J7a{sb_|oJt>DbZdc;YI#Cvv^)+;9mU+-t^7 DpC_RWmcXxOy;WZnR`R3iBs6b`jIP-Dz*y^4w_N&cE;PE{qRZ
zNPL2?%0+~ 9J71eXQ-~tb9RmRH8$_nN1M%Hdm)27!DpwHW!zi)|B9JY8XY0> $`c7;}tI6?J?%
zx1H&p<8ce!z@Iq{h*Mv%Zh|}zcPE+KL_P0MY`|-R2kr~Da3!~avQPI}Hb(BR-wppw
zSIMlZ=k4KDjQ1GBxJ5fj%&n*B^);eLejAHO9)qT}Y{iD{dNFjwXx!Y~i1{;z%A$0R
zC%EC}Hj+U-J7B6KVu0T^p5;_YEEYD03!k|$(Ps|(FMXk|JcZLsRb2`NfP60@iAXer
zRnLUL*$v!S6O7wLAA)vV=5=8eUrw)JbQig0$OtxmET+D?ipwj4aGUH)=-x!$7gh4<
zNIWhM^6WiQ&>C8{W6h&R8gpN=HON)z_D>G#vFy#Pq_QZN;|T${O_@XN$w#D=RPbi|
zT)J4cKqx9rGUW*%68$_mXwz>TJMI@#^zJ^pgJ&_>%?-DqJuwr=Wx?%I#|&4$&AceA
zAp6d4ya$bE#)eCziEDVdccR3y!OVIh`>s6WOw4R%2;UceX3b*TrN>lMRgiHhTqOpP
zYMbS9P=1=xZ`=y*7uS+;{~}Sdym1R$!h!cC)QbzbFsEw^a#a3)>%Y4$ILOC>d``}C
zXm&9E-Q-})ZD`G{Sl_lQ&cjAYWEXZiU2+)tbGabPQMmN&jD@ZU{l1gg_WTRt#sV%U
zcuD0fm-k3ldUv!!SJVOjC8xOY=P?puCgJWblra{sX)Cz!u!y>f4;)@N?(expZWWix
zviJ#dP}G&sKQpMP7W4G-4rb4mVr#~XaCV*ENlf3)5{1{k?*?8r)ROsR598$4YL){C
z*?pV5s*fC-Is^kP;qW?{BiSWnC!Z!-Qr0xW|G2>C%t}hr4l~Q%3=Pp>w%`21_2u&j
zkjr-ZbXFd}N^VIRIk%TH*jNuO?P2V^mBi%(OBC&LbDKf*x?5xy)=-swgh|@cgSe)y
z0Ui1dlZ@B&xSc#s6Z82(JpNune}>3hpAG0VXc38pHRQgy&C+?(aGM>&j;Gm(D~h=t
zDtC`Hrst@|+$qZ9@zvGL@e#@hVENwHRMj+)^J*C*L|U}(=gq0lh1}Z{#z4hwu5=u_
zN8d+WRm#nkBNSGZKb(_taxncaD;CaF#hxIi(Zd
r@WC5L5d44%_F=V4_g2pra)hQe4$U4vIPqBl_42D(X@=w_`q@
z;{~~dvg3IrRaH%H#hu14;L^t`iXLBLjo%bJgz~~Sb2+t`lIrHZ
zq@Gw$knoP6qwV^W=XqkP>VBRa6ph(PW@SA^xtB=@5bB)5{FN7Yo>@nI{kLw#Mh^Xm
zyOc{~=__{38}(D3B+7Wv(9*%a_fUy^r!HK_&CeAiUt7bxX+j%@(#=dD(X_Gc62zbR
z)f9Yq!n)AuxcM(+
wI
zGLxr)Q76^Q1=U4PJMJl5oXy2kTd3pf9${WY2{hDFyFG)e(7*92bQAF#Pg$aFbFKGu
zKjA-Q4s`&^fp+|3<;BFegf8Ri>4D~Adb*Kbe96d-#=Cqp=Hj`_%c>9A5s=pJ-PVD-
zG8gL$wF}uXDMX<$azrAP#wmX&vX4DcCmu|=8Xk_C8r~S4#_waBufqI7%tV&(<`%jI
zV-XBTz#DuLp7gzb@9q8Du`Hn$0-*Ms
Xm~k;@%GVCsQCAK
zo@vynv8rY657FC252D*4@Pl+p%()Ko7{S#hApSg3V!LoNs|%Ch)x+f&fIoXPbw;#o
z;7i(|@EDGd96>>{nVX#>0Feb$Qq