From ec49bd8aec17a48266201065736cfc4e7ba63e32 Mon Sep 17 00:00:00 2001 From: hemiao Date: Tue, 24 Jun 2025 16:37:48 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/tools/community_tools/_toc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/tools/community_tools/_toc.yaml b/docs/en/tools/community_tools/_toc.yaml index d5a6c3a66..24b9d27a8 100644 --- a/docs/en/tools/community_tools/_toc.yaml +++ b/docs/en/tools/community_tools/_toc.yaml @@ -4,7 +4,7 @@ sections: sections: - href: ./image_custom/isocut/_toc.yaml - href: ./image_custom/image_tailor/_toc.yaml - - label: GCC User Guide + - label: Compilation sections: - href: ../../server/development/gcc/_toc.yaml - label: Performance Optimization -- Gitee From cdd50cd44d22b48d0cbcd572409010548c7249ba Mon Sep 17 00:00:00 2001 From: hemiao Date: Wed, 25 Jun 2025 09:52:41 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=90=AD=E5=BB=BA?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E7=AB=A0=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../administrator => archive}/common-issues-and-solutions.md | 0 docs/en/server/administration/administrator/_toc.yaml | 1 - .../server/administration/administrator/configuring-services.md | 1 - .../server/administration/administrator/configuring-services.md | 1 - 4 files changed, 3 deletions(-) rename {docs/en/server/administration/administrator => archive}/common-issues-and-solutions.md (100%) delete mode 100644 docs/en/server/administration/administrator/configuring-services.md delete mode 100644 docs/zh/server/administration/administrator/configuring-services.md diff --git a/docs/en/server/administration/administrator/common-issues-and-solutions.md b/archive/common-issues-and-solutions.md similarity index 100% rename from docs/en/server/administration/administrator/common-issues-and-solutions.md rename to archive/common-issues-and-solutions.md diff --git a/docs/en/server/administration/administrator/_toc.yaml b/docs/en/server/administration/administrator/_toc.yaml index 730acb796..47a46754d 100644 --- a/docs/en/server/administration/administrator/_toc.yaml +++ b/docs/en/server/administration/administrator/_toc.yaml @@ -16,7 +16,6 @@ sections: - label: Process Management href: ./process-management.md - label: Service Configuration - href: ./configuring-services.md sections: - label: Configuring the Repo Server href: ./configuring-the-repo-server.md diff --git a/docs/en/server/administration/administrator/configuring-services.md b/docs/en/server/administration/administrator/configuring-services.md deleted file mode 100644 index e33439eb2..000000000 --- a/docs/en/server/administration/administrator/configuring-services.md +++ /dev/null @@ -1 +0,0 @@ -# Configuring Services diff --git a/docs/zh/server/administration/administrator/configuring-services.md b/docs/zh/server/administration/administrator/configuring-services.md deleted file mode 100644 index 72cb121e1..000000000 --- a/docs/zh/server/administration/administrator/configuring-services.md +++ /dev/null @@ -1 +0,0 @@ -# 搭建服务 -- Gitee From da1f12007f0ec6982cbc453d3730f4bbd3d56be5 Mon Sep 17 00:00:00 2001 From: hemiao Date: Wed, 25 Jun 2025 09:53:37 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=90=AD=E5=BB=BA?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E7=AB=A0=E8=8A=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/server/administration/administrator/_toc.yaml | 2 -- docs/zh/server/administration/administrator/_toc.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/docs/en/server/administration/administrator/_toc.yaml b/docs/en/server/administration/administrator/_toc.yaml index 47a46754d..a92cedddc 100644 --- a/docs/en/server/administration/administrator/_toc.yaml +++ b/docs/en/server/administration/administrator/_toc.yaml @@ -25,5 +25,3 @@ sections: href: ./configuring-the-web-server.md - label: Setting Up the Database Server href: ./setting-up-the-database-server.md - - label: Common Issues and Solutions - href: ./common-issues-and-solutions.md diff --git a/docs/zh/server/administration/administrator/_toc.yaml b/docs/zh/server/administration/administrator/_toc.yaml index b02090e6b..7de0029c8 100644 --- a/docs/zh/server/administration/administrator/_toc.yaml +++ b/docs/zh/server/administration/administrator/_toc.yaml @@ -15,7 +15,6 @@ sections: - label: 管理进程 href: ./process-management.md - label: 搭建服务 - href: ./configuring-services.md sections: - label: 搭建repo服务器 href: ./configuring-the-repo-server.md -- Gitee From 6dc57b3a0db5177c79c6b8f79fc2ead52b4ece98 Mon Sep 17 00:00:00 2001 From: hemiao Date: Wed, 25 Jun 2025 11:27:18 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=B1=82=E7=BA=A7=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...n-aggregation-environment-establishment.md | 56 ++++++++++--------- ...n-aggregation-environment-establishment.md | 56 ++++++++++--------- 2 files changed, 58 insertions(+), 54 deletions(-) diff --git a/docs/en/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md b/docs/en/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md index 42e5933ad..90280c4bd 100644 --- a/docs/en/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md +++ b/docs/en/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md @@ -1,6 +1,8 @@ -# 1 Hardware Preparation +# Libvirt Direct Connection Aggregation Environment Establishment -## Test Mode +## 1 Hardware Preparation + +### Test Mode Prepare two physical machines (VMs have not been tested) that can communicate with each other. @@ -9,31 +11,31 @@ One physical machine functions as the DPU, and the other functions as the host. > [!NOTE]NOTE > In the test mode, network ports are exposed without connection authentication, which is risky and should be used only for internal tests and verification. Do not use this mode in the production environment. -## vsock mode +### vsock mode The DPU and HOST are required. The DPU must be able to provide vsock communication through virtio. This document describes only the test mode usage. -# 2 libvirt offload architecture +## 2 libvirt offload architecture ![arch](./figures/arch.png) -# 3 Environment Setup +## 3 Environment Setup -## 3.1 qtfs File System Deployment +### 3.1 qtfs File System Deployment For details, visit . To establish a qtfs connection, you need to disable the firewall. -## 3.2 Deploying the udsproxyd Service +### 3.2 Deploying the udsproxyd Service -### 3.2.1 Introduction +#### 3.2.1 Introduction udsproxyd is a cross-host Unix domain socket (UDS) proxy service, which needs to be deployed on both the host and DPU. The udsproxyd components on the host and dpu are peers. They implement seamless UDS communication between the host and DPU, which means that if two processes can communicate with each other through UDSs on the same host, they can do the same between the host and DPU. The code of the processes does not need to be modified, only that the client process needs to run with the **LD_PRELOAD=libudsproxy.so** environment variable. As a cross-host Unix socket service, udsproxyd can be used by running with `LD_PRELOAD=libudsproxy.so`. With the support of qtfs, udsproxyd can also be used transparently. You need to configure the allowlist in advance. The specific operations are described later. -### 3.2.2 Deploying udsproxyd +#### 3.2.2 Deploying udsproxyd Build udsproxyd in the dpu-utilities project: @@ -70,13 +72,13 @@ If the qtfs engine service is not started, you can start udsproxyd on the server nohup /usr/bin/udsproxyd 1 192.168.10.11 12121 192.168.10.10 12121 2>&1 & ``` -### 3.2.3 Using udsproxyd +#### 3.2.3 Using udsproxyd -#### 3.2.3.1 Using udsproxyd Independently +##### 3.2.3.1 Using udsproxyd Independently When starting the client process of the Unix socket application that uses the UDS service, add the **LD_PRELOAD=libudsproxy.so** environment variable to intercept the **connect** API of glibc for UDS interconnection. In the libvirt offload scenario, you can copy **libudsproxy.so**, which will be used by the libvirtd service, to the **/usr/lib64** directory in the chroot directory of libvirt. -#### 3.2.3.2 Using the udsproxyd Service Transparently +##### 3.2.3.2 Using the udsproxyd Service Transparently Configure the UDS service allowlist for qtfs. The allowlist is the sock file address bound to the Unix socket server. For example, the files of the Unix socket server created by the libvirt VM are in the **/var/lib/libvirt** directory. In this case, add the directory path to the allowlist in either of the following ways: @@ -121,15 +123,15 @@ The parameter is the index number listed when you query the allowlist. > [!NOTE]NOTE > The allowlist prevents irrelevant Unix sockets from establishing remote connections, causing errors or wasting resources. You are advised to set the allowlist as precisely as possible. For example, in this document, **/var/lib/libvirt** is set in the libvirt scenario. It would be risky to directly add **/var/lib**, **/var**, or the root directory. -## 3.3 rexec Service Deployment +### 3.3 rexec Service Deployment -### 3.3.1 Introduction +#### 3.3.1 Introduction rexec is a remote execution component developed using the C language. It consists of the rexec client and rexec server. The server is a daemon process, and the client is a binary file. After being started, the client establishes a UDS connection with the server using the udsproxyd service, and the server daemon process starts a specified program on the server machine. During libvirt virtualization offload, libvirtd is offloaded to the DPU. When libvirtd needs to start the QEMU process on the HOST, the rexec client is invoked to remotely start the process. -### 3.3.2 Deploying rexec +#### 3.3.2 Deploying rexec -#### 3.3.2.1 Configuring the Environment Variables and Allowlist +##### 3.3.2.1 Configuring the Environment Variables and Allowlist Configure the rexec server allowlist on the host. Put the **whitelist** file in the **/etc/rexec** directory, and change the file permission to read-only. @@ -147,7 +149,7 @@ Before starting the rexec_server service on the server, check whether the **/var mkdir /var/run/rexec ``` -#### 3.3.2.2 Starting the Service +##### 3.3.2.2 Starting the Service You can start the rexec_server service on the server in either of the following ways. @@ -183,9 +185,9 @@ You can start the rexec_server service on the server in either of the following nohup /usr/bin/rexec_server 2>&1 & ``` -## 3.4 libvirt Service Deployment +### 3.4 libvirt Service Deployment -### 3.4.1 Deploying on the HOST +#### 3.4.1 Deploying on the HOST Install the VM runtime and libvirt. (libvirt is installed to create related directories.) @@ -195,9 +197,9 @@ yum install -y qemu libvirt edk2-aarch64 # (required for starting VMs in the Arm Put the VM image on the HOST. The VM image will be mounted to the client through qtfs and shared with libvirt. -### 3.4.2 Deploying on the DPU +#### 3.4.2 Deploying on the DPU -#### 3.4.2.1 Creating the Chroot Environment +##### 3.4.2.1 Creating the Chroot Environment (a) Download the QCOW image from the openEuler official website, for example, openEuler 23.09: . @@ -227,7 +229,7 @@ qemu-nbd -d /dev/nbd0 mount --bind /root/new_root_origin /root/new_root ``` -#### 3.4.2.2 Installing libvirt +##### 3.4.2.2 Installing libvirt Compile the source code with a patch. @@ -266,7 +268,7 @@ meson build --prefix=/usr -Ddriver_remote=enabled -Ddriver_network=enabled -Ddri ninja -C build install ``` -#### 3.4.2.3 Starting the libvirtd Service +##### 3.4.2.3 Starting the libvirtd Service To use libvirt direct connection aggregation, you need to start the libvirtd service in the chroot environment, which requires the libvirtd service outside the chroot environment to be stopped. @@ -309,11 +311,11 @@ qtcfg -z Check whether the allowlist contains **/var/lib/libvirt**. -## 3.5 VM Startup +### 3.5 VM Startup After the service is deployed, you can manage the VM life cycle from the DPU. -### 3.5.1 Defining the VM +#### 3.5.1 Defining the VM (a) Place the VM boot image in a directory on the HOST, for example: @@ -337,7 +339,7 @@ mount -t qtfs /home/VMs /home/VMs virsh define xxx.xml ``` -### 3.5.2 Starting the VM +#### 3.5.2 Starting the VM ```bash virsh start domain @@ -347,7 +349,7 @@ virsh start domain Some libvirt directories are shared between the DPU and the HOST. Therefore, you need to unmount these directories before uninstalling the environment. Generally, stop the libvirtd and virtlogd processes and run the **virt_umount.sh** script. If a VM is running on the HOST, stop the VM before unmounting the directories. -# 5 Common Errors +## 5 Common Errors 1. libvirt compilation failure: Check whether the dependency packages are installed. If an external directory or HOST directory is mounted to the chroot environment, the compilation may fail. In this case, unmount the directory first. diff --git a/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md b/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md index 7db6caf78..73a8f778a 100644 --- a/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md +++ b/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md @@ -1,6 +1,8 @@ -# **1** 硬件准备 +# 直连聚合环境搭建 -## 测试模式 +## **1** 硬件准备 + +### 测试模式 需准备2台物理机（虚拟机当前未试过），网络互通。 @@ -9,31 +11,31 @@ > [!NOTE]说明 > 测试模式因为会暴露网络端口且不做连接认证，存在网络安全风险，仅能用于内部测试验证，不要用于实际生产环境。 -## vsock模式 +### vsock模式 需要DPU加HOST，且DPU能支持通过virtio提供vsock通信方式。 目前还未基于真实的支持DPU vsock的环境调试过，本文档当前仅描述基于测试模式的方法，下面的内容依然默认使用测试模式。 -# **2** libvirt卸载架构图 +## **2** libvirt卸载架构图 ![arch](./figures/arch.png) -# **3** 环境搭建 +## **3** 环境搭建 -## **3.1** QTFS文件系统部署 +### **3.1** QTFS文件系统部署 可参考qtfs主页： QTFS建联需要关闭防火墙。 -## **3.2** UDSPROXYD服务部署 +### **3.2** UDSPROXYD服务部署 -### 3.2.1 简介 +#### 3.2.1 简介 udsproxyd是一个跨主机的unix domain socket代理服务，需要分别部署在host和dpu上，在host和dpu上的udsproxyd组件是对等的关系，可以实现分布在host与dpu上的2个进程之间的uds通信，通信进程是无感的，也就是说如果这两个进程在同一主机内通过uds正常通信的功能，拉远到host和dpu之间也可以，不需要做代码适配，只需要作为client的一端加一个环境变量`LD_PRELOAD=libudsproxy.so`。udsproxyd作为一个跨主机的unix socket服务，本身可以用LD_PRELOAD=libudsproxy.so的方式对接使用，在qtfs的支持下也可以无感应用，这需要提前做好白名单相关配置，具体有两种方式，将在后面详述。 -### 3.2.2 部署方式 +#### 3.2.2 部署方式 首先，在dpu-utilities工程内编译udsproxyd： @@ -75,9 +77,9 @@ nohup /usr/bin/udsproxyd 1 192.168.10.10 12121 192.168.10.11 12121 2>&1 & nohup /usr/bin/udsproxyd 1 192.168.10.11 12121 192.168.10.10 12121 2>&1 & ``` -### 3.2.3 应用方式 +#### 3.2.3 应用方式 -#### 3.2.3.1 独立使用udsproxyd服务 +##### 3.2.3.1 独立使用udsproxyd服务 需要在使用uds服务的unix socket应用程序的client端进程启动时添加LD_PRELOAD=libudsproxy.so环境变量，以接管glibc的connect api进行uds对接，在libvirt卸载场景中可以将libudsproxy.so拷贝到libvirt的chroot目录下的/usr/lib64中以提供给libvirtd服务使用，这一步在后面介绍。 @@ -126,15 +128,15 @@ b) 通过配置文件增加，这需要在qtfs或qtfs_server内核模块加载 > [!NOTE]说明 > 白名单是为了防止不相干的unix socket链接也进行远程连接产生错误，或者浪费不必要的资源，所以白名单尽量设置得精确一些，比如本文中针对libvirt场景设置为/var/lib/libvirt/比较好，而直接将/var/lib/或/var/或直接将根目录加入的做法是有较大风险的。 -## **3.3** REXEC服务部署 +### **3.3** REXEC服务部署 -### 3.3.1 简介 +#### 3.3.1 简介 rexec是一个用c语言开发的远程执行组件，分为rexec client和rexec server。server端为一个常驻服务进程，client端为一个二进制文件，client端被执行后会基于udsproxyd服务与server端建立uds连接，并由server常驻进程在server端拉起指定程序。在libvirt虚拟化卸载中，libvirtd卸载到DPU上，当它需要在HOST拉起虚拟机qemu进程时调起rexec client进行远程拉起。 -### 3.3.2 部署方法 +#### 3.3.2 部署方法 -#### 3.3.2.1 配置环境变量与白名单 +##### 3.3.2.1 配置环境变量与白名单 在host侧配置rexec server的白名单，将文件whitelist放置在/etc/rexec/目录下并修改权限为只读： @@ -152,7 +154,7 @@ chmod 400 /etc/rexec/whitelist mkdir /var/run/rexec ``` -#### 3.3.2.2 服务方式 +##### 3.3.2.2 服务方式 server端可以通过两种方式拉起rexec_server服务。 @@ -190,9 +192,9 @@ systemctl start rexec nohup /usr/bin/rexec_server 2>&1 & ``` -## **3.4** libvirt服务部署 +### **3.4** libvirt服务部署 -### 3.4.1 HOST侧部署 +#### 3.4.1 HOST侧部署 HOST无需额外部署，只需要安装虚拟机启动环境以及libvirt即可（安装libvirt主要是为了创建对应的目录）： @@ -202,9 +204,9 @@ yum install -y qemu libvirt edk2-aarch64 #(arm环境虚拟机启动需要) HOST需要放置虚拟机镜像，后面通过qtfs挂载到client端共享给libvirt。 -### 3.4.2 DPU侧部署 +#### 3.4.2 DPU侧部署 -#### 3.4.2.1 创建chroot环境 +##### 3.4.2.1 创建chroot环境 a) 从openEuler官网下载qcow镜像，例如23.09版本 @@ -234,7 +236,7 @@ c) 此时new_root_origin有解压出来的镜像根目录，再将new_root绑定 mount --bind /root/new_root_origin /root/new_root ``` -#### 3.4.2.2 安装libvirt +##### 3.4.2.2 安装libvirt 此处介绍patch方式源码编译，如果计算提供rpm包则参考计算提供的安装方法。 @@ -273,7 +275,7 @@ g) 安装成功 ninja -C build install ``` -#### 3.4.2.3 启动libvirtd服务 +##### 3.4.2.3 启动libvirtd服务 libvirt直连聚合卸载模式，需要从chroot内启动libvirtd服务，首先需要把chroot之外的libvirtd服务停掉。 @@ -318,11 +320,11 @@ qtcfg -z 查看列举出来的白名单是否包含/var/lib/libvirt/。 -## **3.5** 拉起虚拟机 +### **3.5** 拉起虚拟机 服务部署完成后，即可以在DPU侧进行虚拟机的生命周期管理。 -### 3.5.1 虚拟机define +#### 3.5.1 虚拟机define a) 将虚拟机启动镜像放置在HOST侧某目录，例如： @@ -346,17 +348,17 @@ e) 执行以下命令。 virsh define xxx.xml ``` -### 3.5.2 虚拟机start +#### 3.5.2 虚拟机start ```bash virsh start domain ``` -# **4** 环境重置 +## **4** 环境重置 由于libvirt在DPU和HOST之间共享了部分目录，卸载环境时需要先将这部分目录全部umount。一般先停掉libvirtd和virtlogd进程，调用virt_umount脚本即可。如果HOST还有虚拟机运行，也需要先杀掉才能umount。 -# **5** 部分问题定位思路 +## **5** 部分问题定位思路 1、 libvirt编译失败：检查依赖包安装是否完全，如果chroot挂载了外部目录或者host目录，也可能导致编译失败，需先解除挂载。 -- Gitee From aee5006b61b29c05cefe3a6cc9662e44bb42c6c3 Mon Sep 17 00:00:00 2001 From: hemiao Date: Wed, 25 Jun 2025 14:55:35 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=87=E7=82=B9?= =?UTF-8?q?=E7=AC=A6=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...direct-connection-aggregation-environment-establishment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md b/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md index 73a8f778a..2540e49f2 100644 --- a/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md +++ b/docs/zh/server/diversified_computing/dpu_offload/libvirt-direct-connection-aggregation-environment-establishment.md @@ -208,7 +208,7 @@ HOST需要放置虚拟机镜像，后面通过qtfs挂载到client端共享给lib ##### 3.4.2.1 创建chroot环境 -a) 从openEuler官网下载qcow镜像，例如23.09版本 +a) 从openEuler官网下载qcow镜像，例如23.09版本。 b) 将qcow2挂载出来： @@ -255,7 +255,7 @@ b) 安装libvirt编译需要的依赖包： yum install -y rpcgen python3-docutils glib2-devel gnutls-devel libxml2-devel libpciaccess-devel libtirpc-devel yajl-devel systemd-devel dmidecode glusterfs-api numactl ``` -c) 下载libvirt-x.x.x。源码包： +c) 下载libvirt-x.x.x。源码包：。 d) 获取直连聚合libvirt patch： -- Gitee