From 926c60bf6a0d2380ef8c8d39fb060f9024bc4afa Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 18 Sep 2024 08:28:40 +0000 Subject: [PATCH 01/35] =?UTF-8?q?=E5=8A=A0=E9=80=9F=E5=BA=93=20KAE=20?= =?UTF-8?q?=E7=89=B9=E6=80=A7=E9=AA=8C=E6=94=B6=E6=94=AF=E6=8C=81=20KAE=20?= =?UTF-8?q?=E8=A7=A3=E5=8E=8B=E7=BC=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ksun --- tools/collect_msg.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 113d2d8..4229574 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -288,6 +288,8 @@ collect_acceleration_library(){ math_jar=$4 math_java=$5 openssl speed -elapsed -engine kae rsa2048 > $log_path/acceleration_library.log 2>&1 + lspci | grep ZIP >> $log_path/acceleration_library.log + ll /usr/local/kaezip/lib/ >> $log_path/acceleration_library.log ldd $1 >> $log_path/acceleration_library.log ldd $2 >> $log_path/acceleration_library.log ldd $3 >> $log_path/acceleration_library.log -- Gitee From e38dde140f47c95f81ea3985030870fea22aed02 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 23 Oct 2024 03:10:16 +0000 Subject: [PATCH 02/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 4229574..c008135 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -216,8 +216,8 @@ collect_database_other_db(){ greenplum_port=$3 kae_version=$4 greenplum_kae_sql=$5 - readelf -a $other_other_db_bin | grep bolt > $log_path/database_other_db.log - objdump -d $other_other_db_bin | grep crc32cb >> $log_path/database_other_db.log + readelf -a $other_db_bin | grep bolt > $log_path/database_other_db.log + objdump -d $other_db_bin | grep crc32cb >> $log_path/database_other_db.log if [ "$kae_version" == "1.0" ]; then nohup timout 20 watch -gt -n 0.2 cat /sys/class/uacce/hisi_zip*/attrs/available_instances > $log_path/database_greenplum_kae.log & else -- Gitee From 11f4320d2c19d80f60910e6973ddd4db6c135e01 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 23 Oct 2024 08:43:38 +0000 Subject: [PATCH 03/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 13a1781..2387995 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -102,7 +102,11 @@ spark的安装路径，默认为/home/ > omnimv_dir=/omnimv -omnimv 目录路径, 用于物化视图特性验证 +omnimv 目录路径, 用于spark场景下的物化视图特性验证 + +> omnimv_dir_clickhouse=/opt/OminiMV + +配置 OmniMV 工作文件的存储路径, 用于clickhouse场景下的物化视图特性验证 > omnidata_launcher_server= @@ -404,3 +408,4 @@ bash collect_msg.sh # 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 ``` + -- Gitee From 609d8ffe388ad093ceb8d49138d50c2d9b3d11c4 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 23 Oct 2024 08:45:58 +0000 Subject: [PATCH 04/35] update tools/config.ini. Signed-off-by: ksun --- tools/config.ini | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/config.ini b/tools/config.ini index 1b54908..9f597d3 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -56,12 +56,12 @@ vm_name=vm1 check=False # 工具默认支持验收 KAE 特性，不需要作额外配置 #引用系统库二进制位置 -system_lib=/home/system_lib.so +system_lib=/home/system_bin #引用HMPP库二进制位置 -HMPP_lib=/home/HMPP_lib.so +HMPP_lib=/home/HMPP_bin # 数学库三个配置只需填其中之一即可 # 引用数学库二进制位置 -math_lib=/home/math_lib.so +math_lib=/home/math_bin # 引用数学库的jar包位置 math_jar=/home/math.jar # 引用数学库的字节码文件位置 @@ -87,7 +87,10 @@ mysql_password=123456 mysql_database_name=xxxxx # 物化视图 +# 物化视图 - spark 场景 omnimv_dir=/omnimv +# 物化视图 - clickhouse 场景 +omnimv_dir_clickhouse=/opt/OminiMV # 算子下推 omnidata_launcher_server= @@ -98,7 +101,8 @@ push_down_conf=spark.executor.extraClassPath=./* zookeeper_address=agent1:2181,agent2:2181,agent3:2181 zookeeper_path=/sdi/status # shuffle加速 -shuffle_jars=/home/ockadmin/opt/ock/jars/* +shuffle_jars=/home/ockadmin/opt/ock/jar/* +shuffle_ock_path=/home/ockadmin/opt/ock [Storage] check=False @@ -117,4 +121,3 @@ acc_lib=/home/lib sme=/home/lib # 填写用到 sve 的源码目录或文件的绝对路径 sve_source_code=/tmp/src - -- Gitee From d9c3ba59a405a245175f03e604f70972436955d7 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 23 Oct 2024 08:48:34 +0000 Subject: [PATCH 05/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index c008135..d45ba52 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -613,7 +613,9 @@ collect_bigdata_tune_up(){ collect_bigdata_omnimv(){ omnimv_dir=$1 + omnimv_dir_clickhouse=$2 hdfs dfs -ls $omnimv_dir > $log_path/bigdata_omnimv.log + ls $omnimv_dir_clickhouse/BoostKit-omnimv_1.1.0/clickhouse/schema.sql >> $log_path/bigdata_omnimv.log } @@ -842,6 +844,7 @@ main(){ mysql_password=$(acquire_value Bigdata mysql_password) mysql_database_name=$(acquire_value Bigdata mysql_database_name) omnimv_dir=$(acquire_value Bigdata omnimv_dir) + omnimv_dir_clickhouse=$(acquire_value Bigdata omnimv_dir_clickhouse) omnidata_launcher=$(acquire_value Bigdata omnidata_launcher) omnidata_launcher_server=$(acquire_value Bigdata omnidata_launcher_server) omnidata_install_path=$(acquire_value Bigdata omnidata_install_path) @@ -856,7 +859,7 @@ main(){ collect_bigdata_operator $spark_path $database $omnioperator_dir collect_bigdata_hbase collect_bigdata_tune_up $omniadvisor_dir $mysql_username $mysql_password $mysql_database_name - collect_bigdata_omnimv "$omnimv_dir" + collect_bigdata_omnimv "$omnimv_dir" "$omnimv_dir_clickhouse" collect_bigdata_omni_push_down "$omnidata_launcher_server" "$omnidata_launcher" "$push_down_jars" "$push_down_conf" "$spark_path" "$database" "$omnidata_install_path" "$zookeeper_address" "$zookeeper_path" collect_bigdata_omni_shuffle "$spark_path" "$shuffle_jars" "$database" "$shuffle_ock_path" echo "Bigdata collect msg Done..." -- Gitee From d1dde415f162947f50121458f115fcdfe05a7b48 Mon Sep 17 00:00:00 2001 From: ksun Date: Tue, 29 Oct 2024 02:57:05 +0000 Subject: [PATCH 06/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index d45ba52..3e1a8d4 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -599,7 +599,7 @@ collect_bigdata_tune_up(){ mysql_username=$2 mysql_password=$3 mysql_database=$4 - if [ -e $omniadvisor_dir/omniadvisor ]; + if [ -e $omniadvisor_dir/logs/omniadvisor ]; then echo "omniadvisor.log" >> $log_path/bigdata_tune_up.log else @@ -880,6 +880,8 @@ main(){ vm_name=$(acquire_value Virtual vm_name) collect_virtual_ceph "$vm_ip" "$vm_user" "$vm_password" "$vm_ceph_disk_name" "$vm_name" collect_virtual_ovs_xpf + collect_vmpam + collect_irqbypass echo "Virtual collect msg Done..." elif [ $per_project = "HPC" ]; then -- Gitee From cc28ccc24556484e2ef1da5591c30f932d8b86d2 Mon Sep 17 00:00:00 2001 From: ksun Date: Tue, 29 Oct 2024 02:58:27 +0000 Subject: [PATCH 07/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 3e1a8d4..e9c66de 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -880,8 +880,6 @@ main(){ vm_name=$(acquire_value Virtual vm_name) collect_virtual_ceph "$vm_ip" "$vm_user" "$vm_password" "$vm_ceph_disk_name" "$vm_name" collect_virtual_ovs_xpf - collect_vmpam - collect_irqbypass echo "Virtual collect msg Done..." elif [ $per_project = "HPC" ]; then -- Gitee From ba86848d2451472bb3cb409707cd2ec8e11a6ff2 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 1 Nov 2024 02:44:43 +0000 Subject: [PATCH 08/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index e9c66de..95c1317 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -289,7 +289,7 @@ collect_acceleration_library(){ math_java=$5 openssl speed -elapsed -engine kae rsa2048 > $log_path/acceleration_library.log 2>&1 lspci | grep ZIP >> $log_path/acceleration_library.log - ll /usr/local/kaezip/lib/ >> $log_path/acceleration_library.log + ls -l /usr/local/kaezip/lib/ >> $log_path/acceleration_library.log ldd $1 >> $log_path/acceleration_library.log ldd $2 >> $log_path/acceleration_library.log ldd $3 >> $log_path/acceleration_library.log -- Gitee From 485659fd409f7cac96d30f82c2ef4662d5457668 Mon Sep 17 00:00:00 2001 From: ksun Date: Tue, 12 Nov 2024 02:53:38 +0000 Subject: [PATCH 09/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 95c1317..b179c82 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -739,7 +739,7 @@ collect_sme_acceleration_library(){ ldd $bin_path | grep SME >> $log_path/hpc_SME_library.log 2>&1 objdump -d $bin_path >> $log_path/hpc_SME_library.log 2>&1 else - echo "架构不支持SME" >> $log_path/hpc_SME_library.log + echo "架构不支持sme" >> $log_path/hpc_SME_library.log fi } -- Gitee From 2419c543cbb81f5535cd8bd69df74227a0527683 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 15 Nov 2024 09:48:09 +0000 Subject: [PATCH 10/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index b179c82..1fa9de5 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -585,6 +585,19 @@ collect_bigdata_operator(){ collect_bigdata_hbase(){ + hbase shell << EOF + create 'OnlySingleIndexTable', 'C0','C1' + put 'OnlySingleIndexTable', 'row1', 'C0:F0,', 'aaa' + put 'OnlySingleIndexTable', 'row1', 'C0:F1', 'bbb,bbb' + put 'OnlySingleIndexTable', 'row1', 'C1:F0', 'bbb' + put 'OnlySingleIndexTable', 'row2', 'C0:F0', 'opo' + put 'OnlySingleIndexTable', 'row2', 'C0:F1', 'bbb' + put 'OnlySingleIndexTable', 'row2', 'C1:F0', 'sdasd' + put 'OnlySingleIndexTable', 'row3', 'C0:F0', 'icic' + put 'OnlySingleIndexTable', 'row3', 'C0:F1', '23123' + put 'OnlySingleIndexTable', 'row3', 'C1:F0', 'bbb' + exit +EOF hbase com.huawei.boostkit.hindex.mapreduce.GlobalTableIndexer -Dtablename.to.index=OnlySingleIndexTable -Dindexspecs.to.addandbuild='osindex=>C0:[F1]' hbase shell << EOF > $log_path/bigdata_hbase.log 2>&1 debug -- Gitee From aa7934aef7e21e26db422c9b4f753df20cf3a1f5 Mon Sep 17 00:00:00 2001 From: ksun Date: Tue, 19 Nov 2024 08:14:10 +0000 Subject: [PATCH 11/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 2387995..7c67197 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -381,7 +381,7 @@ check 默认为False, 置为True 则开启HPC相关特性使能信息收集功 > acc_lib -引用HPC加速库改造二进制文件绝对路径 +引用HPC加速库改造二进制文件绝对路径, 多个路径用空格分隔 > sme -- Gitee From 2f88fb080b6771447ed18138de5ca8b160424e39 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 20 Nov 2024 02:04:13 +0000 Subject: [PATCH 12/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 1fa9de5..e05f349 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -772,6 +772,8 @@ collect_os_msg(){ main(){ + # remove \r at end of line + sed -i 's/\r$//' $config_file select_project=$(acquire_select_project) echo "开始收集BoostKit 特性信息如下：$select_project" mkdir -p $log_path -- Gitee From ce245f8722d5d3f94a37f494e7ce849092aa02d9 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 21 Nov 2024 03:50:44 +0000 Subject: [PATCH 13/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index e05f349..24dc078 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -509,7 +509,7 @@ collect_bigdata_kal(){ } -collect_bigdata_operator(){ +collect_bigdata_operator_spark(){ # 日志记录位置 log_path/bigdata_operator.log spark_path=$1 database=$2 @@ -583,6 +583,27 @@ collect_bigdata_operator(){ } +collect_bigdata_operator_hive(){ + hive_bin_path=$1 + hive_local_mode=$2 + omnioperator_dir=$3 + + extra_options='' + if [ "$hive_local_mode" == "1" ]; then + extra_options='--hiveconf tez.local.mode=true --hiveconf tez.runtime.optimize.local.fetch=true' + fi + OMNI_HOME=${omnioperator_dir} HIVE_AUX_JARS_PATH=${omnioperator_dir}/lib LD_LIBRARY_PATH=${omnioperator_dir}/lib:$LD_LIBRARY_PATH \ + hive --database default --hiveconf tez.task.launch.env=OMNI_HOME=${omnioperator_dir},LD_LIBRARY_PATH=${omnioperator_dir}/lib --hiveconf tez.local.mode=true --hiveconf tez.runtime.optimize.local.fetch=true --hiveconf hive.cbo.enable=true --hiveconf hive.exec.reducers.max=600 --hiveconf hive.exec.compress.intermediate=true --hiveconf hive.tez.container.size=8192 --hiveconf tez.am.resource.memory.mb=8192 --hiveconf tez.task.resource.memory.mb=8192 --hiveconf tez.runtime.io.sort.mb=128 --hiveconf hive.merge.tezfiles=true --hiveconf tez.am.container.reuse.enabled=true --hiveconf hive.exec.pre.hooks=com.huawei.boostkit.hive.OmniExecuteWithHookContext --hiveconf hive.vectorized.execution.enabled=false --hiveconf hive.fetch.task.conversion=none $extra_options > $log_path/bigdata_operator.log << EOF + drop table if exists test_omni_hive; + create table if not exists test_omni_hive(id int, name string, age int) stored as orc; + insert into test_omni_hive values(1, 'alice', 22),(2, 'bob', 23),(3, 'amy', 24); + explain select * from test_omni_hive order by id; + select * from test_omni_hive order by id; + exit; +EOF +} + + collect_bigdata_hbase(){ hbase shell << EOF @@ -853,6 +874,8 @@ main(){ dataset_list=$(acquire_value Bigdata dataset_list) spark_path=$(acquire_value Bigdata spark_path) database=$(acquire_value Bigdata database) + hive_bin_path=$(acquire_value Bigdata hive_bin_path) + hive_local_mode=$(acquire_value Bigdata hive_local_mode) omnioperator_dir=$(acquire_value Bigdata omnioperator_dir) omniadvisor_dir=$(acquire_value Bigdata omniadvisor_dir) mysql_username=$(acquire_value Bigdata mysql_username) @@ -871,7 +894,8 @@ main(){ shuffle_ock_path=$(acquire_value Bigdata shuffle_ock_path) collect_bigdata_components $spark_path collect_bigdata_kal "${algorithms_list[@]}" $algorithms_path "${dataset_list[@]}" - collect_bigdata_operator $spark_path $database $omnioperator_dir + collect_bigdata_operator_spark $spark_path $database $omnioperator_dir + collect_bigdata_operator_hive $hive_bin_path $hive_local_mode $omnioperator_dir collect_bigdata_hbase collect_bigdata_tune_up $omniadvisor_dir $mysql_username $mysql_password $mysql_database_name collect_bigdata_omnimv "$omnimv_dir" "$omnimv_dir_clickhouse" -- Gitee From 4af092cac8cf5b2574bf9ccae2e2488056f1c6b4 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 21 Nov 2024 03:51:29 +0000 Subject: [PATCH 14/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 7c67197..960c1f2 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -72,6 +72,18 @@ spark的安装路径，默认为/home/ +> hive_bin_path=hive + +hive可执行路径 + + + +> hive_local_mode=0 + +是否使用 local 模式启动 hive + + + > omnioperator_dir=/opt/omni-operator 算子加速的安装路径，默认为/opt/omni-operator -- Gitee From cc42833a9e5ea766f16fbc80c3b7bac56c56c11c Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 21 Nov 2024 03:57:02 +0000 Subject: [PATCH 15/35] update tools/config.ini. Signed-off-by: ksun --- tools/config.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/config.ini b/tools/config.ini index 9f597d3..877b831 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -75,9 +75,13 @@ algorithms_path=/home/KAL_TEST_DIR algorithms_name=algorithms_name1 algorithms_name2 algorithms_name3 dataset_list=dataset1 dataset2 dataset3 -# 算子加速 spark_path=/usr/local/spark +# 算子加速 Spark 场景配置 database=tpcds_bin_partitioned_varchar_orc_2 +# 算子加速 Hive 场景配置 +hive_bin_path=hive +hive_local_mode=0 +# 算子加速安装目录 omnioperator_dir=/opt/omni-operator # 参数调优 -- Gitee From 6daccaaa3d5db4ee548257ce782837216c0d0d36 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 22 Nov 2024 07:20:44 +0000 Subject: [PATCH 16/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 24dc078..d79c823 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -926,7 +926,7 @@ main(){ acc_lib=$(acquire_value HPC acc_lib) sme=$(acquire_value HPC sme) sve_source_code=$(acquire_value HPC sve_source_code) - collect_hpc_acceleration_library $acc_lib + collect_hpc_acceleration_library "$acc_lib" collect_sme_acceleration_library $sme collect_sve_source_code $sve_source_code echo "HPC collect msg Done..." -- Gitee From 7ec2a9a54de817d23c01a64e8849d6bf8bcfc702 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 27 Feb 2025 12:30:31 +0000 Subject: [PATCH 17/35] =?UTF-8?q?update=20tools/config.ini.=202025=20?= =?UTF-8?q?=E6=96=B0=E7=89=88=E6=A0=87=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ksun --- tools/config.ini | 55 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/tools/config.ini b/tools/config.ini index 877b831..a70f0ba 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -38,18 +38,22 @@ greenplum_kae_sql= [Virtual] check=False -# OpenStack 验证需要 -server_name=vm1 -network=port1_vdpa01 -flavor=4U4G80G -volume=ceph1_centos2 -availability_zone=nova:dpu01 -# 高性能云盘 -vm_ip=71.14.48.104 -vm_user=root -vm_password=Huawei12#$ -vm_ceph_disk_name=vdb -vm_name=vm1 + +# 拓扑感知 +topo_vm_name=vm_name +topo_vm_user=root +topo_vm_password=password +topo_numa_count=4 + +# 虚实协同 +real_vm_name=vm_name +real_vm_user=root +real_vm_password=password + +# 虚实协同 +vcpu_vm_name=vm_name +vcpu_vm_user=root +vcpu_vm_password=password [Acclib] @@ -67,6 +71,11 @@ math_jar=/home/math.jar # 引用数学库的字节码文件位置 math_java=/home/math.class +crc32_path=/home/test.bin + +# 压缩库 +zstd_bin=/home/test.bin +lz4_bin=/home/test.bin [Bigdata] check=False @@ -107,6 +116,11 @@ zookeeper_path=/sdi/status # shuffle加速 shuffle_jars=/home/ockadmin/opt/ock/jar/* shuffle_ock_path=/home/ockadmin/opt/ock +# 机密大数据 +omnishield_jar=/home/omnishield/omnishield-1.0-SNAPSHOT.jar + +# doris 加速 +doris_install_dir=/opt/tools/installed/doris-2.1.2-rc04 [Storage] check=False @@ -119,9 +133,26 @@ ucache_bin=/tmp/ucache.bin non_ceph_bin=/tmp/non_ceph.bin non_ceph_pid=38799 +# KAE使能SPDK 中的 crc 特性验证 +nvmf_tgt_bin=/tmp/nvmf_tgt +# RDMA网络加速 +# ceph14 请填写ceph配置文件路径，ceph17请填写osd配置文件路径，比如 /var/lib/ceph/{ceph cluster id}/osd.0/config +rdma_ceph_conf=/etc/ceph/ceph.conf +ceph_bin=ceph + +# BoostIO +boostio_bin=/tmp/bio_test + [HPC] check=False acc_lib=/home/lib sme=/home/lib # 填写用到 sve 的源码目录或文件的绝对路径 sve_source_code=/tmp/src + +[BishengJDK] +check=False +bisheng_jdk_dir=/home/bisheng_jdk +bisheng_jdk_heap_pid=6321 +kae_log_path=/tmp/kae.log +kae_zlib_pid=313125 -- Gitee From 879017ec7d0a0ae4c7674ecfa647c7ac7bdd2dd1 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 27 Feb 2025 12:30:51 +0000 Subject: [PATCH 18/35] =?UTF-8?q?update=20tools/collect=5Fmsg.sh.=202025?= =?UTF-8?q?=E6=96=B0=E7=89=88=E6=A0=87=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ksun --- tools/collect_msg.sh | 273 +++++++++++++++++++++++++++++++++---------- 1 file changed, 210 insertions(+), 63 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index d79c823..3eb37fe 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -2,7 +2,7 @@ current_path=$(pwd) config_file=$current_path/config.ini log_path=$current_path/log -default_project="Bigdata Database Storage Arm Virt Acclib Virtual HPC" +default_project="Bigdata Database Storage Arm Virt Acclib Virtual HPC BishengJDK" @@ -107,65 +107,84 @@ collect_virtual_host(){ } -collect_virtual_dpu(){ - server_name=$1 - network=$2 - flavor=$3 - volume=$4 - availability_zone=$5 +collect_virtual_topo(){ + vm_name=$1 + vm_user=$2 + vm_password=$3 + numa_count=$4 + expect << EOF | grep available | awk '{print $2}' > $log_path/virtual_topo.log + spawn virsh console $vm_name + expect "Escape character is \\^]" + send "\r" + expect "login:" + send "$vm_user\r" + expect "Password:" + send "$vm_password\r" + expect "# " + send "numactl -H \r" + expect "# " + send "exit\r" + expect eof +EOF + echo $numa_count >> $log_path/virtual_topo.log +} - # 需要再DPU测执行 - dpak_ovs_ctl_res=$(dpak-ovs-ctl -h 2>&1) - if [[ $dpak_ovs_ctl_res =~ "command not found" ]]; - then - echo "请确定已在DPU测执行该工具" - echo "dpak_ovs_ctl: command not found" > $log_path/virtual_dpu.log - else - echo "dpak_ovs_ctl version: $dpak_ovs_ctl_res" > $log_path/virtual_dpu.log - script -a -c 'dpak-ovs-ctl hwoff/dump-hwoff-flows' $log_path/virtual_dpu.log - fi - /usr/libexec/spdk/scripts/hw_dpu_rpc.py get_version >> $log_path/virtual_dpu.log 2>&1 - /usr/libexec/spdk/scripts/hw_dpu_rpc.py get_controllers >> $log_path/virtual_dpu.log 2>&1 +collect_virtual_real(){ + vm_name=$1 + vm_user=$2 + vm_password=$3 + expect << EOF > $log_path/virtual_real.log + spawn virsh console $vm_name + expect "Escape character is \\^]" + send "\r" + expect "login:" + send "$vm_user\r" + expect "Password:" + send "$vm_password\r" + expect "# " + send "dmesg | grep PV \r" + expect "# " + send "exit\r" + expect eof +EOF +} - # 创建虚机 - openstack_res=$(openstack --version 2>&1) - if [[ $openstack_res =~ "command not found" ]]; - then - echo "请确定已在DPU测执行该工具" - echo "openstack: command not found" >> $log_path/virtual_dpu.log - else - echo "openstack version: $openstack_res" >> $log_path/virtual_dpu.log - openstack server create $server_name --network $network --flavor $flavor --volume $volume --availability-zone $availability_zone >> $log_path/virtual_dpu.log - echo "等待虚机创建完成" - sleep 120 - echo "server_name: $server_name" >> $log_path/virtual_dpu.log - openstack server list >> $log_path/virtual_dpu.log +collect_virtual_vcpu(){ + vm_name=$1 + vm_user=$2 + vm_password=$3 + vcpu_count=$4 + echo $vcpu_count > $log_path/virtual_vcpu.log + if virsh setvcpus test_vm --count $vcpu_count --live; then + expect << EOF >> $log_path/virtual_vcpu.log + spawn virsh console $vm_name + expect "Escape character is \\^]" + send "\r" + expect "login:" + send "$vm_user\r" + expect "Password:" + send "$vm_password\r" + expect "# " + send "lscpu | grep 'CPU(s)' | head -n 1 | cut -d: \r" + expect "# " + send "exit\r" + expect eof +EOF fi } -collect_virtual_ceph(){ - vm_ip=$1 - vm_user=$2 - vm_password=$3 - vm_ceph_disk_name=$4 - vm_name=$5 - set -x - sshpass -p "$vm_password" ssh "$vm_user"@"$vm_ip" ls -la /sys/block/ | grep $vm_ceph_disk_name | tail -n 1 > $log_path/virtual_ceph_disk.log - virsh dumpxml $vm_name > $log_path/virtual_ceph_vm_xml.log - rpc vhost_get_controllers > $log_path/virtual_ceph_vm_controllers.log - rpc bdev_get_bdevs > $log_path/virtual_ceph_bdev.log +collect_k8s_numa(){ + kubectl get pod | grep numaadj- > $log_path/virtual_k8s_numa.log } -collect_virtual_ovs_xpf(){ - ovs-appctl hwoff/dump-hwoff-flows > $log_path/virtual_ovs_xpf.log - echo $? >> $log_path/virtual_ovs_xpf.log +collect_k8s_mpam(){ + kubectl get pod | grep mpam-controller-daemon-agent- > $log_path/virtual_k8s_mpam.log } - ################数据库特性信息收集################## collect_database(){ mysql_install_path=$1 @@ -219,7 +238,7 @@ collect_database_other_db(){ readelf -a $other_db_bin | grep bolt > $log_path/database_other_db.log objdump -d $other_db_bin | grep crc32cb >> $log_path/database_other_db.log if [ "$kae_version" == "1.0" ]; then - nohup timout 20 watch -gt -n 0.2 cat /sys/class/uacce/hisi_zip*/attrs/available_instances > $log_path/database_greenplum_kae.log & + nohup timeout 20 watch -gt -n 0.2 cat /sys/class/uacce/hisi_zip*/attrs/available_instances > $log_path/database_greenplum_kae.log & else nohup timeout 20 watch -gt -n 0.2 cat /sys/class/uacce/hisi_zip*/available_instances > $log_path/database_greenplum_kae.log & fi @@ -298,6 +317,25 @@ collect_acceleration_library(){ } +collect_acceleration_library_crc32(){ + crc32_path=$1 + objdump -d $crc32_path | grep crc32cb > $log_path/acceleration_library_crc32.log +} + + +collect_acceleration_library_kail_dnn(){ + env | grep LD_LIBRARY_PATH | grep '/usr/local/kail/lib' > $log_path/acceleration_library_kail_dnn.log +} + + +collect_acceleration_library_compress(){ + zstd_bin=$1 + lz4_bin=$2 + ldd $zstd_bin > $log_path/acceleration_library_compress.log + ldd $lz4_bin | grep liblz4.so | awk '{print $3}' | xargs -I {} nm -D {} >> $log_path/acceleration_library_compress.log + ldd gzip >> $log_path/acceleration_library_compress.log +} + ###############分布式存储特性信息收集############### # $1 ec_pool 名字 collect_storage_acc(){ @@ -344,6 +382,27 @@ collect_storage_acc(){ } +collect_storage_spdk(){ + nvmf_tgt_bin=$1 + ls /usr/local/lib/engines-*/ > $log_path/storage_kae.log + ldd $nvmf_tgt_bin > $log_path/storage_spdk_crc.log +} + + +collect_storage_rdma(){ + rdma_ceph_conf=$1 + ceph_bin=$2 + cat $rdma_ceph_conf > $log_path/storage_rdma.log + $ceph_bin -s >> $log_path/storage_rdma.log +} + + +collect_storage_boostio(){ + boostio_bin=$1 + ps -ef | grep bio > $log_path/storage_boostio.log + ldd $boostio_bin >> $log_path/storage_boostio.log +} + ###############大数据特性信息收集################## collect_bigdata_kal(){ algotithm_list=$1 @@ -744,11 +803,31 @@ EOF } +collect_bigdata_omni_shield(){ + spark_path=$1 + omnishield_jar=$2 + $spark_path/bin/spark-submit --class org.apache.spark.examples.SparkPi --master local \ + --conf spark.driver.extraClassPath=$omnishield_jar --conf spark.executor.extraClassPath=$omnishield_jar \ + --jars $omnishield_jar --executor-memory 20G --num-executors 2 --conf spark.authenticate=true \ + --conf spark.authenticate.secret=testsecret --conf spark.network.crypto.enabled=true \ + --conf spark.network.crypto.config.cipher.classes=com.huawei.boostkit.omnishield.cipher.SM4Cipher \ + --conf spark.network.crypto.cipher=SM4/GCM/NOPadding $spark_path/examples/jars/spark-examples_2.12-3.3.1.jar > $log_path/bigdata_omni_shield.log 2>&1 +} + + +collect_bigdata_doris() { + doris_install_dir=$1 + grep neonmovemask_bulk $doris_install_dir/thirdparty/src/bitshuffle-0.5.1/src/bitshuffle_core.c > $log_path/bigdata_doris.log 2>&1 +} + + collect_bigdata_components(){ spark_path=$1 + doris_install_dir=$2 $spark_path/bin/spark-sql --version && echo spark > $log_path/bigdata_components.log hive --version && echo hive >> $log_path/bigdata_components.log hbase version && echo hbase >> $log_path/bigdata_components.log + ls $doris_install_dir && echo doris >> $log_path/bigdata_components.log } @@ -783,6 +862,44 @@ collect_sve_source_code(){ } +################毕昇JDK信息收集####################### +collect_bisheng_jdk_heap(){ + check_sysbol=`objdump -tT /usr/lib64/libjvm*_kunpeng.so | grep HeapDict_Add` + if [[ $check_sysbol = '' ]];then + echo "failed" > $log_path/bisheng_jdk_heap.log + return + fi + + jmap_cmd=$1"/bin/jmap" + pid=$2 + dump_file="/tmp/"${pid}"_dump.hprof" + $jmap_cmd -dump:format=b,file=$dump_file,HeapDumpRedact=names,RedactMap="value:#**#" $pid > $log_path/bisheng_jdk_heap.log + + rm -rf $dump_file +} + + +collect_bisheng_jdk_jbooster(){ + readelf -Ws /usr/lib64/libjvm17_kunpeng.so | grep JBooster > $log_path/bisheng_jdk_jbooster.log +} + + +collect_bisheng_jdk_jbolt(){ + readelf -Ws /usr/lib64/libjvm*_kunpeng.so | grep JBolt > $log_path/bisheng_jdk_jbolt.log +} + + +collect_bisheng_jdk_kae_provider(){ + kae_log_path=$1 + cp $kae_log_path $log_path/bisheng_jdk_kae_provider.log +} + + +collect_bisheng_jdk_kae_zlib(){ + kae_zlib_pid=$1 + pmap -p $kae_zlib_pid > $log_path/bisheng_jdk_kae_zlib.log +} + ################环境信息收集####################### collect_os_msg(){ echo os: $(cat /etc/os-release |grep PRETTY_NAME=|awk -F= '{print $2}') > $log_path/os_msg.log @@ -853,6 +970,12 @@ main(){ math_jar=$(acquire_value Acclib math_jar) math_java=$(acquire_value Acclib math_java) collect_acceleration_library "$system_lib" "$hmpp_lib" "$math_lib" "$math_jar" "$math_java" + crc32_path=$(acquire_value Acclib crc32_path) + collect_acceleration_library_crc32 $crc32_path + collect_acceleration_library_kail_dnn + zstd_bin=$(acquire_value Acclib zstd_bin) + lz4_bin=$(acquire_value Acclib lz4_bin) + collect_acceleration_library_compress "$zstd_bin" "$lz4_bin" echo "acceleration_library collect msg Done..." elif [ $per_project = "Storage" ]; then @@ -865,6 +988,13 @@ main(){ non_ceph_bin=$(acquire_value Storage non_ceph_bin) non_ceph_pid=$(acquire_value Storage non_ceph_pid) collect_storage_acc "$ec_pool_name" "$ceph_conf" "$storage_maintain_bin" "$rocksdb_bin" "$ucache_bin" "$non_ceph_bin" "$non_ceph_pid" + nvmf_tgt_bin=$(acquire_value Storage nvmf_tgt_bin) + collect_storage_spdk "$nvmf_tgt_bin" + rdma_ceph_conf=$(acquire_value Storage rdma_ceph_conf) + ceph_bin=$(acquire_value Storage ceph_bin) + collect_storage_rdma "$rdma_ceph_conf" "$ceph_bin" + boostio_bin=$(acquire_value Storage boostio_bin) + collect_storage_boostio $boostio_bin echo "Storage collect msg Done..." elif [ $per_project = "Bigdata" ]; then @@ -891,8 +1021,9 @@ main(){ zookeeper_address=$(acquire_value Bigdata zookeeper_address) zookeeper_path=$(acquire_value Bigdata zookeeper_path) shuffle_jars=$(acquire_value Bigdata shuffle_jars) - shuffle_ock_path=$(acquire_value Bigdata shuffle_ock_path) - collect_bigdata_components $spark_path + omnishield_jar=$(acquire_value Bigdata omnishield_jar) + doris_install_dir=$(acquire_value Bigdata doris_install_dir) + collect_bigdata_components $spark_path $doris_install_dir collect_bigdata_kal "${algorithms_list[@]}" $algorithms_path "${dataset_list[@]}" collect_bigdata_operator_spark $spark_path $database $omnioperator_dir collect_bigdata_operator_hive $hive_bin_path $hive_local_mode $omnioperator_dir @@ -901,24 +1032,29 @@ main(){ collect_bigdata_omnimv "$omnimv_dir" "$omnimv_dir_clickhouse" collect_bigdata_omni_push_down "$omnidata_launcher_server" "$omnidata_launcher" "$push_down_jars" "$push_down_conf" "$spark_path" "$database" "$omnidata_install_path" "$zookeeper_address" "$zookeeper_path" collect_bigdata_omni_shuffle "$spark_path" "$shuffle_jars" "$database" "$shuffle_ock_path" + collect_bigdata_omni_shield $spark_path $omnishield_jar + collect_bigdata_doris $doris_install_dir echo "Bigdata collect msg Done..." elif [ $per_project = "Virtual" ]; then echo "start collect Virtual msg..." collect_virtual_host - server_name=$(acquire_value Virtual server_name) - network=$(acquire_value Virtual network) - flavor=$(acquire_value Virtual flavor) - volume=$(acquire_value Virtual volume) - availability_zone=$(acquire_value Virtual availability_zone) - collect_virtual_dpu $server_name $network $flavor $volume $availability_zone - vm_ip=$(acquire_value Virtual vm_ip) - vm_user=$(acquire_value Virtual vm_user) - vm_password=$(acquire_value Virtual vm_password) - vm_ceph_disk_name=$(acquire_value Virtual vm_ceph_disk_name) - vm_name=$(acquire_value Virtual vm_name) - collect_virtual_ceph "$vm_ip" "$vm_user" "$vm_password" "$vm_ceph_disk_name" "$vm_name" - collect_virtual_ovs_xpf + topo_vm_name=$(acquire_value Virtual topo_vm_name) + topo_vm_user=$(acquire_value Virtual topo_vm_user) + topo_vm_password=$(acquire_value Virtual topo_vm_password) + topo_numa_count=$(acquire_value Virtual topo_numa_count) + collect_virtual_topo "$topo_vm_name" "$topo_vm_user" "$topo_vm_password" "$topo_numa_count" + real_vm_name=$(acquire_value Virtual real_vm_name) + real_vm_user=$(acquire_value Virtual real_vm_user) + real_vm_password=$(acquire_value Virtual real_vm_password) + collect_virtual_real "$real_vm_name" "$real_vm_user" "$real_vm_password" + vcpu_vm_name=$(acquire_value Virtual vcpu_vm_name) + vcpu_vm_user=$(acquire_value Virtual vcpu_vm_user) + vcpu_vm_password=$(acquire_value Virtual vcpu_vm_password) + vcpu_count=$(acquire_value Virtual vcpu_count) + collect_virtual_vcpu "$vcpu_vm_name" "$vcpu_vm_user" "$vcpu_vm_password" "$vcpu_count" + collect_k8s_numa + collect_k8s_mpam echo "Virtual collect msg Done..." elif [ $per_project = "HPC" ]; then @@ -930,6 +1066,17 @@ main(){ collect_sme_acceleration_library $sme collect_sve_source_code $sve_source_code echo "HPC collect msg Done..." + elif [ $per_project = "BishengJDK" ]; + then + bisheng_jdk_dir=$(acquire_value Acclib bisheng_jdk_dir) + bisheng_jdk_heap_pid=$(acquire_value Acclib bisheng_jdk_heap_pid) + collect_bisheng_jdk_heap $bisheng_jdk_dir $bisheng_jdk_heap_pid + collect_bisheng_jdk_jbooster + collect_bisheng_jdk_jbolt + kae_log_path=$(acquire_value Acclib kae_log_path) + collect_bisheng_jdk_kae_provider $kae_log_path + kae_zlib_pid=$(acquire_value Acclib kae_zlib_pid) + collect_bisheng_jdk_kae_zlib $kae_zlib_pid fi done } -- Gitee From 09e19db788b1fb5cdb0fe77f2adda82a6cc20ed8 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 27 Feb 2025 12:31:29 +0000 Subject: [PATCH 19/35] =?UTF-8?q?update=20tools/Boostkit=5FISV=5FTool.md.2?= =?UTF-8?q?025=E6=96=B0=E7=89=88=E6=A0=87=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 109 +++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 22 deletions(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 960c1f2..8bfd80a 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -14,6 +14,7 @@ - **注意**：只需修改要采集的特性相关的配置，其余配置保持不变即可 - **注意**：采集过程中会尝试采集对应解决方案下的所有特性，因此对未使能特性的采集出现报错是正常现象，不影响最终结果 +- 新版工具同时支持对毕昇JDK、HPCkit特性的信息收集 ### **1.2采集工具配置文件说明** @@ -156,6 +157,14 @@ shuffle加速相关jar包路径，用于shuffle加速特性验证 shuffle加速 ock 安装路径，用于shuffle加速特性验证 +> omnishield_jar=/home/omnishield/omnishield-1.0-SNAPSHOT.jar + +omnishield相关的jar包路径，用于机密大数据特性验证 + +> doris_install_dir=/opt/tools/installed/doris-2.1.2-rc04 + +doris的安装目录，用于Doris指令优化特性验证 + #### **1.2.2分布式存储配置文件说明** ![bigdata](./BoostKit_pic/storage.png) @@ -194,13 +203,28 @@ ceph 配置文件路径，用于“压缩算法”特性验收 > non_ceph_bin=/tmp/non_ceph.bin -非ceph场景下的软件二进制路径，用于“存储加速算法库”特性验收 +非ceph场景下的软件二进制路径，用于“存储加速算法库(KSAL)”特性验收 > non_ceph_pid=38799 -非ceph场景下的软件进程id，用于“存储加速算法库”特性验收 +非ceph场景下的软件进程id，用于“存储加速算法库(KSAL)”特性验收 +> nvmf_tgt_bin=/tmp/nvmf_tgt +nvmf_tgt 二进制路径，用于“KAE使能SPDK“中的crc特性验收 + +> rdma_ceph_conf=/etc/ceph/ceph.conf + +ceph14 请填写ceph配置文件路径，ceph17请填写osd配置文件路径，比如 /var/lib/ceph/{ceph cluster id}/osd.0/config. +用于“RDMA网络加速“特性验收 + +> ceph_bin=ceph + +ceph 二进制路径，用于“RDMA网络加速”特性验收 + +> boostio_bin=/tmp/bio_test + +集成BOOSTIO的软件可执行文件路径。boostio集成部署时，需要填写该字段，用于“BoostIO”特性验收 #### **1.2.3数据库配置文件说明** @@ -309,6 +333,17 @@ check 默认为False, 置为True 则开启加速库特性使能信息收集功 引用数学库的字节码文件位置 +> crc32_path=/home/test.bin + +使用到了“CRC32指令优化”的二进制或动态库的路径，用于“CRC32指令优化”特性验收 + +> zstd_bin=/home/test.bin + +使用了 zstd压缩算法的二进制文件路径，用于“Zstd压缩算法”特性验收 + +> lz4_bin=/home/test.bin + +使用了 lz4压缩算法的二进制文件路径，用于“Lz4压缩算法”特性验收 #### **1.2.6虚拟化配置文件说明** @@ -319,43 +354,51 @@ check 默认为False, 置为True 则开启加速库特性使能信息收集功 注意： 1.OVS流表网卡加速特性需再host测执行，配置文件无需修改 +2.K8S NUMA自适应插件, K8S MPAM插件 验收时配置文件无需修改 + +> topo_vm_name=vm_name + +虚拟机名称，用于“拓扑感知”特性验收 -2.虚拟化DPU卸载 特性需要再DPU测执行采集工具， 配置文件无需修改 +> topo_vm_user=root -3.OVS流表归一化 特性验证前需准备环境，让物理机B上的虚拟机持续ping物理机A上的虚拟机，然后在物理机A上运行采集工具 -该特性默认支持验收， 配置文件无需修改 +虚拟机用户名，用于“拓扑感知”特性验收 -OpenStack 验证需要以下信息 +> topo_vm_password=password -> server_name=vm1 +虚拟机用户密码，用于“拓扑感知”特性验收 -> network=port1_vdpa01 +> topo_numa_count=4 -> flavor=4U4G80G +虚拟机xml中配置的 NUMA 个数，用于“拓扑感知”特性验收 -> volume=ceph1_centos2 +> real_vm_name=vm_name -> availability_zone=nova:dpu01 +虚拟机名称，用于“虚实协同”特性验收 -> vm_ip=71.14.48.104 +> real_vm_user=root -虚拟机ip, 用于验收高性能云盘优化特性 +虚拟机用户名，用于“虚实协同”特性验收 -> vm_user=root +> real_vm_password=password -虚拟机用户名, 用于验收高性能云盘优化特性 +虚拟机用户密码，用于“虚实协同”特性验收 -> vm_password=Huawei12#$ +> vcpu_vm_name=vm_name -虚拟机密码, 用于验收高性能云盘优化特性 +虚拟机名称，用于“vCPU热插拔”特性验收 -> vm_ceph_disk_name=vdb +> vcpu_vm_user=root -虚拟机使用的高性能云盘名称, 用于验收高性能云盘优化特性 +虚拟机用户名，用于“vCPU热插拔”特性验收 -> vm_name=vm1 +> vcpu_vm_password=password -虚拟机名称, 用于验收高性能云盘优化特性 +虚拟机用户密码，用于“vCPU热插拔”特性验收 + +> vcpu_count=4 + +虚拟机cpu热插拔后cpu数量，用于“vCPU热插拔”特性验收 #### **1.2.7 机密计算配置文件说明** @@ -404,6 +447,29 @@ check 默认为False, 置为True 则开启HPC相关特性使能信息收集功 填写有用到 sve 的源码目录或文件的绝对路径 + +#### **1.2.9 毕昇JDK配置文件说明** + +> check=False + +check 默认为False, 置为True 则开启毕昇JDK相关特性使能信息收集功能 + +> bisheng_jdk_dir=/home/bisheng_jdk + +毕昇jdk安装目录，用于"毕昇JDK加速库(堆转储增强)"特性验收 + +> bisheng_jdk_heap_pid=6321 + +使用了堆转储增强的进程id，用于"毕昇JDK加速库(堆转储增强)"特性验收 + +> kae_log_path=/tmp/kae.log + +kae日志路径，用于"KAEProvider"特性验收 + +> kae_zlib_pid=313125 + +使能了KAE zlib的进程id，用于"KAE zlib"特性验收 + ### 1.3 采集工具使用说明 #### 1.3.1 二进制使用 @@ -420,4 +486,3 @@ bash collect_msg.sh # 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 ``` - -- Gitee From b63b13dcbf34f2012372ded0d6415be8d9809a00 Mon Sep 17 00:00:00 2001 From: ksun Date: Wed, 5 Mar 2025 09:04:12 +0000 Subject: [PATCH 20/35] =?UTF-8?q?update=20tools/collect=5Fmsg.sh.=20arm?= =?UTF-8?q?=E5=8E=9F=E7=94=9F=E9=87=87=E9=9B=86=E8=84=9A=E6=9C=AC=E5=8E=BB?= =?UTF-8?q?=E9=99=A4=E5=AE=B9=E5=99=A8=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ksun --- tools/collect_msg.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 3eb37fe..d0bfc6b 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -67,11 +67,7 @@ collect_arm_native(){ kbox_container=$1 video_container=$2 instruction_container=$3 - # check tested container whether exist - containers=($1 $2 $3) - for i in ${containers[@]};do - docker ps -a | grep -wq $i || { echo "the $i container doesn't found,please check!"; exit 1; } - done + # kbox基础云手机 rm -f $log_path/arm_native.log -- Gitee From 4ddba541f544e93a1034145a4f78692f3f463988 Mon Sep 17 00:00:00 2001 From: ksun Date: Sat, 29 Mar 2025 08:54:35 +0000 Subject: [PATCH 21/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index d0bfc6b..a8abda5 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -139,7 +139,7 @@ collect_virtual_real(){ expect "Password:" send "$vm_password\r" expect "# " - send "dmesg | grep PV \r" + send "dmesg | grep PV --color=never \r" expect "# " send "exit\r" expect eof -- Gitee From 04be86a124d6abf469143634a16d7719039f74cc Mon Sep 17 00:00:00 2001 From: ksun Date: Sat, 29 Mar 2025 08:55:07 +0000 Subject: [PATCH 22/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 8bfd80a..013760e 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -355,6 +355,7 @@ check 默认为False, 置为True 则开启加速库特性使能信息收集功 1.OVS流表网卡加速特性需再host测执行，配置文件无需修改 2.K8S NUMA自适应插件, K8S MPAM插件 验收时配置文件无需修改 +3.virsh console 命令同时只能有一个用户在线上，采集前需让已登录的用户退出 > topo_vm_name=vm_name -- Gitee From 8d1ccf2e01f17f341a36894a19af19f513ea6899 Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 31 Mar 2025 09:05:22 +0000 Subject: [PATCH 23/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 013760e..e03bd24 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -297,7 +297,7 @@ check 默认为False, 置为True 则开启ARM原生特性使能信息收集功 > instuction_container=kbox_3 -分别为待采集的kbox、视频流、指令流容器名称。要求3个容器均存在，否则会退出脚本。 +分别为待采集的kbox、视频流、指令流容器名称。 @@ -435,17 +435,13 @@ check 默认为False, 置为True 则开启数据库相关特性使能信息收 check 默认为False, 置为True 则开启HPC相关特性使能信息收集功能 -> acc_lib +> common_acc_lib_bin=/tmp/binary -引用HPC加速库改造二进制文件绝对路径, 多个路径用空格分隔 +引用HPC通用加速库改造的二进制文件绝对路径, 多个路径用空格分隔 -> sme +> domain_acc_lib_bin=/tmp/binary -引用HPC SME改造二进制文件绝对路径 - -> sve_source_code - -填写有用到 sve 的源码目录或文件的绝对路径 +引用HPC领域加速库改造的二进制文件绝对路径, 多个路径用空格分隔 @@ -487,3 +483,24 @@ bash collect_msg.sh # 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 ``` + + +## 1. **分析工具** + +### 1.1 **分析工具能力说明** + + 用于分析鲲鹏使能套件BoostKit 特性使能情况，以及系统信息。工具输入信息来源为信息收集工具生成的tar.gz 压缩包，输出特性使能情况 以及服务器相关信息的html 报告 + + 如下图 new_report.html + +![bigdata](./BoostKit_pic/all2.png)36723.png) + +### 1.2 **分析工具使用说明** + +``` +./isv_tool -tool BoostKit -package ./xxx技术限公司_isv_msg_202404081100.tar.gz +``` + +-tool # 指定分析BoostKit 特性使能情况 + +-package # 输入采集压缩包，一般由采集工具一键收集生成的压缩包名称 \ No newline at end of file -- Gitee From f951274c7cd140b86b86db6f5b0fb294043e963d Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 31 Mar 2025 09:05:38 +0000 Subject: [PATCH 24/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 39 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index a8abda5..b016c2e 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -830,34 +830,17 @@ collect_bigdata_components(){ #################HPC特性信息收集################## # $1 #用户可执行文件路径 -collect_hpc_acceleration_library(){ - bin_path=$1 - rm -rf $log_path/hpc_acceleration_library.log - touch $log_path/hpc_acceleration_library.log - ldd $bin_path > $log_path/hpc_acceleration_library.log 2>&1 +collect_hpc_common_acceleration_library(){ + common_acc_lib_bin=$1 + ldd $common_acc_lib_bin > $log_path/hpc_common_acceleration_library.log 2>&1 } -# $1 #用户可执行文件路径 -collect_sme_acceleration_library(){ - bin_path=$1 - ifsme=`lscpu|grep Flags|grep sme` - if [ -n "$ifsme" ]; then - bin_path=$1 #用户可执行文件路径 - rm -rf $log_path/hpc_SME_library.log - touch $log_path/hpc_SME_library.log - ldd $bin_path | grep SME >> $log_path/hpc_SME_library.log 2>&1 - objdump -d $bin_path >> $log_path/hpc_SME_library.log 2>&1 - else - echo "架构不支持sme" >> $log_path/hpc_SME_library.log - fi +collect_hpc_domain_acceleration_library(){ + domain_acc_lib_bin=$1 + ldd $domain_acc_lib_bin > $log_path/hpc_domain_acceleration_library.log 2>&1 } -collect_sve_source_code(){ - grep -r arm_sve.h "$1" > $log_path/hpc_sve.log -} - - ################毕昇JDK信息收集####################### collect_bisheng_jdk_heap(){ check_sysbol=`objdump -tT /usr/lib64/libjvm*_kunpeng.so | grep HeapDict_Add` @@ -1055,12 +1038,10 @@ main(){ elif [ $per_project = "HPC" ]; then echo "start collect HPC msg..." - acc_lib=$(acquire_value HPC acc_lib) - sme=$(acquire_value HPC sme) - sve_source_code=$(acquire_value HPC sve_source_code) - collect_hpc_acceleration_library "$acc_lib" - collect_sme_acceleration_library $sme - collect_sve_source_code $sve_source_code + common_acc_lib_bin=$(acquire_value HPC common_acc_lib_bin) + collect_hpc_common_acceleration_library "$common_acc_lib_bin" + domain_acc_lib_bin=$(acquire_value HPC domain_acc_lib_bin) + collect_hpc_domain_acceleration_library "$domain_acc_lib_bin" echo "HPC collect msg Done..." elif [ $per_project = "BishengJDK" ]; then -- Gitee From 8a28e8430a8fc289073b70d35bb90f5ce644c29d Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 31 Mar 2025 09:05:53 +0000 Subject: [PATCH 25/35] update tools/config.ini. Signed-off-by: ksun --- tools/config.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/config.ini b/tools/config.ini index a70f0ba..42b441c 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -145,10 +145,8 @@ boostio_bin=/tmp/bio_test [HPC] check=False -acc_lib=/home/lib -sme=/home/lib -# 填写用到 sve 的源码目录或文件的绝对路径 -sve_source_code=/tmp/src +common_acc_lib_bin=/tmp/binary +domain_acc_lib_bin=/tmp/binary [BishengJDK] check=False -- Gitee From 2ae0f6ed9a801079a45036d97e82d97aedab0b61 Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 31 Mar 2025 09:07:55 +0000 Subject: [PATCH 26/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index e03bd24..471b192 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -469,38 +469,9 @@ kae日志路径，用于"KAEProvider"特性验收 ### 1.3 采集工具使用说明 -#### 1.3.1 二进制使用 - -``` -./collect_msg -# 同级目录下具有config.ini 配置文件，且已按1.2中对应解决方案完成相关参数配置 -``` - -#### 1.3.2 shell 脚本使用 - ``` bash collect_msg.sh # 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 ``` - -## 1. **分析工具** - -### 1.1 **分析工具能力说明** - - 用于分析鲲鹏使能套件BoostKit 特性使能情况，以及系统信息。工具输入信息来源为信息收集工具生成的tar.gz 压缩包，输出特性使能情况 以及服务器相关信息的html 报告 - - 如下图 new_report.html - -![bigdata](./BoostKit_pic/all2.png)36723.png) - -### 1.2 **分析工具使用说明** - -``` -./isv_tool -tool BoostKit -package ./xxx技术限公司_isv_msg_202404081100.tar.gz -``` - --tool # 指定分析BoostKit 特性使能情况 - --package # 输入采集压缩包，一般由采集工具一键收集生成的压缩包名称 \ No newline at end of file -- Gitee From 281fab1ccba57753ac7be1c37932379fa880a54f Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 3 Apr 2025 02:14:46 +0000 Subject: [PATCH 27/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index b016c2e..9c5d2de 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -879,6 +879,11 @@ collect_bisheng_jdk_kae_zlib(){ pmap -p $kae_zlib_pid > $log_path/bisheng_jdk_kae_zlib.log } + +collect_bisheng_jdk_g1gc(){ + ps -ef| grep java > $log_path/bisheng_jdk_g1gc.log +} + ################环境信息收集####################### collect_os_msg(){ echo os: $(cat /etc/os-release |grep PRETTY_NAME=|awk -F= '{print $2}') > $log_path/os_msg.log @@ -1054,6 +1059,7 @@ main(){ collect_bisheng_jdk_kae_provider $kae_log_path kae_zlib_pid=$(acquire_value Acclib kae_zlib_pid) collect_bisheng_jdk_kae_zlib $kae_zlib_pid + collect_bisheng_jdk_g1gc fi done } -- Gitee From 8f3a4abe821068eba92aca2e2057e25c37897dd2 Mon Sep 17 00:00:00 2001 From: ksun Date: Thu, 3 Apr 2025 02:43:39 +0000 Subject: [PATCH 28/35] update tools/collect_msg.sh. fix Signed-off-by: ksun --- tools/collect_msg.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 9c5d2de..20546fc 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -1050,14 +1050,14 @@ main(){ echo "HPC collect msg Done..." elif [ $per_project = "BishengJDK" ]; then - bisheng_jdk_dir=$(acquire_value Acclib bisheng_jdk_dir) - bisheng_jdk_heap_pid=$(acquire_value Acclib bisheng_jdk_heap_pid) + bisheng_jdk_dir=$(acquire_value BishengJDK bisheng_jdk_dir) + bisheng_jdk_heap_pid=$(acquire_value BishengJDK bisheng_jdk_heap_pid) collect_bisheng_jdk_heap $bisheng_jdk_dir $bisheng_jdk_heap_pid collect_bisheng_jdk_jbooster collect_bisheng_jdk_jbolt - kae_log_path=$(acquire_value Acclib kae_log_path) + kae_log_path=$(acquire_value BishengJDK kae_log_path) collect_bisheng_jdk_kae_provider $kae_log_path - kae_zlib_pid=$(acquire_value Acclib kae_zlib_pid) + kae_zlib_pid=$(acquire_value BishengJDK kae_zlib_pid) collect_bisheng_jdk_kae_zlib $kae_zlib_pid collect_bisheng_jdk_g1gc fi -- Gitee From c59692e02cbc8a5b107a7c7d36b79d63ae1d3fbc Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 7 Apr 2025 03:04:19 +0000 Subject: [PATCH 29/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 20546fc..eb55459 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -163,7 +163,7 @@ collect_virtual_vcpu(){ expect "Password:" send "$vm_password\r" expect "# " - send "lscpu | grep 'CPU(s)' | head -n 1 | cut -d: \r" + send "lscpu | grep 'CPU(s)' --color=never \r" expect "# " send "exit\r" expect eof -- Gitee From df170792df254b97af1b688493b8c2ec90928009 Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 7 Apr 2025 12:40:34 +0000 Subject: [PATCH 30/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index eb55459..854b47d 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -163,7 +163,7 @@ collect_virtual_vcpu(){ expect "Password:" send "$vm_password\r" expect "# " - send "lscpu | grep 'CPU(s)' --color=never \r" + send "lscpu | grep 'On-line CPU' --color=never \r" expect "# " send "exit\r" expect eof -- Gitee From 0b4d9d36839b9dc50f542bd20474a9985abec8fe Mon Sep 17 00:00:00 2001 From: ksun Date: Mon, 7 Apr 2025 12:40:46 +0000 Subject: [PATCH 31/35] update tools/config.ini. Signed-off-by: ksun --- tools/config.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/config.ini b/tools/config.ini index 42b441c..fe375c1 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -54,6 +54,7 @@ real_vm_password=password vcpu_vm_name=vm_name vcpu_vm_user=root vcpu_vm_password=password +vcpu_count=4 [Acclib] -- Gitee From 9a92a30455bcbedd17d9fe81804b37076feb74ce Mon Sep 17 00:00:00 2001 From: ksun Date: Tue, 8 Apr 2025 01:14:37 +0000 Subject: [PATCH 32/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 854b47d..82349b0 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -153,7 +153,7 @@ collect_virtual_vcpu(){ vm_password=$3 vcpu_count=$4 echo $vcpu_count > $log_path/virtual_vcpu.log - if virsh setvcpus test_vm --count $vcpu_count --live; then + if virsh setvcpus $vm_name --count $vcpu_count --live; then expect << EOF >> $log_path/virtual_vcpu.log spawn virsh console $vm_name expect "Escape character is \\^]" -- Gitee From 7ff1ff98580979c6b11a42542f1871464caf8976 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 25 Apr 2025 08:12:47 +0000 Subject: [PATCH 33/35] update tools/collect_msg.sh. Signed-off-by: ksun --- tools/collect_msg.sh | 287 ++++++++++++++++++++++++------------------- 1 file changed, 163 insertions(+), 124 deletions(-) diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 82349b0..480f1ae 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -2,7 +2,7 @@ current_path=$(pwd) config_file=$current_path/config.ini log_path=$current_path/log -default_project="Bigdata Database Storage Arm Virt Acclib Virtual HPC BishengJDK" +default_project="Bigdata Database Storage Arm Virt Acclib Virtual HPC" @@ -87,59 +87,100 @@ collect_arm_native(){ #################虚拟化特性信息收集################## -collect_virtual_host(){ - sudo systemctl status waasagent.service |grep "Active" > $log_path/virtual_sense.log - waasctl --version >> $log_path/virtual_sense.log +collect_virtual_ovs_xpf(){ + ovs-appctl hwoff/dump-hwoff-flows > $log_path/virtual_ovs_xpf.log + echo $? >> $log_path/virtual_ovs_xpf.log +} - ovs_appctl_res=$(ovs-appctl --version 2>&1) - if [[ $ovs_appctl_res =~ "command not found" ]]; - then - echo "ovs-appctl: command not found" > $log_path/virtual_dpu_flow.log - else - echo "ovs-appctl version: $ovs_appctl_res" > $log_path/virtual_dpu_flow.log - script -a -c 'ovs-appctl hwoff/dump-hwoff-flows' $log_path/virtual_dpu_flow.log - fi +collect_virtual_ceph(){ + vm_ip=$1 + vm_user=$2 + vm_password=$3 + vm_ceph_disk_name=$4 + vm_name=$5 + set -x + sshpass -p "$vm_password" ssh "$vm_user"@"$vm_ip" ls -la /sys/block/ | grep $vm_ceph_disk_name | tail -n 1 > $log_path/virtual_ceph_disk.log + virsh dumpxml $vm_name > $log_path/virtual_ceph_vm_xml.log + rpc vhost_get_controllers > $log_path/virtual_ceph_vm_controllers.log + rpc bdev_get_bdevs > $log_path/virtual_ceph_bdev.log } -collect_virtual_topo(){ +collect_virtual_schedule(){ vm_name=$1 vm_user=$2 vm_password=$3 - numa_count=$4 - expect << EOF | grep available | awk '{print $2}' > $log_path/virtual_topo.log + expect << EOF > $log_path/virtual_schedule.log spawn virsh console $vm_name expect "Escape character is \\^]" send "\r" expect "login:" - send "$vm_user\r" + send {$vm_user} + send "\r" expect "Password:" - send "$vm_password\r" + send {$vm_password} + send "\r" expect "# " - send "numactl -H \r" + send "numactl -H | grep available --color=never \r" + expect "# " + send "cat /proc/sys/kernel/sched_cluster \r" + expect "# " + send "dmesg | grep PV --color=never \r" + expect "# " + send "dmesg | grep 'NMI watchdog' --color=never \r" + expect "# " + send "lscpu \r" expect "# " send "exit\r" expect eof EOF - echo $numa_count >> $log_path/virtual_topo.log + + virsh dumpxml $vm_name | grep memnode > $log_path/virtual_schedule_xml.log + virsh dumpxml $vm_name | grep cacheinfo >> $log_path/virtual_schedule_xml.log } -collect_virtual_real(){ - vm_name=$1 - vm_user=$2 - vm_password=$3 - expect << EOF > $log_path/virtual_real.log +collect_virtual_hot_swap(){ + hot_swap_vm_name=$1 + hot_swap_vm_user=$2 + hot_swap_vm_password=$3 + hot_swap_cpu_count=$4 + hot_swap_mem_xml_path=$5 + + echo $hot_swap_cpu_count > $log_path/virtual_hot_swap_vcpu.log + if virsh setvcpus $hot_swap_vm_name --count $hot_swap_cpu_count --live; then + expect << EOF >> $log_path/virtual_hot_swap_vcpu.log + spawn virsh console $vm_name + expect "Escape character is \\^]" + send "\r" + expect "login:" + send {$vm_user} + send "\r" + expect "Password:" + send {$vm_password} + send "\r" + expect "# " + send "lscpu | grep 'On-line CPU' --color=never \r" + expect "# " + send "exit\r" + expect eof +EOF + fi + + cp $hot_swap_mem_xml_path $log_path/virtual_hot_swap_mem_xml.log + expect << EOF > $log_path/virtual_hot_swap_mem.log spawn virsh console $vm_name expect "Escape character is \\^]" send "\r" expect "login:" - send "$vm_user\r" + send {$vm_user} + send "\r" expect "Password:" - send "$vm_password\r" + send {$vm_password} + send "\r" expect "# " - send "dmesg | grep PV --color=never \r" + send "numactl -H \r" expect "# " send "exit\r" expect eof @@ -147,28 +188,35 @@ EOF } -collect_virtual_vcpu(){ - vm_name=$1 - vm_user=$2 - vm_password=$3 - vcpu_count=$4 - echo $vcpu_count > $log_path/virtual_vcpu.log - if virsh setvcpus $vm_name --count $vcpu_count --live; then - expect << EOF >> $log_path/virtual_vcpu.log - spawn virsh console $vm_name - expect "Escape character is \\^]" - send "\r" - expect "login:" - send "$vm_user\r" - expect "Password:" - send "$vm_password\r" - expect "# " - send "lscpu | grep 'On-line CPU' --color=never \r" - expect "# " - send "exit\r" - expect eof +collect_virtual_live_migrate(){ + kae_live_migrate_vm_name=$1 + virsh dumpxml $kae_live_migrate_vm_name > $log_path/virtual_live_migrate_xml.log + virsh start $kae_live_migrate_vm_name + nohup timeout 20 watch -gt -n 0.2 cat /sys/class/uacce/hisi_zip-*/available_instances > $log_path/virtual_live_migrate.log & +} + + +collect_virtual_hardware_acc(){ + vm_name=$1 + vm_user=$2 + vm_password=$3 + dmesg | grep GIC > $log_path/virtual_hardware_acc.log + expect << EOF >> $log_path/virtual_hardware_acc.log + spawn virsh console $vm_name + expect "Escape character is \\^]" + send "\r" + expect "login:" + send {$vm_user} + send "\r" + expect "Password:" + send {$vm_password} + send "\r" + expect "# " + send "dmesg | grep SGI --color=never \r" + expect "# " + send "exit\r" + expect eof EOF - fi } @@ -177,10 +225,31 @@ collect_k8s_numa(){ } +collect_virtual_sr_iov(){ + pod=$1 + kubectl -n kube-system get pod | grep kube-sriov-device-plugin- > $log_path/virtual_sr_iov.log + kubectl exec -it $pod ls /dev >> $log_path/virtual_sr_iov.log +} + + collect_k8s_mpam(){ kubectl get pod | grep mpam-controller-daemon-agent- > $log_path/virtual_k8s_mpam.log } + +collect_virtual_host(){ + ovs_appctl_res=$(ovs-appctl --version 2>&1) + + if [[ $ovs_appctl_res =~ "command not found" ]]; + then + echo "ovs-appctl: command not found" > $log_path/virtual_dpu_flow.log + else + echo "ovs-appctl version: $ovs_appctl_res" > $log_path/virtual_dpu_flow.log + script -a -c 'ovs-appctl hwoff/dump-hwoff-flows' $log_path/virtual_dpu_flow.log + fi +} + + ################数据库特性信息收集################## collect_database(){ mysql_install_path=$1 @@ -274,9 +343,11 @@ collect_virtcca_msg(){ expect "Escape character is \\^]" send "\r" expect "login:" - send "$username\r" + send {$username} + send "\r" expect "Password:" - send "$passwd\r" + send {$passwd} + send "\r" expect "# " send "ls -l /\r" expect "# " @@ -305,6 +376,8 @@ collect_acceleration_library(){ openssl speed -elapsed -engine kae rsa2048 > $log_path/acceleration_library.log 2>&1 lspci | grep ZIP >> $log_path/acceleration_library.log ls -l /usr/local/kaezip/lib/ >> $log_path/acceleration_library.log + ls -l /usr/local/kaezstd/lib/ >> $log_path/acceleration_library.log + ls -l /usr/local/kaelz4/lib/ >> $log_path/acceleration_library.log ldd $1 >> $log_path/acceleration_library.log ldd $2 >> $log_path/acceleration_library.log ldd $3 >> $log_path/acceleration_library.log @@ -367,11 +440,18 @@ collect_storage_acc(){ systemctl status ceph.target > $log_path/storage_comporess.log \cp "$ceph_conf".bak "$ceph_conf" fi + + ceph daemon osd.0 config show | grep compaction_enabled > $log_path/storage_compaction.log + ceph daemon osd.0 config show | grep compaction_opt_space_enabled >> $log_path/storage_compaction.log + ldd $storage_maintain_bin > $log_path/storage_maintain_tool.log lib_rocksdb=$(ldd $rocksdb_bin | grep librocksdb | awk '{print $3}') ldd $lib_rocksdb > $log_path/storage_rocksdb.log ldd $ucache_bin > $log_path/storage_ucache.log + ceph -s > $log_path/storage_spdk_io.log + cephadm shell sh /var/lib/ceph/spdk_lib/scripts/setup.sh status | grep NVMe >> $log_path/storage_spdk_io.log + if ldd $non_ceph_bin | grep ksal; then timeout 20 perf top -p $non_ceph_pid > $log_path/storage_non_ceph_perf_top.log fi @@ -702,9 +782,7 @@ collect_bigdata_tune_up(){ collect_bigdata_omnimv(){ omnimv_dir=$1 - omnimv_dir_clickhouse=$2 hdfs dfs -ls $omnimv_dir > $log_path/bigdata_omnimv.log - ls $omnimv_dir_clickhouse/BoostKit-omnimv_1.1.0/clickhouse/schema.sql >> $log_path/bigdata_omnimv.log } @@ -811,6 +889,11 @@ collect_bigdata_omni_shield(){ } +collect_bigdata_omni_yarn() { + grep "LoadBasedCapacityScheduler" /usr/local/hadoop/logs/hadoop-root-resourcemanager-*.log > $log_path/bigdata_omni_yarn.log 2>&1 +} + + collect_bigdata_doris() { doris_install_dir=$1 grep neonmovemask_bulk $doris_install_dir/thirdparty/src/bitshuffle-0.5.1/src/bitshuffle_core.c > $log_path/bigdata_doris.log 2>&1 @@ -841,49 +924,6 @@ collect_hpc_domain_acceleration_library(){ ldd $domain_acc_lib_bin > $log_path/hpc_domain_acceleration_library.log 2>&1 } -################毕昇JDK信息收集####################### -collect_bisheng_jdk_heap(){ - check_sysbol=`objdump -tT /usr/lib64/libjvm*_kunpeng.so | grep HeapDict_Add` - if [[ $check_sysbol = '' ]];then - echo "failed" > $log_path/bisheng_jdk_heap.log - return - fi - - jmap_cmd=$1"/bin/jmap" - pid=$2 - dump_file="/tmp/"${pid}"_dump.hprof" - $jmap_cmd -dump:format=b,file=$dump_file,HeapDumpRedact=names,RedactMap="value:#**#" $pid > $log_path/bisheng_jdk_heap.log - - rm -rf $dump_file -} - - -collect_bisheng_jdk_jbooster(){ - readelf -Ws /usr/lib64/libjvm17_kunpeng.so | grep JBooster > $log_path/bisheng_jdk_jbooster.log -} - - -collect_bisheng_jdk_jbolt(){ - readelf -Ws /usr/lib64/libjvm*_kunpeng.so | grep JBolt > $log_path/bisheng_jdk_jbolt.log -} - - -collect_bisheng_jdk_kae_provider(){ - kae_log_path=$1 - cp $kae_log_path $log_path/bisheng_jdk_kae_provider.log -} - - -collect_bisheng_jdk_kae_zlib(){ - kae_zlib_pid=$1 - pmap -p $kae_zlib_pid > $log_path/bisheng_jdk_kae_zlib.log -} - - -collect_bisheng_jdk_g1gc(){ - ps -ef| grep java > $log_path/bisheng_jdk_g1gc.log -} - ################环境信息收集####################### collect_os_msg(){ echo os: $(cat /etc/os-release |grep PRETTY_NAME=|awk -F= '{print $2}') > $log_path/os_msg.log @@ -996,7 +1036,6 @@ main(){ mysql_password=$(acquire_value Bigdata mysql_password) mysql_database_name=$(acquire_value Bigdata mysql_database_name) omnimv_dir=$(acquire_value Bigdata omnimv_dir) - omnimv_dir_clickhouse=$(acquire_value Bigdata omnimv_dir_clickhouse) omnidata_launcher=$(acquire_value Bigdata omnidata_launcher) omnidata_launcher_server=$(acquire_value Bigdata omnidata_launcher_server) omnidata_install_path=$(acquire_value Bigdata omnidata_install_path) @@ -1013,32 +1052,44 @@ main(){ collect_bigdata_operator_hive $hive_bin_path $hive_local_mode $omnioperator_dir collect_bigdata_hbase collect_bigdata_tune_up $omniadvisor_dir $mysql_username $mysql_password $mysql_database_name - collect_bigdata_omnimv "$omnimv_dir" "$omnimv_dir_clickhouse" + collect_bigdata_omnimv "$omnimv_dir" collect_bigdata_omni_push_down "$omnidata_launcher_server" "$omnidata_launcher" "$push_down_jars" "$push_down_conf" "$spark_path" "$database" "$omnidata_install_path" "$zookeeper_address" "$zookeeper_path" collect_bigdata_omni_shuffle "$spark_path" "$shuffle_jars" "$database" "$shuffle_ock_path" collect_bigdata_omni_shield $spark_path $omnishield_jar + collect_bigdata_omni_yarn collect_bigdata_doris $doris_install_dir echo "Bigdata collect msg Done..." elif [ $per_project = "Virtual" ]; then echo "start collect Virtual msg..." - collect_virtual_host - topo_vm_name=$(acquire_value Virtual topo_vm_name) - topo_vm_user=$(acquire_value Virtual topo_vm_user) - topo_vm_password=$(acquire_value Virtual topo_vm_password) - topo_numa_count=$(acquire_value Virtual topo_numa_count) - collect_virtual_topo "$topo_vm_name" "$topo_vm_user" "$topo_vm_password" "$topo_numa_count" - real_vm_name=$(acquire_value Virtual real_vm_name) - real_vm_user=$(acquire_value Virtual real_vm_user) - real_vm_password=$(acquire_value Virtual real_vm_password) - collect_virtual_real "$real_vm_name" "$real_vm_user" "$real_vm_password" - vcpu_vm_name=$(acquire_value Virtual vcpu_vm_name) - vcpu_vm_user=$(acquire_value Virtual vcpu_vm_user) - vcpu_vm_password=$(acquire_value Virtual vcpu_vm_password) - vcpu_count=$(acquire_value Virtual vcpu_count) - collect_virtual_vcpu "$vcpu_vm_name" "$vcpu_vm_user" "$vcpu_vm_password" "$vcpu_count" + collect_virtual_ovs_xpf + vm_ip=$(acquire_value Virtual vm_ip) + vm_user=$(acquire_value Virtual vm_user) + vm_password=$(acquire_value Virtual vm_password) + vm_ceph_disk_name=$(acquire_value Virtual vm_ceph_disk_name) + vm_name=$(acquire_value Virtual vm_name) + collect_virtual_ceph "$vm_ip" "$vm_user" "$vm_password" "$vm_ceph_disk_name" "$vm_name" + schedule_vm_name=$(acquire_value Virtual schedule_vm_name) + schedule_vm_user=$(acquire_value Virtual schedule_vm_user) + schedule_vm_password=$(acquire_value Virtual schedule_vm_password) + collect_virtual_schedule "$schedule_vm_name" "$schedule_vm_user" "$schedule_vm_password" + hot_swap_vm_name=$(acquire_value Virtual hot_swap_vm_name) + hot_swap_vm_user=$(acquire_value Virtual hot_swap_vm_user) + hot_swap_vm_password=$(acquire_value Virtual hot_swap_vm_password) + hot_swap_cpu_count=$(acquire_value Virtual hot_swap_cpu_count) + hot_swap_mem_xml_path=$(acquire_value Virtual hot_swap_mem_xml_path) + collect_virtual_hot_swap "$hot_swap_vm_name" "$hot_swap_vm_user" "$hot_swap_vm_password" "$hot_swap_cpu_count" "$hot_swap_mem_xml_path" + kae_live_migrate_vm_name=$(acquire_value Virtual kae_live_migrate_vm_name) + collect_virtual_live_migrate "$kae_live_migrate_vm_name" + hardware_acc_vm_name=$(acquire_value Virtual hardware_acc_vm_name) + hardware_acc_vm_user=$(acquire_value Virtual hardware_acc_vm_user) + hardware_acc_vm_password=$(acquire_value Virtual hardware_acc_vm_password) + collect_virtual_hardware_acc "$hardware_acc_vm_name" "$hardware_acc_vm_user" "$hardware_acc_vm_password" collect_k8s_numa + sr_iov_pod=$(acquire_value Virtual sr_iov_pod) + collect_virtual_sr_iov "$sr_iov_pod" collect_k8s_mpam + collect_virtual_host echo "Virtual collect msg Done..." elif [ $per_project = "HPC" ]; then @@ -1048,18 +1099,6 @@ main(){ domain_acc_lib_bin=$(acquire_value HPC domain_acc_lib_bin) collect_hpc_domain_acceleration_library "$domain_acc_lib_bin" echo "HPC collect msg Done..." - elif [ $per_project = "BishengJDK" ]; - then - bisheng_jdk_dir=$(acquire_value BishengJDK bisheng_jdk_dir) - bisheng_jdk_heap_pid=$(acquire_value BishengJDK bisheng_jdk_heap_pid) - collect_bisheng_jdk_heap $bisheng_jdk_dir $bisheng_jdk_heap_pid - collect_bisheng_jdk_jbooster - collect_bisheng_jdk_jbolt - kae_log_path=$(acquire_value BishengJDK kae_log_path) - collect_bisheng_jdk_kae_provider $kae_log_path - kae_zlib_pid=$(acquire_value BishengJDK kae_zlib_pid) - collect_bisheng_jdk_kae_zlib $kae_zlib_pid - collect_bisheng_jdk_g1gc fi done } -- Gitee From c2bc0b430d3d481bc01c1dd4faba7e5054bacad2 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 25 Apr 2025 08:13:07 +0000 Subject: [PATCH 34/35] update tools/config.ini. Signed-off-by: ksun --- tools/config.ini | 56 +++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/tools/config.ini b/tools/config.ini index fe375c1..73939a3 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -39,23 +39,35 @@ greenplum_kae_sql= [Virtual] check=False -# 拓扑感知 -topo_vm_name=vm_name -topo_vm_user=root -topo_vm_password=password -topo_numa_count=4 - -# 虚实协同 -real_vm_name=vm_name -real_vm_user=root -real_vm_password=password - -# 虚实协同 -vcpu_vm_name=vm_name -vcpu_vm_user=root -vcpu_vm_password=password -vcpu_count=4 - +# 高性能云盘 +vm_ip=71.14.48.104 +vm_user=root +vm_password=Huawei12#$ +vm_ceph_disk_name=vdb +vm_name=vm1 + +# 虚拟化调度优化 +schedule_vm_name=vm1 +schedule_vm_user=root +schedule_vm_password=password + +# 热插拔 +hot_swap_vm_name=vm_name +hot_swap_vm_user=root +hot_swap_vm_password=password +hot_swap_cpu_count=4 +hot_swap_mem_xml_path=/tmp/mem.xml + +# KAE加速热迁移 +kae_live_migrate_vm_name=vm_name + +# 虚拟化硬件辅助加速 +hardware_acc_vm_name=vm_name +hardware_acc_vm_user=root +hardware_acc_vm_password=password + +# K8s SR-IOV直通插件 +sr_iov_pod=test_pod [Acclib] check=False @@ -101,10 +113,7 @@ mysql_password=123456 mysql_database_name=xxxxx # 物化视图 -# 物化视图 - spark 场景 omnimv_dir=/omnimv -# 物化视图 - clickhouse 场景 -omnimv_dir_clickhouse=/opt/OminiMV # 算子下推 omnidata_launcher_server= @@ -148,10 +157,3 @@ boostio_bin=/tmp/bio_test check=False common_acc_lib_bin=/tmp/binary domain_acc_lib_bin=/tmp/binary - -[BishengJDK] -check=False -bisheng_jdk_dir=/home/bisheng_jdk -bisheng_jdk_heap_pid=6321 -kae_log_path=/tmp/kae.log -kae_zlib_pid=313125 -- Gitee From 7a10d4513b048b10457aeb5c75b96c8722c2b6b4 Mon Sep 17 00:00:00 2001 From: ksun Date: Fri, 25 Apr 2025 08:14:13 +0000 Subject: [PATCH 35/35] update tools/Boostkit_ISV_Tool.md. Signed-off-by: ksun --- tools/Boostkit_ISV_Tool.md | 117 ++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 471b192..99cacf0 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -117,10 +117,6 @@ hive可执行路径 omnimv 目录路径, 用于spark场景下的物化视图特性验证 -> omnimv_dir_clickhouse=/opt/OminiMV - -配置 OmniMV 工作文件的存储路径, 用于clickhouse场景下的物化视图特性验证 - > omnidata_launcher_server= launcher 所在服务器的名称 (需可直接ssh无密码访问, 不填则默认在当前服务器上) @@ -357,49 +353,86 @@ check 默认为False, 置为True 则开启加速库特性使能信息收集功 2.K8S NUMA自适应插件, K8S MPAM插件 验收时配置文件无需修改 3.virsh console 命令同时只能有一个用户在线上，采集前需让已登录的用户退出 -> topo_vm_name=vm_name - -虚拟机名称，用于“拓扑感知”特性验收 +> vm_ip=71.14.48.104 + +虚拟机ip, 用于验收高性能云盘优化特性 + +> vm_user=root + +虚拟机用户名, 用于验收高性能云盘优化特性 + +> vm_password=Huawei12#$ + +虚拟机密码, 用于验收高性能云盘优化特性 + +> vm_ceph_disk_name=vdb + +虚拟机使用的高性能云盘名称, 用于验收高性能云盘优化特性 + +> vm_name=vm1 + +虚拟机名称, 用于验收高性能云盘优化特性 + +> schedule_vm_name=vm1 + +虚拟机名称，用于验收虚拟化调度优化特性 + +> schedule_vm_user=root + +虚拟机用户名，用于验收虚拟化调度优化特性 + +> schedule_vm_password=password -> topo_vm_user=root +虚拟机密码，用于验收虚拟化调度优化特性 -虚拟机用户名，用于“拓扑感知”特性验收 +> hot_swap_vm_name=vm1 -> topo_vm_password=password +虚拟机名称，用于验收热插拔特性 -虚拟机用户密码，用于“拓扑感知”特性验收 +> hot_swap_vm_user=root -> topo_numa_count=4 +虚拟机用户名，用于验收热插拔特性 -虚拟机xml中配置的 NUMA 个数，用于“拓扑感知”特性验收 +> hot_swap_vm_password=password -> real_vm_name=vm_name +虚拟机密码，用于验收热插拔特性 -虚拟机名称，用于“虚实协同”特性验收 +> hot_swap_cpu_count=4 -> real_vm_user=root +热插拔cpu测试时，设置的 cpu 数量，用于验收热插拔特性 -虚拟机用户名，用于“虚实协同”特性验收 +> hot_swap_mem_xml_path=/tmp/mem.xml -> real_vm_password=password +用于热插拔内存测试的xml，用于验收热插拔特性。 +例: 将 NUMA node 3 的 内存设置为 1024M，对应的 xml 内容示例 +```xml + + + 1024 + 3 + + +``` + +> kae_live_migrate_vm_name=vm_name -虚拟机用户密码，用于“虚实协同”特性验收 +虚拟机名称，用于KAE加速热迁移特性验收 -> vcpu_vm_name=vm_name +> hardware_acc_vm_name=vm1 -虚拟机名称，用于“vCPU热插拔”特性验收 +虚拟机名称，用于验收虚拟化硬件辅助加速特性 -> vcpu_vm_user=root +> hardware_acc_vm_user=root -虚拟机用户名，用于“vCPU热插拔”特性验收 +虚拟机用户名，用于验收虚拟化硬件辅助加速特性 -> vcpu_vm_password=password +> hardware_acc_vm_password=password -虚拟机用户密码，用于“vCPU热插拔”特性验收 +虚拟机密码，用于验收虚拟化硬件辅助加速特性 -> vcpu_count=4 +> sr_iov_pod=test_pod -虚拟机cpu热插拔后cpu数量，用于“vCPU热插拔”特性验收 +pod 名称，用于验收K8s SR-IOV直通插件特性 #### **1.2.7 机密计算配置文件说明** @@ -445,33 +478,9 @@ check 默认为False, 置为True 则开启HPC相关特性使能信息收集功 -#### **1.2.9 毕昇JDK配置文件说明** - -> check=False - -check 默认为False, 置为True 则开启毕昇JDK相关特性使能信息收集功能 - -> bisheng_jdk_dir=/home/bisheng_jdk - -毕昇jdk安装目录，用于"毕昇JDK加速库(堆转储增强)"特性验收 - -> bisheng_jdk_heap_pid=6321 - -使用了堆转储增强的进程id，用于"毕昇JDK加速库(堆转储增强)"特性验收 - -> kae_log_path=/tmp/kae.log - -kae日志路径，用于"KAEProvider"特性验收 - -> kae_zlib_pid=313125 - -使能了KAE zlib的进程id，用于"KAE zlib"特性验收 - ### 1.3 采集工具使用说明 ``` -bash collect_msg.sh -# 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 -``` - - +bash collect_msg.sh +# 同级目录下具有config.ini 配置文件，且已按1.2中对应解决方案完成相关参数配置 +``` \ No newline at end of file -- Gitee