diff --git a/tools/BoostKit_pic/bigdata.png b/tools/BoostKit_pic/bigdata.png index 7997da0cd314860d4bbcd14a312220e924ace0ad..56b6f53de8425b7dc0cf8c90ed0ba26acc2c5a8e 100644 Binary files a/tools/BoostKit_pic/bigdata.png and b/tools/BoostKit_pic/bigdata.png differ diff --git a/tools/Boostkit_ISV_Tool.md b/tools/Boostkit_ISV_Tool.md index 534dd47db34f0ce693958c7cb3271359efdd7bee..fac4c7f6591a9f12bd5408e44c5bcb2649abda1b 100644 --- a/tools/Boostkit_ISV_Tool.md +++ b/tools/Boostkit_ISV_Tool.md @@ -70,6 +70,12 @@ spark的安装路径，默认为/home/ +> omnioperator_dir=/opt/omni-operator + +算子加速的安装路径，默认为/opt/omni-operator + + + > omniadvisor_dir=/home 参数调优的安装路径，默认为/home @@ -256,6 +262,26 @@ check 默认为False, 置为True 则开启HPC相关特性使能信息收集功 引用HPC SME改造二进制文件绝对路径 + + +### 1.3 采集工具使用说明 + +#### 1.3.1 二进制使用 + +``` +./collect_msg +# 同级目录下具有config.ini 配置文件，且已按1.2中对应解决方案完成相关参数配置 +``` + +#### 1.3.2 shell 脚本使用 + +``` +bash collect_msg.sh +# 同级目录下具有config.ini 配置文件， 且已按1.2中对应解决方案完成相关参数配置 +``` + + + ## 1. **分析工具** ### 1.1 **分析工具能力说明** diff --git a/tools/collect_msg.sh b/tools/collect_msg.sh index 162b2ce16c7822594674cb2eb3071d03df74ca4d..2b334bd90c14a23403840e8aa2b7ddb56fb3e465 100644 --- a/tools/collect_msg.sh +++ b/tools/collect_msg.sh @@ -4,45 +4,7 @@ config_file=$current_path/config.ini log_path=$current_path/log default_project="Bigdata Database Storage Arm Virt Acclib Virtual HPC" -spark_omni_func=( - --deploy-mode client - --driver-cores 1 - --driver-memory 980M - --num-executors 3 - --executor-cores 1 - --executor-memory 600M - --master yarn - --conf spark.memory.offHeap.enabled=true - --conf spark.memory.offHeap.size=1025M - --conf spark.task.cpus=1 - --conf spark.driver.extraClassPath=/opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar:/opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar:/opt/omni-operator/lib/dependencies/* - --conf spark.executor.extraClassPath=/opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar:/opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar:/opt/omni-operator/lib/dependencies/* - --driver-java-options -Djava.library.path=/opt/omni-operator/lib - --conf spark.sql.codegen.wholeStage=false - --conf spark.executorEnv.LD_LIBRARY_PATH=/opt/omni-operator/lib - --conf spark.executorEnv.OMNI_HOME=/opt/omni-operator/ - --conf spark.driverEnv.LD_LIBRARY_PATH=/opt/omni-operator/lib - --conf spark.driverEnv.OMNI_HOME=/opt/omni-operator/ - --conf spark.executor.extraLibraryPath=/opt/omni-operator/lib - --conf spark.driverEnv.LD_PRELOAD=/opt/omni-operator/lib/libjemalloc.so.2 - --conf spark.executorEnv.LD_PRELOAD=/opt/omni-operator/lib/libjemalloc.so.2 - --conf spark.sql.extensions=com.huawei.boostkit.spark.ColumnarPlugin - --jars /opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar - --jars /opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar - --conf spark.sql.orc.impl=native - --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.OmniColumnarShuffleManager - --conf spark.omni.sql.columnar.fusion=false - --conf spark.omni.sql.columnar.sortSpill.enabled=true - --conf spark.omni.sql.columnar.sortSpill.rowThreshold=4000000 - --conf spark.omni.sql.columnar.sortSpill.dirDiskReserveSize=214748364800 - --conf spark.locality.wait=8 - --conf spark.sql.autoBroadcastJoinThreshold=10M - --conf spark.sql.broadcastTimeout=500 - --conf spark.sql.cbo.enabled=false - --conf spark.default.parallelism=200 - --conf spark.sql.shuffle.partitions=200 - --conf spark.executorEnv.MALLCO_CONF=narenas:2 -) + @@ -50,14 +12,14 @@ spark_omni_func=( acquire_value(){ project=$1 key=$2 - grep $project -A 15 $config_file |grep -m 1 $key|awk -F= '{print $2}'|awk '{print $1}' + grep $project -A 100 $config_file |grep -m 1 $key|awk -F= '{print $2}' } ######################获取配置文件中选择校验的解决方案########### acquire_select_project(){ all_name="" - for per_project in $default_project + for per_poject in $default_project do status=$(acquire_value $per_project check) if [[ $status = True ]]; then @@ -190,16 +152,30 @@ collect_database(){ mysql_password=$3 database_name=$4 plugin_path=$1/lib/plugin + timeout=60 $mysql_install_path/bin/mysqld_safe --defaults-file=/etc/my.cnf & - sleep 20 - mysql -u $mysql_username -p$mysql_password -D $database_name -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'thread_pool%'" > $log_path/database_mysql.log - mysql -u $mysql_username -p$mysql_password -D $database_name -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'kovae%'" >> $log_path/database_mysql.log - echo thread_pool: $(ls $plugin_path |grep thread_pool.so) >> $log_path/database_mysql.log - echo kovae_path: $(ls $plugin_path |grep ha_kovae.so) >> $log_path/database_mysql.log - readelf -a $mysql_install_path/bin/mysqld|grep bolt >> $log_path/database_mysql.log - echo no_lock: $(objdump -d $mysql_install_path/bin/mysqld|grep -c row_vers_build_for_semi_consistent_readP5trx_t) >> $log_path/database_mysql.log - objdump -d $mysql_install_path/bin/mysqld |grep crc32cb >> $log_path/database_mysql.log - pkill -9 mysql + # 循环判断进程是否启动 + for ((i=0; i<=$timeout; i++)); do + if ps -ef | grep -v grep | grep "mysql" > /dev/null; then + echo "Process mysql started." + # 执行下一步操作 + sleep 5 + $mysql_install_path/bin/mysql -u $mysql_username -p$mysql_password -D $database_name -h127.0.0.1 -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'thread_pool%'" > $log_path/database_mysql.log + $mysql_install_path/bin/mysql -u $mysql_username -p$mysql_password -D $database_name -h127.0.0.1 -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'kovae%'" >> $log_path/database_mysql.log + echo thread_pool: $(ls $plugin_path |grep thread_pool.so) >> $log_path/database_mysql.log + echo kovae_path: $(ls $plugin_path |grep ha_kovae.so) >> $log_path/database_mysql.log + readelf -a $mysql_install_path/bin/mysqld|grep bolt >> $log_path/database_mysql.log + echo no_lock: $(objdump -d $mysql_install_path/bin/mysqld|grep -c row_vers_build_for_semi_consistent_readP5trx_t) >> $log_path/database_mysql.log + objdump -d $mysql_install_path/bin/mysqld |grep crc32cb >> $log_path/database_mysql.log + pkill -9 mysql + break + fi + sleep 1 + if [ $timeout -eq $i ];then + echo "Timeout error: mysql process not started in $timeout seconds.i" + exit 1 + fi + done } @@ -458,16 +434,72 @@ collect_bigdata_operator(){ # 日志记录位置 log_path/bigdata_operator.log spark_path=$1 database=$2 - if [ -e $spark_path ]; + omnioperator_dir=$3 + if [ -e $spark_path ] && [ -e $omnioperator_dir ]; then + spark_version=`awk '{print $2}' $spark_path/RELEASE | head -n 1` + omnioperator_version=`cat $omnioperator_dir/version.txt | grep 'Component Version' | awk -F ": " '{print $2}'` + echo "spark version:$spark_version omnioperator version:$omnioperator_version" >$log_path/bigdata_operator.log 2>&1 + spark_omni_func=( + --deploy-mode client + --driver-cores 5 + --driver-memory 5g + --num-executors 6 + --executor-cores 2 + --executor-memory 6g + --master yarn + --conf spark.memory.offHeap.enabled=true + --conf spark.memory.offHeap.size=18g + --conf spark.task.cpus=1 + --conf spark.driver.extraClassPath=${omnioperator_dir}/lib/boostkit-omniop-spark-${spark_version}-${omnioperator_version}-aarch64.jar:${omnioperator_dir}/lib/boostkit-omniop-bindings-${omnioperator_version}-aarch64.jar:${omnioperator_dir}/lib/dependencies/* + --conf spark.executor.extraClassPath=${omnioperator_dir}/lib/boostkit-omniop-spark-${spark_version}-${omnioperator_version}-aarch64.jar:${omnioperator_dir}/lib/boostkit-omniop-bindings-${omnioperator_version}-aarch64.jar:${omnioperator_dir}/lib/dependencies/* + --driver-java-options -Djava.library.path=${omnioperator_dir}/lib + --conf spark.sql.codegen.wholeStage=false + --conf spark.executorEnv.LD_LIBRARY_PATH=${omnioperator_dir}/lib + --conf spark.executorEnv.OMNI_HOME=${omnioperator_dir}/ + --conf spark.driverEnv.LD_LIBRARY_PATH=${omnioperator_dir}/lib + --conf spark.driverEnv.OMNI_HOME=${omnioperator_dir}/ + --conf spark.executor.extraLibraryPath=${omnioperator_dir}/lib + --conf spark.driverEnv.LD_PRELOAD=${omnioperator_dir}/lib/libjemalloc.so.2 + --conf spark.executorEnv.LD_PRELOAD=${omnioperator_dir}/lib/libjemalloc.so.2 + --conf spark.sql.extensions=com.huawei.boostkit.spark.ColumnarPlugin + --jars ${omnioperator_dir}/lib/boostkit-omniop-spark-${spark_version}-${omnioperator_version}-aarch64.jar + --jars ${omnioperator_dir}/lib/boostkit-omniop-bindings-${omnioperator_version}-aarch64.jar + --conf spark.sql.orc.impl=native + --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.OmniColumnarShuffleManager + --conf spark.omni.sql.columnar.fusion=false + --conf spark.omni.sql.columnar.sortSpill.enabled=true + --conf spark.omni.sql.columnar.sortSpill.rowThreshold=4000000 + --conf spark.omni.sql.columnar.sortSpill.dirDiskReserveSize=214748364800 + --conf spark.locality.wait=8 + --conf spark.sql.autoBroadcastJoinThreshold=10M + --conf spark.sql.broadcastTimeout=500 + --conf spark.sql.cbo.enabled=false + --conf spark.default.parallelism=200 + --conf spark.sql.shuffle.partitions=200 + --conf spark.executorEnv.MALLCO_CONF=narenas:2 +) + spark_conf_path=$1/conf - if ! cat < $spark_conf_path/log4j.properties|grep "^log4j.logger.com.huawei.boostkit.spark=INFO"; - then - echo "log4j.logger.com.huawei.boostkit.spark=INFO" >> $spark_conf_path/log4j.properties - fi - $spark_path/bin/spark-sql "${spark_omni_func[@]}" --database $database -e "WITH customer_total_return AS ( SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, sum(sr_return_amt) AS ctr_total_return FROM store_returns, date_dim WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 GROUP BY sr_customer_sk, sr_store_sk) SELECT c_customer_id FROM customer_total_return ctr1, store, customer WHERE ctr1.ctr_total_return > (SELECT avg(ctr_total_return) * 1.2 FROM customer_total_return ctr2 WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) AND s_store_sk = ctr1.ctr_store_sk AND s_state = 'TN' AND ctr1.ctr_customer_sk = c_customer_sk ORDER BY c_customer_id LIMIT 100;" 1>$log_path/bigdata_operator.log 2>&1 + if [ X"${spark_version}" == "X3.1.1" ];then + if [ ! -f $spark_conf_path/log4j.properties ];then + cp $spark_conf_path/log4j.properties.template $spark_conf_path/log4j.properties + fi + if ! cat < $spark_conf_path/log4j.properties|grep "^log4j.logger.com.huawei.boostkit.spark=INFO"; + then + echo "log4j.logger.com.huawei.boostkit.spark=INFO" >> $spark_conf_path/log4j.properties + fi + elif [ X"${spark_version}" == "X3.3.1" ];then + if [ ! -f $spark_conf_path/log4j2.properties ];then + cp $spark_conf_path/log4j2.properties.template $spark_conf_path/log4j2.properties + fi + sed -i "s/^logger.thriftserver.level =.*/logger.thriftserver.level = INFO/" $spark_conf_path/log4j2.properties + else + echo "Spark ${spark_version} is not supported." >>$log_path/bigdata_operator.log 2>&1 + fi + $spark_path/bin/spark-sql "${spark_omni_func[@]}" --database $database -e "WITH customer_total_return AS ( SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, sum(sr_return_amt) AS ctr_total_return FROM store_returns, date_dim WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 GROUP BY sr_customer_sk, sr_store_sk) SELECT c_customer_id FROM customer_total_return ctr1, store, customer WHERE ctr1.ctr_total_return > (SELECT avg(ctr_total_return) * 1.2 FROM customer_total_return ctr2 WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) AND s_store_sk = ctr1.ctr_store_sk AND s_state = 'TN' AND ctr1.ctr_customer_sk = c_customer_sk ORDER BY c_customer_id LIMIT 100;" 1>>$log_path/bigdata_operator.log 2>&1 else - echo "$spark_path not exist" >$log_path/bigdata_operator.log 2>&1 + echo "$spark_path or $omnioperator_dir does not exist" >$log_path/bigdata_operator.log 2>&1 fi } @@ -595,12 +627,13 @@ main(){ dataset_list=$(acquire_value Bigdata dataset_list) spark_path=$(acquire_value Bigdata spark_path) database=$(acquire_value Bigdata database) + omnioperator_dir=$(acquire_value Bigdata omnioperator_dir) omniadvisor_dir=$(acquire_value Bigdata omniadvisor_dir) mysql_username=$(acquire_value Bigdata mysql_username) mysql_password=$(acquire_value Bigdata mysql_password) mysql_database_name=$(acquire_value Bigdata mysql_database_name) collect_bigdata_kal "${algorithms_list[@]}" $algorithms_path "${dataset_list[@]}" - collect_bigdata_operator $spark_path $database + collect_bigdata_operator $spark_path $database $omnioperator_dir collect_bigdata_hbase collect_bigdata_tune_up $omniadvisor_dir $mysql_username $mysql_password $mysql_database_name echo "Bigdata collect msg Done..." @@ -645,3 +678,7 @@ tar_log_file $customer_information + + + + diff --git a/tools/config.ini b/tools/config.ini index 5f0e9d5a4188bb69cbee20d13e8ca05ba6be7db9..1b6c48defa214a460c7d695d967095ee3e001fb3 100644 --- a/tools/config.ini +++ b/tools/config.ini @@ -55,8 +55,9 @@ algorithms_name=algorithms_name1 algorithms_name2 algorithms_name3 dataset_list=dataset1 dataset2 dataset3 # 算子加速 -spark_path=/home/ -database=tpcds_bin_partitioned_orc_1000 +spark_path=/usr/local/spark +database=tpcds_bin_partitioned_varchar_orc_2 +omnioperator_dir=/opt/omni-operator # 参数调优 omniadvisor_dir=/home @@ -77,3 +78,4 @@ acc_lib=/home/lib sme=/home/lib +