From acf9d5a4c2882e60b5b1680f489c532de7eb358b Mon Sep 17 00:00:00 2001
From: liangruifeng <960641173@qq.com>
Date: Mon, 20 Jan 2025 22:20:37 +0800
Subject: [PATCH 1/5] omniop deploy tool

---
 .../command/command_line_tmp                  |  23 ++
 omnioperator/omniop-deploy-tool/conf/config   |   9 +
 .../omniop-deploy-tool/conf/omni.conf         |  28 ++
 .../omniop-deploy-tool/conf/omnioperator.conf |  29 ++
 omnioperator/omniop-deploy-tool/deploy.sh     | 265 ++++++++++++++++++
 omnioperator/omniop-deploy-tool/readme.md     |  30 ++
 6 files changed, 384 insertions(+)
 create mode 100644 omnioperator/omniop-deploy-tool/command/command_line_tmp
 create mode 100644 omnioperator/omniop-deploy-tool/conf/config
 create mode 100644 omnioperator/omniop-deploy-tool/conf/omni.conf
 create mode 100644 omnioperator/omniop-deploy-tool/conf/omnioperator.conf
 create mode 100644 omnioperator/omniop-deploy-tool/deploy.sh
 create mode 100644 omnioperator/omniop-deploy-tool/readme.md

diff --git a/omnioperator/omniop-deploy-tool/command/command_line_tmp b/omnioperator/omniop-deploy-tool/command/command_line_tmp
new file mode 100644
index 000000000..059c4521e
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/command/command_line_tmp
@@ -0,0 +1,23 @@
+/usr/local/spark/bin/spark-sql --archives hdfs://{hostname}:9000/user/{user}/{omni_tar_name}.tar.gz#omni \
+--deploy-mode client \
+--driver-cores 8 \
+--driver-memory 40g \
+--master yarn \
+--executor-cores 12 \
+--executor-memory 5g \
+--conf spark.memory.offHeap.enabled=true \
+--conf spark.memory.offHeap.size=35g \
+--num-executors 24 \
+--conf spark.executor.extraJavaOptions='-XX:+UseG1GC' \
+--conf spark.locality.wait=0 \
+--conf spark.network.timeout=600 \
+--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+--conf spark.sql.adaptive.enabled=true \
+--conf spark.sql.adaptive.skewedJoin.enabled=true \
+--conf spark.sql.autoBroadcastJoinThreshold=100M \
+--conf spark.sql.broadcastTimeout=600 \
+--conf spark.sql.shuffle.partitions=600 \
+--conf spark.sql.orc.impl=native \
+--conf spark.task.cpus=1 \
+--properties-file /usr/local/spark/conf/omnioperator.conf \
+--database tpcds_bin_partitioned_varchar_orc_2
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/conf/config b/omnioperator/omniop-deploy-tool/conf/config
new file mode 100644
index 000000000..335624e63
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/conf/config
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# omnioperator期待的spark版本号
+expect_spark_version=3.3.1
+# omnioperator版本号
+omnioperator_version=1.7.0
+# 部署目录
+target_path=/opt/omni-operator
+# 如果为openEuler系统，是否安装sve版本的omnioperator，默认为false(小写字母)
+sve_flag=false
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/conf/omni.conf b/omnioperator/omniop-deploy-tool/conf/omni.conf
new file mode 100644
index 000000000..d4392f5de
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/conf/omni.conf
@@ -0,0 +1,28 @@
+# <----Spark template---->
+#数学运算中小数舍入模式，默认为DOWN。HALF_UP表示向最接近数字方向舍入，如果与两个相邻数字的距离相等，则向上舍入，就是通常讲的四舍五入。DOWN表示截断，即向零方向舍入。
+RoundingRule=DOWN
+#Decimal操作结果是否检查溢出，默认为CHECK_RESCALE。CHECK_RESCALE表示检查溢出，NOT_CHECK_RESCALE表示不检查溢出。
+CheckReScaleRule=CHECK_RESCALE
+#Replace操作中，对待空字符是否替换，默认为NOT_REPLACE。REPLAEC表示替换，NOT_REPLACE表示不替换。
+#例如，InputStr="apple", ReplaceStr="*", SearchStr=""，openLooKeng会将字母中间的空字符替换，得到OutputStr="*a*p*p*l*e*"。Spark则不替换，得到OutputStr="apple"。
+EmptySearchStrReplaceRule=NOT_REPLACE
+#Decimal转Double过程中，C++直接转换或先转为字符串再进行转换，默认为CONVERT_WITH_STRING。CAST表示直接转换，CONVERT_WITH_STRING表示先转为字符串再进行转换。
+CastDecimalToDoubleRule=CONVERT_WITH_STRING
+#Substr操作中，负数索引超出最小索引，直接返回空串或仍继续取字符串，默认为INTERCEPT_FROM_BEYOND。EMPTY_STRING表示返回空串，INTERCEPT_FROM_BEYOND表示继续取字符串。
+#例如，str="apple", strLength=5, startIndex=-7, subStringLength=3。 字符串长度为5，从索引-7的位置取3个字符。"apple"长度为5，最小负数索引为-4，由于-7小于-4，OLK直接返回空串，Spark则仍从-7的位置取3个字符后仍继续取字符串，直到取到值"a"后返回
+NegativeStartIndexOutOfBoundsRule=INTERCEPT_FROM_BEYOND
+#是否支持ContainerVector，默认为NOT_SUPPORT。SUPPORT表示支持，NOT_SUPPORT表示不支持。
+SupportContainerVecRule=NOT_SUPPORT
+#字符串转Date过程中，是否支持降低精度，默认为ALLOW_REDUCED_PRECISION。NOT_ALLOW_REDUCED_PRECISION表示不允许降低精度，ALLOW_REDUCED_PRECISION表示允许降低精度。
+#例如，openLooKeng必须完整书写ISO日期扩展格式，不能省略Month和Day，如1996-02-08。Spark支持省略Month和Day，如1996-02-28, 1996-02, 1996都支持。
+StringToDateFormatRule=ALLOW_REDUCED_PRECISION
+#VectorBatch是否包含filter column，默认为NO_EXPR。NO_EXPR表示不包含filter column，EXPR_FILTER表示包含filter column。
+SupportExprFilterRule=EXPR_FILTER
+#在substr运算时，默认为IS_SUPPORT，为IS_NOT_SUPPORT时,表示不支持startIndex=0时从第一个元素开始取，因为默认起始索引从1开始，若起始索引为0，默认返回空字符串，为IS_SUPPORT时，表示支持substr函数在startIndex=0时支持从第一个元素开始取。
+ZeroStartIndexSupportRule=IS_SUPPORT
+#表达式是否校验，
+ExpressionVerifyRule=NOT_VERIFY
+
+# <----Other properties---->
+# 是否开启codegen函数批处理，默认关闭
+enableBatchExprEvaluate=false
diff --git a/omnioperator/omniop-deploy-tool/conf/omnioperator.conf b/omnioperator/omniop-deploy-tool/conf/omnioperator.conf
new file mode 100644
index 000000000..506787d53
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/conf/omnioperator.conf
@@ -0,0 +1,29 @@
+spark.master                       yarn
+spark.eventLog.enabled             true
+spark.eventLog.driver              hdfs://{host_name}:9000/spark2-history
+spark.eventLog.compress            true 
+spark.history.fs.logDirectory      hdfs://{host_name}:9000/spark2-history
+spark.sql.optimizer.runtime.bloomFilter.enabled true 
+spark.driverEnv.LD_LIBRARY_PATH {target_path}/lib 
+spark.driverEnv.LD_PRELOAD {target_path}/lib/libjemalloc.so.2 
+spark.driverEnv.OMNI_HOME {target_path} 
+spark.driver.extraClassPath {target_path}/lib/boostkit-omniop-spark-{spark_version}-{omni_version}-aarch64.jar:{target_path}/lib/boostkit-omniop-bindings-{omni_version}-aarch64.jar:{target_path}/lib/dependencies/protobuf-java-3.15.8.jar:{target_path}/lib/dependencies/boostkit-omniop-native-reader-{spark_version}-{omni_version}.jar 
+spark.driver.extraLibraryPath {target_path}/lib 
+spark.driver.defaultJavaOptions -Djava.library.path={target_path}/lib 
+spark.executorEnv.LD_LIBRARY_PATH ${PWD}/omni/{omni_package_name}/lib
+spark.executorEnv.LD_PRELOAD ${PWD}/omni/{omni_package_name}/lib/libjemalloc.so.2 
+spark.executorEnv.MALLOC_CONF narenas:2 
+spark.executorEnv.OMNI_HOME ${PWD}/omni/{omni_package_name} 
+spark.executor.extraClassPath ${PWD}/omni/{omni_package_name}/lib/boostkit-omniop-spark-{spark_version}-{omni_version}-aarch64.jar:${PWD}/omni/{omni_package_name}/lib/boostkit-omniop-bindings-{omni_version}-aarch64.jar:${PWD}/omni/{omni_package_name}/lib/dependencies/protobuf-java-3.15.8.jar:${PWD}/omni/{omni_package_name}/lib/dependencies/boostkit-omniop-native-reader-{spark_version}-{omni_version}.jar
+spark.executor.extraLibraryPath ${PWD}/omni/{omni_package_name}/lib 
+spark.omni.sql.columnar.fusion false 
+spark.shuffle.manager org.apache.spark.shuffle.sort.OmniColumnarShuffleManager 
+spark.sql.codegen.wholeStage false 
+spark.sql.extensions com.huawei.boostkit.spark.ColumnarPlugin 
+spark.omni.sql.columnar.RewriteSelfJoinInInPredicate true 
+spark.sql.execution.filterMerge.enabled true 
+spark.omni.sql.columnar.dedupLeftSemiJoin true 
+spark.omni.sql.columnar.radixSort.enabled true 
+spark.executorEnv.MALLOC_CONF tcache:false 
+spark.sql.adaptive.coalescePartitions.minPartitionNum 200 
+spark.sql.join.columnar.preferShuffledHashJoin true
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/deploy.sh b/omnioperator/omniop-deploy-tool/deploy.sh
new file mode 100644
index 000000000..950269c3b
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/deploy.sh
@@ -0,0 +1,265 @@
+
+#!/bin/bash
+tool_root_dir=(cd
+(dirname "0")||exit1;pwd)packagesdir="
+tool_root_dir/omnioperator"
+
+chmod -R 771 ${tool_root_dir}
+chown -R :hadoop /home/lrf/deploy
+host_name=(hostname)user=
+(whoami)
+
+#环境变量生效
+dos2unix ./conf/config
+source ./conf/config
+
+################################## 函数 ################################################
+print_with_spaces() {
+    echo ""
+    echo "$1"
+    echo ""
+}
+
+check_spark_version(){
+    print_with_spaces "------start checking spark version------"
+
+    # 获取当前系统的spark版本
+    spark_version=$(spark-sql --version 2>&1 | grep 'version' | head -n 1 | sed -E 's/.*version ([0-9]+\.[0-9]+\.[0-9]+).*/\1/')
+
+    if [ $spark_version == ${expect_spark_version} ]; then
+        echo "INFO: Spark version is right. Expect Spark version is consistent with the SYSTEM spark version."
+    else
+        echo "ERROR: Spark version is wrong! Expect spark version is consistent with the SYSTEM spark version."
+        exit 1
+    fi
+
+    print_with_spaces "------finish checking spark version------"
+}
+
+check_cpu_model(){
+    print_with_spaces "------start checking cpu model------"
+
+    cpu_model=$(lscpu | grep "Model name")
+    if [ $cpu_model == "Kunpeng-920" ] && [ ${sve_flag} == true ]; then
+        echo "ERROR: Kunpeng-920 don't support omnioperator-SVE version!"\
+        exit 1
+    else
+        echo "INFO: Check over."
+
+    print_with_spaces "------finish checking cpu model------"
+}
+
+generate_dir(){
+    echo "------start creating target dir------"
+
+    # 检查用户是否有权限新建该文件夹
+    if [ ! -w "$(dirname "$target_path")" ]; then  
+        echo "Error: You do not have permission to create a directory in $(dirname "$target_path")."  
+        exit 1
+    fi  
+
+    # 提示用户确认输入  
+    read -p "Target path is: "$target_path", please confirm create it or not. (y/n): " confirm  
+
+    # 检查用户确认的输入  
+    if [[ "$confirm" == "y" || "$confirm" == "Y" ]]; then  
+        # 检查目录是否已存在  
+        if [ -d "$target_path" ]; then  
+            read -p "dir "$target_path" is existed, do you want to replace it? (y/n): " delete_confirm  
+            if [[ "$delete_confirm" == "y" || "$delete_confirm" == "Y" ]]; then  
+                rm -rf "$target_path"  
+                echo "INFO: The old directory has been replaced: "$target_path"。"  
+            else  
+                echo "INFO: Process exit."  
+                exit 1  
+            fi  
+        fi  
+
+        # 创建目录  
+        mkdir -p "$target_path"
+        mkdir -p "$target_path"/lib
+        mkdir -p "$target_path"/conf
+        echo "INFO: dir "$target_path" has been created。"  
+    else  
+        echo "INFO: Process exit."
+        exit 1  
+    fi 
+
+    echo "------Create target dir is finished------"
+}
+
+unzip_package(){
+    echo "------Start unziping package------"
+    # 检查系统类型
+    os_type=$(grep -i "^NAME=" /etc/os-release | awk -F= '{print tolower($2)}' | tr -d '"')
+
+    mkdir $packages_dir/unzip_file
+    unzip_file="$packages_dir/unzip_file"
+    cd $packages_dir
+
+    # 检查文件是否存在  
+    if [[ ! -f "Dependency_library_${os_type}.zip" && ! -f "BoostKit-omniop_${omnioperator_version}.zip" && ! -f "boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip" ]]; then  
+        echo "ERROR: Omniop-related packages are missing, Process exit."  
+        exit 1  
+    fi  
+
+    #解压Dependency_library
+    unzip -q Dependency_library_${os_type}.zip  #大小写忽略
+    mv ${packages_dir}/Dependency_library_${os_type}/* ${unzip_file}
+    find . -type d -name "Dependency_library_${os_type}*" ! -name '*.zip' -exec rm -rf {} +
+    echo "Info: Dependency_library_${os_type}.zip has been unzipped"
+
+    if [ "$os_type" != "openeuler" ] && [ "$os_type" != "centos" ]; then
+        echo "Error: do not support: $os_type"  
+        exit 1
+    fi
+
+    if [ "$os_type" == "centos" ] && [ ${sve_flag} == true ]; then
+        echo "Error: CentOS don't support 'SVE' version."   
+        exit 1
+    fi
+
+    if [ ${sve_flag} != true ] && [ ${sve_flag} != false ]; then
+        echo "Error: sve_flag is not a boolean value."   
+        exit 1
+    fi
+
+    #解压BoostKit-omniop_${omnioperator_version}.zip
+    if [ ${sve_flag} == false ]; then
+        unzip -q BoostKit-omniop_${omnioperator_version}.zip boostkit-omniop-operator-${omnioperator_version}-aarch64-${os_type}.tar.gz
+        tar -zxf boostkit-omniop-operator-${omnioperator_version}-aarch64-${os_type}.tar.gz -C ${packages_dir}
+    else
+        unzip -q BoostKit-omniop_${omnioperator_version}.zip boostkit-omniop-operator-${omnioperator_version}-aarch64-${os_type}-sve.tar.gz 
+        tar -zxf boostkit-omniop-operator-${omnioperator_version}-aarch64-${os_type}-sve.tar.gz -C ${packages_dir}
+    fi
+    mv ${packages_dir}/boostkit-omniop-operator-${omnioperator_version}-aarch64/* ${unzip_file}
+    rm -rf boostkit-omniop-operator-${omnioperator_version}-aarch64*
+    echo "Info: BoostKit-omniop_${omnioperator_version}.zip has been unzipped"
+
+    #解压boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip
+    if [ ${sve_flag} == false ]; then
+        unzip -q boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}.zip
+        unzip -q boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}.zip -d ${unzip_file}
+        rm -rf boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}.zip
+    else
+        unzip -q boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}-sve.zip
+        unzip -q boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}-sve.zip -d ${unzip_file}
+        rm -rf boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64-${os_type}-sve.zip
+    fi
+
+    cd ${unzip_file}
+    tar -zxf dependencies.tar.gz
+    rm -rf dependencies.tar.gz
+    cd ${packages_dir}
+    echo "Info: boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip has been unzipped"
+
+    mv ${unzip_file}/* ${target_path}/lib
+    rm -rf ${unzip_file}
+    echo "Info: all unzipped files have been moved to ${target_path}/lib"
+
+    echo "------Unzip package is finished------"
+}
+
+generate_omniconf(){
+    cp ${tool_root_dir}/conf/omni.conf ${target_path}/conf/
+}
+
+upload_hdfs(){
+    echo "------Start uploading hdfs------"
+    username=$(whoami)
+    hdfsdir="/user/${username}"
+
+    # 如果hdfs上用户目录不存在，新增一个
+    hdfs dfs -test -e ${hdfs_dir}
+    if [ $? -eq 1 ]; then
+        echo "Info: hdfs: ${hdfs_dir} is not exist, creating now"
+        hdfs dfs -mkdir ${hdfs_dir}
+    fi
+
+    # 打包omnioperator，准备传到hdfs ${hdfs_dir}，存在同名旧文件会直接覆盖
+    tar -czf ${target_path}.tar.gz -C $(dirname "$target_path") $(basename "$target_path")
+
+    # 检查 HDFS 路径下是否存在同名文件，如存在，则先删除再推送omnioperator包到hdfs
+    hdfs dfs -test -e ${hdfs_dir}/${target_path##*/}.tar.gz
+    if [ $? -eq 0 ]; then
+        hadoop fs -rm ${hdfs_dir}/${target_path##*/}.tar.gz
+    fi
+    hadoop fs -put /opt/${target_path##*/}.tar.gz ${hdfs_dir}
+    echo "Info: successfully upload hdfs"
+
+    echo "------Upload hdfs is finished------"
+}
+
+generate_spark_defaults(){
+    echo "------Start generate_spark_defaults------"
+    # omni版本，omni配置路径
+    conf_file="${tool_root_dir}/conf/omnioperator.conf"
+    target_dir="/usr/local/spark/conf" # 目标目录
+
+    # 检查 omni.conf 文件是否存在
+    if [ ! -f "$conf_file" ]; then
+        echo "Error: $conf_file does not exist!"
+        exit 1
+    fi
+
+    cp $conf_file $(dirname "$conf_file")/omnioperatorbak.cof
+
+    # 替换文件中的内容
+    sed -i "s|{host_name}|${host_name}|g" "$conf_file"
+    sed -i "s|{target_path}|${target_path}|g" "$conf_file"
+    sed -i "s|{spark_version}|${expect_spark_version}|g" "$conf_file"
+    sed -i "s|{omni_version}|${omnioperator_version}|g" "$conf_file"
+    sed -i "s|{omni_package_name}|${target_path##*/}|g" "$conf_file"
+
+    # 如果目标目录中存在同名文件，删除它
+    if [ -f "$target_dir/omnioperator.conf" ]; then
+        echo "File exists in target directory. Removing old omnioperator.conf..."
+        rm "$target_dir/omnioperator.conf"
+    fi
+
+    # 移动修改后的文件到目标目录
+    mv "$conf_file" "$target_dir"
+    mv $(dirname "$conf_file")/omnioperatorbak.cof $conf_file
+    echo "Modified omni.conf has been moved to $target_dir"
+
+    echo "------Generate_spark_defaults is finished------"
+}
+
+generate_command_line(){
+    echo "------Start generate command line------"
+    command_line_tmp="${tool_root_dir}/command/command_line_tmp"
+
+    # 检查 command_line 文件是否存在
+    if [ ! -f "$command_line_tmp" ]; then
+        echo "Error: $command_line_tmp does not exist!"
+        exit 1
+    fi
+
+    # 生成目标文件副本
+    cp $command_line_tmp $(dirname "$command_line_tmp")/command_line
+    command_line=$(dirname "$command_line_tmp")/command_line
+
+    # 替换文件中的内容
+    sed -i "s|{host_name}|${host_name}|g" "$command_line"
+    sed -i "s|{user}|${user}|g" "$command_line"
+    sed -i "s|{omni_tar_name}|${target_path##*/}|g" "$command_line"
+
+    echo "------Generating command line is finished------"
+
+    if [ ${sve_flag} == true ]; then
+        echo "Deployment Successful. OmniOperator version: ${omnioperator_version}-sve; Spark version: ${expect_spark_version}."
+    else
+        echo "Deployment Successful. OmniOperator version: ${omnioperator_version}; Spark version: ${expect_spark_version}."
+    # 结束脚本
+    exit 0
+}
+
+# 修改1 有些检查需要提前
+################################## 执行 ################################################
+check_spark_version
+generate_dir
+unzip_package
+generate_omniconf
+upload_hdfs
+generate_spark_defaults
+generate_command_line
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/readme.md b/omnioperator/omniop-deploy-tool/readme.md
new file mode 100644
index 000000000..de77b6349
--- /dev/null
+++ b/omnioperator/omniop-deploy-tool/readme.md
@@ -0,0 +1,30 @@
+# OmniOperator自动部署工具使用说明：
+
+## 1. 修改配置文件./conf/config
+1.1. `expect_spark_version`：需要部署OmniOperator包对应的spark版本
+1.2. `omnioperator_version`：OmniOperator的版本
+1.3. `target_path`：OmniOperator的部署路径
+1.4. `sve_flag`：是否选择sve版本（centos系统暂不支持，cpu为鲲鹏920的机器暂不支持）
+若机器不支持OmniOperator SVE版本，而选择SVE版本安装，将安装失败
+
+## 2. 获取OmniOperator安装包
+获取：
+
+BoostKit-omniop_{omni_version}.zip
+boostkit-omniop-spark-{spark_version}-{omni_version}-aarch64.zip
+Dependency_library_{os_type}.zip
+
+三个包，并放置在脚本的根目录/omnioperator文件夹下
+
+## 3. 脚本执行
+在脚本的根目录下，执行：
+```shell
+bash deploy.sh
+```
+
+## 4. 获取命令行
+在脚本的根目录下，执行：
+```shell
+vim ./command/command_line
+```
+并复制文件中的内容，粘贴到机器上，执行即可
\ No newline at end of file
-- 
Gitee


From f7e1f7455219cf2923f04c4acd68cbeff9559745 Mon Sep 17 00:00:00 2001
From: liangruifeng <960641173@qq.com>
Date: Tue, 21 Jan 2025 20:32:30 +0800
Subject: [PATCH 2/5] 20250121 changed

---
 ...mnioperator.conf => omnioperator_tmp.conf} |   0
 omnioperator/omniop-deploy-tool/deploy.sh     | 114 ++++++++++++++----
 omnioperator/omniop-deploy-tool/readme.md     |   6 +-
 3 files changed, 94 insertions(+), 26 deletions(-)
 rename omnioperator/omniop-deploy-tool/conf/{omnioperator.conf => omnioperator_tmp.conf} (100%)

diff --git a/omnioperator/omniop-deploy-tool/conf/omnioperator.conf b/omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf
similarity index 100%
rename from omnioperator/omniop-deploy-tool/conf/omnioperator.conf
rename to omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf
diff --git a/omnioperator/omniop-deploy-tool/deploy.sh b/omnioperator/omniop-deploy-tool/deploy.sh
index 950269c3b..526f4d4c7 100644
--- a/omnioperator/omniop-deploy-tool/deploy.sh
+++ b/omnioperator/omniop-deploy-tool/deploy.sh
@@ -1,14 +1,11 @@
 
 #!/bin/bash
-tool_root_dir=(cd
-(dirname "0")||exit1;pwd)packagesdir="
-tool_root_dir/omnioperator"
-
-chmod -R 771 ${tool_root_dir}
-chown -R :hadoop /home/lrf/deploy
-host_name=(hostname)user=
-(whoami)
+tool_root_dir=$(cd $(dirname "$0")||exit 1;pwd)
+packagesdir="$tool_root_dir/omnioperator"
 
+host_name=$(hostname)
+user=$(whoami)
+ 
 #环境变量生效
 dos2unix ./conf/config
 source ./conf/config
@@ -24,9 +21,9 @@ check_spark_version(){
     print_with_spaces "------start checking spark version------"
 
     # 获取当前系统的spark版本
-    spark_version=$(spark-sql --version 2>&1 | grep 'version' | head -n 1 | sed -E 's/.*version ([0-9]+\.[0-9]+\.[0-9]+).*/\1/')
+    spark_version=$(spark-sql --version 2>&1 | grep 'version' | head -n 1 | sed -E 's/.*version ([0-9]+\.[0-9]+)\.[0-9]+.*/\1/')
 
-    if [ $spark_version == ${expect_spark_version} ]; then
+    if [ $spark_version == $(echo ${expect_spark_version} | cut -d'.' -f1,2) ]; then
         echo "INFO: Spark version is right. Expect Spark version is consistent with the SYSTEM spark version."
     else
         echo "ERROR: Spark version is wrong! Expect spark version is consistent with the SYSTEM spark version."
@@ -45,6 +42,7 @@ check_cpu_model(){
         exit 1
     else
         echo "INFO: Check over."
+    fi
 
     print_with_spaces "------finish checking cpu model------"
 }
@@ -193,34 +191,35 @@ upload_hdfs(){
 generate_spark_defaults(){
     echo "------Start generate_spark_defaults------"
     # omni版本，omni配置路径
-    conf_file="${tool_root_dir}/conf/omnioperator.conf"
+    tmp="${tool_root_dir}/conf/omnioperator_tmp.conf"
     target_dir="/usr/local/spark/conf" # 目标目录
 
     # 检查 omni.conf 文件是否存在
-    if [ ! -f "$conf_file" ]; then
-        echo "Error: $conf_file does not exist!"
+    if [ ! -f "$tmp" ]; then
+        echo "Error: $tmp does not exist!"
         exit 1
     fi
 
-    cp $conf_file $(dirname "$conf_file")/omnioperatorbak.cof
+    conf_file_name=${omnioperator_${expect_spark_version}_${omnioperator_version}.conf}
+    conf_file_path="$(dirname "$tmp")/${conf_file_name}"
+    cp $tmp $conf_file_path
 
     # 替换文件中的内容
-    sed -i "s|{host_name}|${host_name}|g" "$conf_file"
-    sed -i "s|{target_path}|${target_path}|g" "$conf_file"
-    sed -i "s|{spark_version}|${expect_spark_version}|g" "$conf_file"
-    sed -i "s|{omni_version}|${omnioperator_version}|g" "$conf_file"
-    sed -i "s|{omni_package_name}|${target_path##*/}|g" "$conf_file"
+    sed -i "s|{host_name}|${host_name}|g" "$conf_file_path"
+    sed -i "s|{target_path}|${target_path}|g" "$conf_file_path"
+    sed -i "s|{spark_version}|${expect_spark_version}|g" "$conf_file_path"
+    sed -i "s|{omni_version}|${omnioperator_version}|g" "$conf_file_path"
+    sed -i "s|{omni_package_name}|${target_path##*/}|g" "$conf_file_path"
 
     # 如果目标目录中存在同名文件，删除它
-    if [ -f "$target_dir/omnioperator.conf" ]; then
-        echo "File exists in target directory. Removing old omnioperator.conf..."
-        rm "$target_dir/omnioperator.conf"
+    if [ -f "$target_dir/${conf_file_name}" ]; then
+        echo "INFO: File ${conf_file_name} has been existed in target path:${target_dir}. Replace by new one..."
+        rm "$target_dir/${conf_file_name}"
     fi
 
     # 移动修改后的文件到目标目录
-    mv "$conf_file" "$target_dir"
-    mv $(dirname "$conf_file")/omnioperatorbak.cof $conf_file
-    echo "Modified omni.conf has been moved to $target_dir"
+    mv "$conf_file_path" "$target_dir"
+    echo "File ${conf_file_name} has been moved to $target_dir"
 
     echo "------Generate_spark_defaults is finished------"
 }
@@ -250,10 +249,75 @@ generate_command_line(){
         echo "Deployment Successful. OmniOperator version: ${omnioperator_version}-sve; Spark version: ${expect_spark_version}."
     else
         echo "Deployment Successful. OmniOperator version: ${omnioperator_version}; Spark version: ${expect_spark_version}."
+    fi
     # 结束脚本
     exit 0
 }
 
+check_omni_function(){
+    echo "------Start checking omni function------"
+
+    omni_start_command=$(cat "${tool_root_dir}/command/command_line")
+
+    result=$(echo "
+    CREATE DATABASE IF NOT EXIST omni_test_db;
+    USE omni_test_db;
+    CREATE TABLE IF NOT EXIST omni_test_table (
+        id INT,
+        name STRING
+    );
+    INSERT INTO omni_test_table VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Omni'), (4, 'Eve');
+    EXPLAIN SELECT * FROM omni_test_table ORDER BY id DESC;
+    DROP TABLE IF EXISTS omni_test_table;
+    DROP DATABASE IF EXISTS omni_test_db;
+    exit;
+    " | $omni_start_command 2>&1)
+
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Error occurred during spark-sql execution."
+        echo "Error details: ${result}"
+        exit 1
+    fi 
+
+    if echo "result" | grep -q "Omni"; then 
+        echo "INFO: Omnioperator is effective."
+    else
+        echo "ERROR: Omnioperator is NOT effective."
+    fi
+    echo "------Finish checking omni function------"
+}
+
+set_env_var(){
+    echo "------Start setting OMNI_HOME && LD_LIBRARY_PATH------"
+
+    # 定义要添加的环境变量及其值
+    OMNI_HOME=${target_path}
+    LD_LIBRARY_PATH=${target_path}/lib
+    LD_PRELOAD=${target_path}/lib/libjemalloc.so.2
+
+    if grep -q "^export OMNI_HOME=" /etc/profile; then
+        sed -i "/^export OMNI_HOME=/c\export OMNI_HOME=$OMNI_HOME" /etc/profile
+    else
+        echo "export OMNI_HOME=$OMNI_HOME" >> /etc/profile
+    fi
+
+    if grep -q "^export LD_LIBRARY_PATH=" /etc/profile; then
+        sed -i "/^export LD_LIBRARY_PATH=/c\export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" /etc/profile
+    else
+        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile
+    fi
+
+    if grep -q "^export LD_PRELOAD=" /etc/profile; then
+        sed -i "/^export LD_PRELOAD=/c\export LD_PRELOAD=$LD_PRELOAD" /etc/profile
+    else
+        echo "export LD_PRELOAD=$LD_PRELOAD" >> /etc/profile
+    fi
+
+    source /etc/profile
+
+    echo "------Finish------"
+}
+
 # 修改1 有些检查需要提前
 ################################## 执行 ################################################
 check_spark_version
diff --git a/omnioperator/omniop-deploy-tool/readme.md b/omnioperator/omniop-deploy-tool/readme.md
index de77b6349..00dada22c 100644
--- a/omnioperator/omniop-deploy-tool/readme.md
+++ b/omnioperator/omniop-deploy-tool/readme.md
@@ -1,9 +1,13 @@
 # OmniOperator自动部署工具使用说明：
 
 ## 1. 修改配置文件./conf/config
-1.1. `expect_spark_version`：需要部署OmniOperator包对应的spark版本
+1.1. `expect_spark_version`：需要部署OmniOperator包对应的spark版本，而不是填系统当前的spark版本
+例如：当前系统的spark版本为3.4.1, 但OmniOperator的包对应spark版本是3.4.3，则expect_spark_version应填3.4.3，否则会报错找不到对应的包
+
 1.2. `omnioperator_version`：OmniOperator的版本
+
 1.3. `target_path`：OmniOperator的部署路径
+
 1.4. `sve_flag`：是否选择sve版本（centos系统暂不支持，cpu为鲲鹏920的机器暂不支持）
 若机器不支持OmniOperator SVE版本，而选择SVE版本安装，将安装失败
 
-- 
Gitee


From de6c8df0c2715105bda45e170f4534c50501a69e Mon Sep 17 00:00:00 2001
From: liangruifeng <960641173@qq.com>
Date: Wed, 22 Jan 2025 10:57:12 +0800
Subject: [PATCH 3/5] 20250122 changed

---
 omnioperator/omniop-deploy-tool/conf/config |   4 +-
 omnioperator/omniop-deploy-tool/deploy.sh   | 176 +++++++++++++++-----
 2 files changed, 134 insertions(+), 46 deletions(-)

diff --git a/omnioperator/omniop-deploy-tool/conf/config b/omnioperator/omniop-deploy-tool/conf/config
index 335624e63..ea8a429cf 100644
--- a/omnioperator/omniop-deploy-tool/conf/config
+++ b/omnioperator/omniop-deploy-tool/conf/config
@@ -6,4 +6,6 @@ omnioperator_version=1.7.0
 # 部署目录
 target_path=/opt/omni-operator
 # 如果为openEuler系统，是否安装sve版本的omnioperator，默认为false(小写字母)
-sve_flag=false
\ No newline at end of file
+sve_flag=false
+# spark的conf目录
+spark_conf_path=/opt/client/Spark/spark/conf
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/deploy.sh b/omnioperator/omniop-deploy-tool/deploy.sh
index 526f4d4c7..f422c7011 100644
--- a/omnioperator/omniop-deploy-tool/deploy.sh
+++ b/omnioperator/omniop-deploy-tool/deploy.sh
@@ -1,8 +1,7 @@
 
 #!/bin/bash
 tool_root_dir=$(cd $(dirname "$0")||exit 1;pwd)
-packagesdir="$tool_root_dir/omnioperator"
-
+packages_dir="$tool_root_dir/omnioperator"
 host_name=$(hostname)
 user=$(whoami)
  
@@ -11,14 +10,18 @@ dos2unix ./conf/config
 source ./conf/config
 
 ################################## 函数 ################################################
-print_with_spaces() {
+function_start() {
     echo ""
     echo "$1"
+}
+
+function_end() {
+    echo "$1"
     echo ""
 }
 
 check_spark_version(){
-    print_with_spaces "------start checking spark version------"
+    function_start "------start checking spark version------"
 
     # 获取当前系统的spark版本
     spark_version=$(spark-sql --version 2>&1 | grep 'version' | head -n 1 | sed -E 's/.*version ([0-9]+\.[0-9]+)\.[0-9]+.*/\1/')
@@ -30,11 +33,11 @@ check_spark_version(){
         exit 1
     fi
 
-    print_with_spaces "------finish checking spark version------"
+    function_end "------FINISH checking spark version------"
 }
 
 check_cpu_model(){
-    print_with_spaces "------start checking cpu model------"
+    function_start "------start checking cpu model------"
 
     cpu_model=$(lscpu | grep "Model name")
     if [ $cpu_model == "Kunpeng-920" ] && [ ${sve_flag} == true ]; then
@@ -44,11 +47,11 @@ check_cpu_model(){
         echo "INFO: Check over."
     fi
 
-    print_with_spaces "------finish checking cpu model------"
+    function_end "------FINISH checking cpu model------"
 }
 
 generate_dir(){
-    echo "------start creating target dir------"
+    function_start "------start creating target dir------"
 
     # 检查用户是否有权限新建该文件夹
     if [ ! -w "$(dirname "$target_path")" ]; then  
@@ -83,11 +86,11 @@ generate_dir(){
         exit 1  
     fi 
 
-    echo "------Create target dir is finished------"
+    function_end "------Create target dir is FINISHED------"
 }
 
 unzip_package(){
-    echo "------Start unziping package------"
+    function_start "------Start unziping package------"
     # 检查系统类型
     os_type=$(grep -i "^NAME=" /etc/os-release | awk -F= '{print tolower($2)}' | tr -d '"')
 
@@ -96,10 +99,26 @@ unzip_package(){
     cd $packages_dir
 
     # 检查文件是否存在  
-    if [[ ! -f "Dependency_library_${os_type}.zip" && ! -f "BoostKit-omniop_${omnioperator_version}.zip" && ! -f "boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip" ]]; then  
-        echo "ERROR: Omniop-related packages are missing, Process exit."  
-        exit 1  
-    fi  
+    missing_files=()
+    if [[ ! -f "${packages_dir}/Dependency_library_${os_type}.zip" ]]; then
+        missing_files+=("Dependency_library_${os_type}.zip")
+    fi
+
+    if [[ ! -f "${packages_dir}/BoostKit-omniop_${omnioperator_version}.zip" ]]; then
+        missing_files+=("BoostKit-omniop_${omnioperator_version}.zip")
+    fi
+
+    if [[ ! -f "${packages_dir}/boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip" ]]; then
+        missing_files+=("boostkit-omniop-spark-${expect_spark_version}-${omnioperator_version}-aarch64.zip")
+    fi    
+    
+    if [[ ${#missing_files[@]} -gt 0 ]]; then
+        echo "ERROR: The following packages are missing in ${packages_dir}:"
+        for file in "${missing_files[@]}"; do
+            echo "- $file"
+        done
+        exit 1
+    fi
 
     #解压Dependency_library
     unzip -q Dependency_library_${os_type}.zip  #大小写忽略
@@ -155,7 +174,7 @@ unzip_package(){
     rm -rf ${unzip_file}
     echo "Info: all unzipped files have been moved to ${target_path}/lib"
 
-    echo "------Unzip package is finished------"
+    function_end "------Unzip package is FINISHED------"
 }
 
 generate_omniconf(){
@@ -163,9 +182,9 @@ generate_omniconf(){
 }
 
 upload_hdfs(){
-    echo "------Start uploading hdfs------"
+    function_start "------Start uploading hdfs------"
     username=$(whoami)
-    hdfsdir="/user/${username}"
+    hdfs_dir="/user/${username}"
 
     # 如果hdfs上用户目录不存在，新增一个
     hdfs dfs -test -e ${hdfs_dir}
@@ -185,14 +204,14 @@ upload_hdfs(){
     hadoop fs -put /opt/${target_path##*/}.tar.gz ${hdfs_dir}
     echo "Info: successfully upload hdfs"
 
-    echo "------Upload hdfs is finished------"
+    function_end "------Upload hdfs is FINISHED------"
 }
 
 generate_spark_defaults(){
-    echo "------Start generate_spark_defaults------"
-    # omni版本，omni配置路径
+    function_start "------Start generate_spark_defaults------"
+
     tmp="${tool_root_dir}/conf/omnioperator_tmp.conf"
-    target_dir="/usr/local/spark/conf" # 目标目录
+    target_dir="${spark_conf_path}" 
 
     # 检查 omni.conf 文件是否存在
     if [ ! -f "$tmp" ]; then
@@ -211,7 +230,6 @@ generate_spark_defaults(){
     sed -i "s|{omni_version}|${omnioperator_version}|g" "$conf_file_path"
     sed -i "s|{omni_package_name}|${target_path##*/}|g" "$conf_file_path"
 
-    # 如果目标目录中存在同名文件，删除它
     if [ -f "$target_dir/${conf_file_name}" ]; then
         echo "INFO: File ${conf_file_name} has been existed in target path:${target_dir}. Replace by new one..."
         rm "$target_dir/${conf_file_name}"
@@ -221,11 +239,11 @@ generate_spark_defaults(){
     mv "$conf_file_path" "$target_dir"
     echo "File ${conf_file_name} has been moved to $target_dir"
 
-    echo "------Generate_spark_defaults is finished------"
+    function_end "------Generate_spark_defaults is FINISHED------"
 }
 
 generate_command_line(){
-    echo "------Start generate command line------"
+    function_start "------Start generate command line------"
     command_line_tmp="${tool_root_dir}/command/command_line_tmp"
 
     # 检查 command_line 文件是否存在
@@ -235,42 +253,108 @@ generate_command_line(){
     fi
 
     # 生成目标文件副本
-    cp $command_line_tmp $(dirname "$command_line_tmp")/command_line
-    command_line=$(dirname "$command_line_tmp")/command_line
+    command_line=$(dirname "$command_line_tmp")/command_line_${expect_spark_version}_${omnioperator_version}
+    cp $command_line_tmp $command_line
 
     # 替换文件中的内容
-    sed -i "s|{host_name}|${host_name}|g" "$command_line"
+    sed -i "s|{hostname}|${host_name}|g" "$command_line"
     sed -i "s|{user}|${user}|g" "$command_line"
     sed -i "s|{omni_tar_name}|${target_path##*/}|g" "$command_line"
 
-    echo "------Generating command line is finished------"
+    function_end "------Generating command line is FINISHED------"
 
     if [ ${sve_flag} == true ]; then
         echo "Deployment Successful. OmniOperator version: ${omnioperator_version}-sve; Spark version: ${expect_spark_version}."
     else
         echo "Deployment Successful. OmniOperator version: ${omnioperator_version}; Spark version: ${expect_spark_version}."
     fi
-    # 结束脚本
     exit 0
 }
 
 check_omni_function(){
-    echo "------Start checking omni function------"
+    function_start "------Start checking omni function------"
 
     omni_start_command=$(cat "${tool_root_dir}/command/command_line")
 
+    # 启动OmniOp后，执行下面sql建表，执行SQL82，验证OmniOp是否生效
     result=$(echo "
-    CREATE DATABASE IF NOT EXIST omni_test_db;
-    USE omni_test_db;
-    CREATE TABLE IF NOT EXIST omni_test_table (
-        id INT,
-        name STRING
+    CREATE DATABASE IF NOT EXISTS test_db;
+
+    USE test_db;
+
+    CREATE TABLE IF NOT EXISTS item (
+        i_item_id INT,
+        i_item_desc STRING,
+        i_current_price DECIMAL(10, 2),
+        i_manufact_id INT,
+        i_item_sk INT
+    );
+
+    CREATE TABLE IF NOT EXISTS inventory (
+        inv_item_sk INT,
+        inv_quantity_on_hand INT,
+        inv_date_sk INT
     );
-    INSERT INTO omni_test_table VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Omni'), (4, 'Eve');
-    EXPLAIN SELECT * FROM omni_test_table ORDER BY id DESC;
-    DROP TABLE IF EXISTS omni_test_table;
-    DROP DATABASE IF EXISTS omni_test_db;
-    exit;
+
+    CREATE TABLE IF NOT EXISTS date_dim (
+        d_date_sk INT,
+        d_date STRING
+    );
+
+    CREATE TABLE IF NOT EXISTS store_sales (
+        ss_item_sk INT,
+        ss_item_id INT,
+        ss_quantity INT
+    );
+
+    INSERT INTO item (i_item_id, i_item_desc, i_current_price, i_manufact_id, i_item_sk)
+    VALUES
+    (1, 'Item A', 80.00, 512, 1),
+    (2, 'Item B', 90.00, 409, 2),
+    (3, 'Item Omni', 100.00, 677, 3),
+    (4, 'Item C', 95.00, 16, 4);
+
+    INSERT INTO inventory (inv_item_sk, inv_quantity_on_hand, inv_date_sk)
+    VALUES
+    (1, 200, 1),
+    (2, 150, 2),
+    (3, 300, 3),
+    (4, 250, 4);
+
+    INSERT INTO date_dim (d_date_sk, d_date)
+    VALUES
+    (1, '1998-06-29'),
+    (2, '1998-07-01'),
+    (3, '1998-08-01'),
+    (4, '1998-08-29');
+
+    INSERT INTO store_sales (ss_item_sk, ss_item_id, ss_quantity)
+    VALUES
+    (1, 1, 50),
+    (2, 2, 60),
+    (3, 3, 70),
+    (4, 4, 80);
+
+    set spark.sql.adaptive.enabled=false;
+    EXPLAIN SELECT i_item_id, i_item_desc, i_current_price
+    FROM item, inventory, date_dim, store_sales
+    WHERE i_current_price BETWEEN 76 AND 106
+    AND inv_item_sk = i_item_sk
+    AND d_date_sk = inv_date_sk
+    AND d_date BETWEEN CAST('1998-06-29' AS DATE) AND CAST('1998-08-29' AS DATE)
+    AND i_manufact_id IN (512, 409, 677, 16)
+    AND inv_quantity_on_hand BETWEEN 100 AND 500
+    AND ss_item_sk = i_item_sk
+    GROUP BY i_item_id, i_item_desc, i_current_price
+    ORDER BY i_item_id
+    LIMIT 100;
+
+    DROP TABLE IF EXISTS store_sales;
+    DROP TABLE IF EXISTS date_dim;
+    DROP TABLE IF EXISTS inventory;
+    DROP TABLE IF EXISTS item;
+
+    DROP DATABASE IF EXISTS test_db;
     " | $omni_start_command 2>&1)
 
     if [ $? -ne 0 ]; then
@@ -284,11 +368,11 @@ check_omni_function(){
     else
         echo "ERROR: Omnioperator is NOT effective."
     fi
-    echo "------Finish checking omni function------"
+    function_end "------FINISH checking omni function------"
 }
 
 set_env_var(){
-    echo "------Start setting OMNI_HOME && LD_LIBRARY_PATH------"
+    function_start "------Start setting OMNI_HOME && LD_LIBRARY_PATH------"
 
     # 定义要添加的环境变量及其值
     OMNI_HOME=${target_path}
@@ -315,15 +399,17 @@ set_env_var(){
 
     source /etc/profile
 
-    echo "------Finish------"
+    function_end "------FINISH------"
 }
 
-# 修改1 有些检查需要提前
 ################################## 执行 ################################################
 check_spark_version
+check_cpu_model
 generate_dir
 unzip_package
 generate_omniconf
 upload_hdfs
 generate_spark_defaults
-generate_command_line
\ No newline at end of file
+generate_command_line
+check_omni_function
+set_env_var
\ No newline at end of file
-- 
Gitee


From d664e2978b337ef966aed20d1e69df510618d33b Mon Sep 17 00:00:00 2001
From: liangruifeng <960641173@qq.com>
Date: Wed, 22 Jan 2025 23:06:02 +0800
Subject: [PATCH 4/5] 20250122 bugfix

---
 .../command/command_line_tmp                  |  3 +-
 omnioperator/omniop-deploy-tool/conf/config   |  8 +-
 .../conf/omnioperator_tmp.conf                |  2 +-
 omnioperator/omniop-deploy-tool/deploy.sh     | 78 ++++++++-----------
 4 files changed, 38 insertions(+), 53 deletions(-)

diff --git a/omnioperator/omniop-deploy-tool/command/command_line_tmp b/omnioperator/omniop-deploy-tool/command/command_line_tmp
index 059c4521e..13e6db273 100644
--- a/omnioperator/omniop-deploy-tool/command/command_line_tmp
+++ b/omnioperator/omniop-deploy-tool/command/command_line_tmp
@@ -19,5 +19,4 @@
 --conf spark.sql.shuffle.partitions=600 \
 --conf spark.sql.orc.impl=native \
 --conf spark.task.cpus=1 \
---properties-file /usr/local/spark/conf/omnioperator.conf \
---database tpcds_bin_partitioned_varchar_orc_2
\ No newline at end of file
+--properties-file {spark_conf_path}/{conf_file_name}
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/conf/config b/omnioperator/omniop-deploy-tool/conf/config
index ea8a429cf..4a20960b6 100644
--- a/omnioperator/omniop-deploy-tool/conf/config
+++ b/omnioperator/omniop-deploy-tool/conf/config
@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
-# omnioperator期待的spark版本号
+# omnioperator期望的spark版本号，当前支持的spark版本有：3.1.1   3.3.1   3.4.3   3.5.2
 expect_spark_version=3.3.1
 # omnioperator版本号
 omnioperator_version=1.7.0
 # 部署目录
 target_path=/opt/omni-operator
-# 如果为openEuler系统，是否安装sve版本的omnioperator，默认为false(小写字母)
+# 是否安装sve版本的omnioperator，默认为false(小写字母)
 sve_flag=false
 # spark的conf目录
-spark_conf_path=/opt/client/Spark/spark/conf
\ No newline at end of file
+spark_conf_path=/usr/local/spark/conf
+# 执行完脚本后是否进行功能验证
+omni_check=true
\ No newline at end of file
diff --git a/omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf b/omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf
index 506787d53..948d219dd 100644
--- a/omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf
+++ b/omnioperator/omniop-deploy-tool/conf/omnioperator_tmp.conf
@@ -1,6 +1,6 @@
 spark.master                       yarn
 spark.eventLog.enabled             true
-spark.eventLog.driver              hdfs://{host_name}:9000/spark2-history
+spark.eventLog.dir              hdfs://{host_name}:9000/spark2-history
 spark.eventLog.compress            true 
 spark.history.fs.logDirectory      hdfs://{host_name}:9000/spark2-history
 spark.sql.optimizer.runtime.bloomFilter.enabled true 
diff --git a/omnioperator/omniop-deploy-tool/deploy.sh b/omnioperator/omniop-deploy-tool/deploy.sh
index f422c7011..a2e13b032 100644
--- a/omnioperator/omniop-deploy-tool/deploy.sh
+++ b/omnioperator/omniop-deploy-tool/deploy.sh
@@ -39,9 +39,9 @@ check_spark_version(){
 check_cpu_model(){
     function_start "------start checking cpu model------"
 
-    cpu_model=$(lscpu | grep "Model name")
+    cpu_model=$(lscpu | grep "Model name" | sed 's/Model name:[[:space:]]*//')
     if [ $cpu_model == "Kunpeng-920" ] && [ ${sve_flag} == true ]; then
-        echo "ERROR: Kunpeng-920 don't support omnioperator-SVE version!"\
+        echo "ERROR: Kunpeng-920 don't support omnioperator-SVE version!"
         exit 1
     else
         echo "INFO: Check over."
@@ -115,17 +115,11 @@ unzip_package(){
     if [[ ${#missing_files[@]} -gt 0 ]]; then
         echo "ERROR: The following packages are missing in ${packages_dir}:"
         for file in "${missing_files[@]}"; do
-            echo "- $file"
+            echo "- ${packages_dir}/$file"
         done
         exit 1
     fi
 
-    #解压Dependency_library
-    unzip -q Dependency_library_${os_type}.zip  #大小写忽略
-    mv ${packages_dir}/Dependency_library_${os_type}/* ${unzip_file}
-    find . -type d -name "Dependency_library_${os_type}*" ! -name '*.zip' -exec rm -rf {} +
-    echo "Info: Dependency_library_${os_type}.zip has been unzipped"
-
     if [ "$os_type" != "openeuler" ] && [ "$os_type" != "centos" ]; then
         echo "Error: do not support: $os_type"  
         exit 1
@@ -141,6 +135,17 @@ unzip_package(){
         exit 1
     fi
 
+    #解压Dependency_library
+    if [ ${sve_flag} == false ]; then
+        unzip -q Dependency_library_${os_type}.zip  #大小写忽略
+        mv ${packages_dir}/Dependency_library_${os_type}/* ${unzip_file}    
+    else
+        unzip -q Dependency_library_${os_type}-sve.zip  #大小写忽略
+        mv ${packages_dir}/Dependency_library_${os_type}-sve/* ${unzip_file}   
+    fi
+    find . -type d -name "Dependency_library_${os_type}*" ! -name '*.zip' -exec rm -rf {} +
+    echo "Info: Dependency_library_${os_type}.zip has been unzipped"
+
     #解压BoostKit-omniop_${omnioperator_version}.zip
     if [ ${sve_flag} == false ]; then
         unzip -q BoostKit-omniop_${omnioperator_version}.zip boostkit-omniop-operator-${omnioperator_version}-aarch64-${os_type}.tar.gz
@@ -201,8 +206,15 @@ upload_hdfs(){
     if [ $? -eq 0 ]; then
         hadoop fs -rm ${hdfs_dir}/${target_path##*/}.tar.gz
     fi
-    hadoop fs -put /opt/${target_path##*/}.tar.gz ${hdfs_dir}
-    echo "Info: successfully upload hdfs"
+    hadoop fs -put ${target_path}.tar.gz ${hdfs_dir}
+    
+    hdfs dfs -test -e ${hdfs_dir}/${target_path##*/}.tar.gz
+    if [ $? -eq 0 ]; then
+        echo "Info: successfully upload hdfs"
+    else
+        echo "ERROR: ${target_path##*/}.tar.gz didn't exist in hdfs."
+        exit 1
+    fi
 
     function_end "------Upload hdfs is FINISHED------"
 }
@@ -219,7 +231,7 @@ generate_spark_defaults(){
         exit 1
     fi
 
-    conf_file_name=${omnioperator_${expect_spark_version}_${omnioperator_version}.conf}
+    conf_file_name="${omnioperator_${expect_spark_version}_${omnioperator_version}.conf}"
     conf_file_path="$(dirname "$tmp")/${conf_file_name}"
     cp $tmp $conf_file_path
 
@@ -254,12 +266,14 @@ generate_command_line(){
 
     # 生成目标文件副本
     command_line=$(dirname "$command_line_tmp")/command_line_${expect_spark_version}_${omnioperator_version}
-    cp $command_line_tmp $command_line
+    cp ${command_line_tmp} ${command_line}
 
     # 替换文件中的内容
     sed -i "s|{hostname}|${host_name}|g" "$command_line"
     sed -i "s|{user}|${user}|g" "$command_line"
     sed -i "s|{omni_tar_name}|${target_path##*/}|g" "$command_line"
+    sed -i "s|{conf_file_name}|"omnioperator_${expect_spark_version}_${omnioperator_version}.conf"|g" "$command_line"
+    sed -i "s|{spark_conf_path}|${spark_conf_path}|g" "$command_line"
 
     function_end "------Generating command line is FINISHED------"
 
@@ -274,7 +288,7 @@ generate_command_line(){
 check_omni_function(){
     function_start "------Start checking omni function------"
 
-    omni_start_command=$(cat "${tool_root_dir}/command/command_line")
+    omni_start_command=$(cat "${tool_root_dir}/command/command_line_${expect_spark_version}_${omnioperator_version}")
 
     # 启动OmniOp后，执行下面sql建表，执行SQL82，验证OmniOp是否生效
     result=$(echo "
@@ -371,37 +385,6 @@ check_omni_function(){
     function_end "------FINISH checking omni function------"
 }
 
-set_env_var(){
-    function_start "------Start setting OMNI_HOME && LD_LIBRARY_PATH------"
-
-    # 定义要添加的环境变量及其值
-    OMNI_HOME=${target_path}
-    LD_LIBRARY_PATH=${target_path}/lib
-    LD_PRELOAD=${target_path}/lib/libjemalloc.so.2
-
-    if grep -q "^export OMNI_HOME=" /etc/profile; then
-        sed -i "/^export OMNI_HOME=/c\export OMNI_HOME=$OMNI_HOME" /etc/profile
-    else
-        echo "export OMNI_HOME=$OMNI_HOME" >> /etc/profile
-    fi
-
-    if grep -q "^export LD_LIBRARY_PATH=" /etc/profile; then
-        sed -i "/^export LD_LIBRARY_PATH=/c\export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" /etc/profile
-    else
-        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/profile
-    fi
-
-    if grep -q "^export LD_PRELOAD=" /etc/profile; then
-        sed -i "/^export LD_PRELOAD=/c\export LD_PRELOAD=$LD_PRELOAD" /etc/profile
-    else
-        echo "export LD_PRELOAD=$LD_PRELOAD" >> /etc/profile
-    fi
-
-    source /etc/profile
-
-    function_end "------FINISH------"
-}
-
 ################################## 执行 ################################################
 check_spark_version
 check_cpu_model
@@ -411,5 +394,6 @@ generate_omniconf
 upload_hdfs
 generate_spark_defaults
 generate_command_line
-check_omni_function
-set_env_var
\ No newline at end of file
+if [ ${omni_check,,} == true ]; then
+    check_omni_function
+fi
\ No newline at end of file
-- 
Gitee


From eb6d6c989c57762b3c06a5113e7e20962f9bc789 Mon Sep 17 00:00:00 2001
From: liangruifeng <960641173@qq.com>
Date: Wed, 22 Jan 2025 23:41:13 +0800
Subject: [PATCH 5/5] 20250122 hyy commit fix

---
 omnioperator/omniop-deploy-tool/deploy.sh | 87 ++++++++++++++---------
 1 file changed, 52 insertions(+), 35 deletions(-)

diff --git a/omnioperator/omniop-deploy-tool/deploy.sh b/omnioperator/omniop-deploy-tool/deploy.sh
index a2e13b032..c2f08d93c 100644
--- a/omnioperator/omniop-deploy-tool/deploy.sh
+++ b/omnioperator/omniop-deploy-tool/deploy.sh
@@ -1,4 +1,4 @@
-
+Start
 #!/bin/bash
 tool_root_dir=$(cd $(dirname "$0")||exit 1;pwd)
 packages_dir="$tool_root_dir/omnioperator"
@@ -21,23 +21,29 @@ function_end() {
 }
 
 check_spark_version(){
-    function_start "------start checking spark version------"
+    function_start "------Start checking spark version------"
 
     # 获取当前系统的spark版本
     spark_version=$(spark-sql --version 2>&1 | grep 'version' | head -n 1 | sed -E 's/.*version ([0-9]+\.[0-9]+)\.[0-9]+.*/\1/')
 
     if [ $spark_version == $(echo ${expect_spark_version} | cut -d'.' -f1,2) ]; then
-        echo "INFO: Spark version is right. Expect Spark version is consistent with the SYSTEM spark version."
+        if [ $spark_version == ${expect_spark_version} ]
+            echo "INFO: Spark version is right. Expect Spark version is consistent with the SYSTEM spark version."
+        else
+            echo "INFO: Spark version is $spark_version."
+            echo "INFO: Omni expected spark version is ${expect_spark_version}."
     else
         echo "ERROR: Spark version is wrong! Expect spark version is consistent with the SYSTEM spark version."
+        echo "Spark version is $spark_version."
+        echo "Omni expected spark version is ${expect_spark_version}."
         exit 1
     fi
 
-    function_end "------FINISH checking spark version------"
+    function_end "------Finish checking spark version------"
 }
 
 check_cpu_model(){
-    function_start "------start checking cpu model------"
+    function_start "------Start checking cpu model------"
 
     cpu_model=$(lscpu | grep "Model name" | sed 's/Model name:[[:space:]]*//')
     if [ $cpu_model == "Kunpeng-920" ] && [ ${sve_flag} == true ]; then
@@ -47,11 +53,11 @@ check_cpu_model(){
         echo "INFO: Check over."
     fi
 
-    function_end "------FINISH checking cpu model------"
+    function_end "------Finish checking cpu model------"
 }
 
 generate_dir(){
-    function_start "------start creating target dir------"
+    function_start "------Start creating target dir------"
 
     # 检查用户是否有权限新建该文件夹
     if [ ! -w "$(dirname "$target_path")" ]; then  
@@ -77,16 +83,16 @@ generate_dir(){
         fi  
 
         # 创建目录  
-        mkdir -p "$target_path"
-        mkdir -p "$target_path"/lib
-        mkdir -p "$target_path"/conf
+        mkdir -p "${target_path}/lib" "${target_path}/conf"
         echo "INFO: dir "$target_path" has been created。"  
     else  
         echo "INFO: Process exit."
+        echo "If you want to install OmniOperator in another dir,"
+        echo "Please change configuration in ${tool_root_dir}/conf/config"
         exit 1  
     fi 
 
-    function_end "------Create target dir is FINISHED------"
+    function_end "------Finish creating target dir------"
 }
 
 unzip_package(){
@@ -94,7 +100,32 @@ unzip_package(){
     # 检查系统类型
     os_type=$(grep -i "^NAME=" /etc/os-release | awk -F= '{print tolower($2)}' | tr -d '"')
 
-    mkdir $packages_dir/unzip_file
+    if [ "$os_type" != "openeuler" ] && [ "$os_type" != "centos" ]; then
+        echo "Error: do not support: $os_type"  
+        exit 1
+    fi
+
+    if [ "$os_type" == "centos" ] && [ ${sve_flag} == true ]; then
+        echo "Error: CentOS don't support 'SVE' version."   
+        exit 1
+    fi
+
+    if [ ${sve_flag} != true ] && [ ${sve_flag} != false ]; then
+        echo "Error: sve_flag is not a boolean value."   
+        exit 1
+    fi
+
+    # 创建unzip_file，检查 是否有权限 及 文件是否存在
+    if [ -f "${packages_dir}/unzip_file" ]; then
+        rm -rf ${packages_dir}/unzip_file
+    fi
+
+    if [ ! -w "$(dirname "${packages_dir}")" ]; then  
+        echo "Error: You do not have permission to create a directory in $(dirname "${packages_dir}")."  
+        exit 1
+    else
+        mkdir $packages_dir/unzip_file
+    fi      
     unzip_file="$packages_dir/unzip_file"
     cd $packages_dir
 
@@ -120,21 +151,6 @@ unzip_package(){
         exit 1
     fi
 
-    if [ "$os_type" != "openeuler" ] && [ "$os_type" != "centos" ]; then
-        echo "Error: do not support: $os_type"  
-        exit 1
-    fi
-
-    if [ "$os_type" == "centos" ] && [ ${sve_flag} == true ]; then
-        echo "Error: CentOS don't support 'SVE' version."   
-        exit 1
-    fi
-
-    if [ ${sve_flag} != true ] && [ ${sve_flag} != false ]; then
-        echo "Error: sve_flag is not a boolean value."   
-        exit 1
-    fi
-
     #解压Dependency_library
     if [ ${sve_flag} == false ]; then
         unzip -q Dependency_library_${os_type}.zip  #大小写忽略
@@ -179,7 +195,7 @@ unzip_package(){
     rm -rf ${unzip_file}
     echo "Info: all unzipped files have been moved to ${target_path}/lib"
 
-    function_end "------Unzip package is FINISHED------"
+    function_end "------Finish unziping package------"
 }
 
 generate_omniconf(){
@@ -216,11 +232,11 @@ upload_hdfs(){
         exit 1
     fi
 
-    function_end "------Upload hdfs is FINISHED------"
+    function_end "------Finish uploading hdfs------"
 }
 
 generate_spark_defaults(){
-    function_start "------Start generate_spark_defaults------"
+    function_start "------Start generating spark defaults------"
 
     tmp="${tool_root_dir}/conf/omnioperator_tmp.conf"
     target_dir="${spark_conf_path}" 
@@ -251,11 +267,11 @@ generate_spark_defaults(){
     mv "$conf_file_path" "$target_dir"
     echo "File ${conf_file_name} has been moved to $target_dir"
 
-    function_end "------Generate_spark_defaults is FINISHED------"
+    function_end "------Finish generating spark defaults------"
 }
 
 generate_command_line(){
-    function_start "------Start generate command line------"
+    function_start "------Start generating command line------"
     command_line_tmp="${tool_root_dir}/command/command_line_tmp"
 
     # 检查 command_line 文件是否存在
@@ -275,7 +291,7 @@ generate_command_line(){
     sed -i "s|{conf_file_name}|"omnioperator_${expect_spark_version}_${omnioperator_version}.conf"|g" "$command_line"
     sed -i "s|{spark_conf_path}|${spark_conf_path}|g" "$command_line"
 
-    function_end "------Generating command line is FINISHED------"
+    function_end "------Finish generating command line------"
 
     if [ ${sve_flag} == true ]; then
         echo "Deployment Successful. OmniOperator version: ${omnioperator_version}-sve; Spark version: ${expect_spark_version}."
@@ -382,7 +398,7 @@ check_omni_function(){
     else
         echo "ERROR: Omnioperator is NOT effective."
     fi
-    function_end "------FINISH checking omni function------"
+    function_end "------Finish checking omni function------"
 }
 
 ################################## 执行 ################################################
@@ -396,4 +412,5 @@ generate_spark_defaults
 generate_command_line
 if [ ${omni_check,,} == true ]; then
     check_omni_function
-fi
\ No newline at end of file
+fi
+echo "-----------ALL Finish----------"
\ No newline at end of file
-- 
Gitee