From 7731e9111c926c57392babaf0018d28b4eb14cab Mon Sep 17 00:00:00 2001 From: wuhuiquan Date: Thu, 28 Nov 2024 21:10:28 +0800 Subject: [PATCH] [build] build code refactoring Signed-off-by: wuhuiquan --- rust-build/build.sh | 95 ----------------- rust-build/function.sh | 81 +++++++++++++++ rust-build/mac_arm64_config.toml | 2 +- rust-build/mac_x8664_config.toml | 2 +- rust-build/mobile_ci_build.sh | 170 +++++++++++++++++++++++++++++++ rust-build/ohos_ci_build.sh | 98 ++++++++++++++++++ 6 files changed, 351 insertions(+), 97 deletions(-) delete mode 100644 rust-build/build.sh create mode 100644 rust-build/function.sh create mode 100644 rust-build/mobile_ci_build.sh create mode 100644 rust-build/ohos_ci_build.sh diff --git a/rust-build/build.sh b/rust-build/build.sh deleted file mode 100644 index 927ea085697..00000000000 --- a/rust-build/build.sh +++ /dev/null @@ -1,95 +0,0 @@ -#! /bin/bash -set -e - -readonly shell_path=$(cd $(dirname $0); pwd) -readonly root_build_dir="${shell_path}/../../../.." -readonly rust_source_dir="${shell_path}/.." -readonly rust_tools="${root_build_dir}/prebuilts" - -case $(uname -s) in - Linux) - host_platform=linux - ;; - Darwin) - host_platform=darwin - ;; - *) - echo "Unsupported host platform: $(uname -s)" - exit 1 -esac - -case $(uname -m) in - arm64) - host_cpu=arm64 - ;; - *) - host_cpu=x86_64 -esac - -function update_config_toml_clang(){ - # $1:clang path, $2:clang name, $3:ar name - - # add clang absolute path to shell tools - sys_clang_dir="$(echo ${1} | sed 's/\//\\\//g')" - sed -i "s/exec ${2}/exec ${sys_clang_dir}\/${2}/g" ${rust_source_dir}/build/${2} - sed -i "s/exec ${2}++/exec ${sys_clang_dir}\/${2}++/g" ${rust_source_dir}/build/${2}++ - # add clang absolute path to config.toml - rust_clang_dir="$(echo ${rust_source_dir}/build | sed 's/\//\\\//g')" - sed -i "s/cc = \"${2}\"/cc = \"${rust_clang_dir}\/${2}\"/g" ${rust_source_dir}/config.toml - sed -i "s/cxx = \"${2}++\"/cxx = \"${rust_clang_dir}\/${2}++\"/g" ${rust_source_dir}/config.toml - sed -i "s/linker = \"${2}\"/linker = \"${sys_clang_dir}\/${2}\"/g" ${rust_source_dir}/config.toml - sed -i "s/ar = \"${3}\"/ar = \"${sys_clang_dir}\/${3}\"/g" ${rust_source_dir}/config.toml -} - -# download rust 1.71 for build -readonly rust_down_dir="${root_build_dir}/rust_download" -mkdir -p ${rust_down_dir} -readonly rust_down_net="https://mirrors.ustc.edu.cn/rust-static/dist" -cd ${rust_down_dir} -mkdir -p ${rust_source_dir}/build/cache/2023-07-13 - -if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rust-std-1.71.0-x86_64-unknown-linux-gnu.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rustc-1.71.0-x86_64-unknown-linux-gnu.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/cargo-1.71.0-x86_64-unknown-linux-gnu.tar.xz - - cp ${shell_path}/config.toml ${rust_source_dir} - chmod 750 ${shell_path}/tools/* - cp ${shell_path}/tools/* ${rust_source_dir}/build/ - update_config_toml_clang ${rust_tools}/ohos-sdk/linux/12/native/llvm/bin clang llvm-ar - update_config_toml_clang ${rust_tools}/ohos-sdk/linux/12/native/llvm/bin aarch64-unknown-linux-ohos-clang llvm-ar - update_config_toml_clang ${rust_tools}/ohos-sdk/linux/12/native/llvm/bin armv7-unknown-linux-ohos-clang llvm-ar - update_config_toml_clang ${rust_tools}/ohos-sdk/linux/12/native/llvm/bin x86_64-unknown-linux-ohos-clang llvm-ar - update_config_toml_clang ${rust_tools}/mingw-w64/ohos/linux-x86_64/clang-mingw/bin x86_64-w64-mingw32-clang x86_64-w64-mingw32-ar -elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "x86_64" ]; then - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rustc-1.71.0-x86_64-apple-darwin.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rust-std-1.71.0-x86_64-apple-darwin.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/cargo-1.71.0-x86_64-apple-darwin.tar.xz - cp ${shell_path}/mac_x8664_config.toml ${rust_source_dir}/config.toml -elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "arm64" ]; then - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rustc-1.71.0-aarch64-apple-darwin.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/rust-std-1.71.0-aarch64-apple-darwin.tar.xz - curl -O -k -m 300 ${rust_down_net}/2023-07-13/cargo-1.71.0-aarch64-apple-darwin.tar.xz - cp ${shell_path}/mac_arm64_config.toml ${rust_source_dir}/config.toml -else - echo "Unsupported platform: $(uname -s) $(uname -m)" -fi - -cp ${rust_down_dir}/*.tar.xz ${rust_source_dir}/build/cache/2023-07-13/ - -curl -O -k -m 300 ${rust_down_net}/rustc-1.72.0-src.tar.gz -tar xf rustc-1.72.0-src.tar.gz -cd ${rust_down_dir}/rustc-1.72.0-src/ -cp -r .cargo ${rust_source_dir} -cp -r vendor ${rust_source_dir} -cp -r library ${rust_source_dir} -cp -r src/doc/* ${rust_source_dir}/src/doc -cp -r src/tools/cargo/* ${rust_source_dir}/src/tools/cargo - -cp -r ${root_build_dir}/../harmony/llvm/* ${rust_source_dir}/src/llvm-project/ - -cd ${rust_source_dir} -if [ "${host_platform}" = "linux" ]; then -export PATH=${rust_tools}/cmake/linux-x86/bin:${rust_tools}/clang/ohos/linux-x86_64/llvm/bin:$PATH -fi -python3 ./x.py dist diff --git a/rust-build/function.sh b/rust-build/function.sh new file mode 100644 index 00000000000..7df79c39f74 --- /dev/null +++ b/rust-build/function.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +detect_platform() { + case $(uname -s) in + Linux) host_platform=linux ;; + Darwin) host_platform=darwin ;; + *) echo "Unsupported host platform: $(uname -s)"; exit 1 ;; + esac + + case $(uname -m) in + arm64) host_cpu=arm64 ;; + *) host_cpu=x86_64 ;; + esac +} + +copy_config() { + if [ "${host_platform}" = "linux" ] && [ "${host_cpu}" = "x86_64" ]; then + cp ${shell_path}/config.toml ${rust_source_dir} + chmod 750 ${shell_path}/tools/* + cp ${shell_path}/tools/* ${rust_source_dir}/build/ + elif [ "${host_platform}" = "darwin" ]; then + if [ "${host_cpu}" = "x86_64" ]; then + cp ${shell_path}/mac_x8664_config.toml ${rust_source_dir}/config.toml + else + cp ${shell_path}/mac_arm64_config.toml ${rust_source_dir}/config.toml + fi + else + echo "Unsupported platform: $(uname -s) $(uname -m)" + exit 1 + fi +} + +update_config_clang_path() { + # $1:clang path, $2:clang name, $3:ar name + + # add clang absolute path to shell tools + sys_clang_dir="$(echo ${1} | sed 's/\//\\\//g')" + sed -i "s/exec ${2}/exec ${sys_clang_dir}\/${2}/g" ${rust_source_dir}/build/${2} + sed -i "s/exec ${2}++/exec ${sys_clang_dir}\/${2}++/g" ${rust_source_dir}/build/${2}++ + # add clang absolute path to config.toml + rust_clang_dir="$(echo ${rust_source_dir}/build | sed 's/\//\\\//g')" + sed -i "s/cc = \"${2}\"/cc = \"${rust_clang_dir}\/${2}\"/g" ${rust_source_dir}/config.toml + sed -i "s/cxx = \"${2}++\"/cxx = \"${rust_clang_dir}\/${2}++\"/g" ${rust_source_dir}/config.toml + sed -i "s/linker = \"${2}\"/linker = \"${sys_clang_dir}\/${2}\"/g" ${rust_source_dir}/config.toml + sed -i "s/ar = \"${3}\"/ar = \"${sys_clang_dir}\/${3}\"/g" ${rust_source_dir}/config.toml +} + +update_config_clang() { + # $1:OH clang path $2:mingw clang path + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + update_config_clang_path ${1} clang llvm-ar + update_config_clang_path ${1} aarch64-unknown-linux-ohos-clang llvm-ar + update_config_clang_path ${1} armv7-unknown-linux-ohos-clang llvm-ar + update_config_clang_path ${1} x86_64-unknown-linux-ohos-clang llvm-ar + update_config_clang_path ${2} x86_64-w64-mingw32-clang x86_64-w64-mingw32-ar + fi +} + +update_version() { + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + sed -i "s/$old_version/$new_version/g" ${rust_source_dir}/config.toml + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "x86_64" ]; then + sed -i "" "s/$old_version/$new_version/g" ${rust_source_dir}/config.toml + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "arm64" ]; then + sed -i "" "s/$old_version/$new_version/g" ${rust_source_dir}/config.toml + else + echo "Unsupported platform: $(uname -s) $(uname -m)" + fi +} + +move_static_rust_source() { + # $1:static rust code path. $2:rust code path + pushd ${1} + tar xf rustc-1.72.0-src.tar.gz + pushd ${1}/rustc-1.72.0-src + cp -r {.cargo,vendor,library} ${2} + cp -r src/doc/* ${2}/src/doc/ + cp -r src/tools/cargo/* ${2}/src/tools/cargo/ + popd + popd +} diff --git a/rust-build/mac_arm64_config.toml b/rust-build/mac_arm64_config.toml index fa904f07aae..f521eac69da 100644 --- a/rust-build/mac_arm64_config.toml +++ b/rust-build/mac_arm64_config.toml @@ -30,7 +30,7 @@ backtrace = false description = "xxxxx" channel = "nightly" strip = true -stack-protector = "all" +stack-protector = "strong" [install] bindir = "bin" diff --git a/rust-build/mac_x8664_config.toml b/rust-build/mac_x8664_config.toml index 51a74539429..17e57d77b86 100644 --- a/rust-build/mac_x8664_config.toml +++ b/rust-build/mac_x8664_config.toml @@ -30,7 +30,7 @@ backtrace = false description = "xxxxx" channel = "nightly" strip = true -stack-protector = "all" +stack-protector = "strong" [install] bindir = "bin" diff --git a/rust-build/mobile_ci_build.sh b/rust-build/mobile_ci_build.sh new file mode 100644 index 00000000000..7ed2a7b71f7 --- /dev/null +++ b/rust-build/mobile_ci_build.sh @@ -0,0 +1,170 @@ +#!/bin/bash +set -e + +readonly shell_path=$(cd $(dirname $0); pwd) +readonly root_build_dir="${shell_path}/../.." +readonly rust_source_dir="${shell_path}/.." +readonly ci_shell_dir="${root_build_dir}/build/rustbuild/" +readonly install_path="${rust_source_dir}/build/dist" +readonly output_install="${root_build_dir}/output/" +readonly oh_tools="/usr1/mobile_cpu/BiSheng/bin" +readonly mingw_tools="/opt/buildtools/llvm-mingw-20220906/bin" +readonly bep_home="/opt/buildtools/secBepkit-2.2.0" +readonly old_version="xxxxx" +new_version="xxxxx" +bep_build=false +bep_time="2024-09-29 12:00:00" +source ${shell_path}/function.sh +export PATH=$rust_source_dir:/usr/local/bin:$PATH + +show_help() { + echo "Usage: $0 [OPTIONS] [ARGUMENTS]" + echo "Options:" + echo " -h Display this help message" + echo " -v Set build version" + echo " -b Bep build, warning: will delete git information" + echo " -t Set bep build time" + echo "Arguments:" + echo " ARGUMENTS are additional values passed to the script" +} + +parse_options() { + local OPTIND=1 + while getopts "hv:b:t:" opt; do + case $opt in + h) show_help; exit 0 ;; + v) new_version="$OPTARG" ;; + b) bep_build="$OPTARG" ;; + t) bep_time="$OPTARG" ;; + \?) echo "unsupport arguments -$OPTARG" >&2 + exit 1 ;; + esac + done + shift $((OPTIND - 1)) + return 0 +} + +download_rust() { + local pre_rust_date="2023-07-13" + mkdir -p ${rust_source_dir}/build/cache/${pre_rust_date} + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + if [ ! -d "${root_build_dir}/opensource" ]; then + artget pull -ap "${root_build_dir}/opensource" -os "${ci_shell_dir}/rust1.72.0.xml" -at opensource + artget pull -ap "${root_build_dir}/opensource" -os "${ci_shell_dir}/rust1.71.0_linux.xml" -at opensource + fi + elif [ "${host_platform}" = "darwin" ]; then + artget pull -ap "${root_build_dir}/opensource" -os "${ci_shell_dir}/rust1.72.0.xml" -at opensource + if [ ${host_cpu} = "x86_64" ]; then + artget pull -ap "${root_build_dir}/opensource" -os "${ci_shell_dir}/rust1.71.0_mac_x86.xml" -at opensource + elif [ ${host_cpu} = "arm64" ]; then + artget pull -ap "${root_build_dir}/opensource" -os "${ci_shell_dir}/rust1.71.0_mac_arm.xml" -at opensource + fi + fi + cp ${root_build_dir}/opensource/Rust/1.71.0/* ${rust_source_dir}/build/cache/${pre_rust_date} +} + +bep_prepare() { + # bep patch + if [ -n "$(grep -rn "HashMap" $root_build_dir/rust/vendor/compiler_builtins/build.rs)" ];then + pushd ${rust_source_dir}/vendor/compiler_builtins/ + patch < ${ci_shell_dir}/vendor.patch + popd + fi + if [ -n "$(grep -rn "3f2f9589f896c5dc7e7ce39f5809f4f63d1199e5f80ae23b343a7f1d889d0206" $root_build_dir/rust/vendor/compiler_builtins/.cargo-checksum.json)" ];then + cp ${ci_shell_dir}/.cargo-checksum.json ${rust_source_dir}/vendor/compiler_builtins/ + fi + + # remove .git .gitlab .gitattributes for bep + pushd ${rust_source_dir} + (find . -name ".git" -print && find . -name ".gitlab" -print && find . -name ".gitattributes" -print) | xargs rm -rf + (find . -name ".gitignore" -print && find . -name ".github" -print && find . -name ".gitkeep" -print) | xargs rm -rf + popd +} + +lock_bep_time() { + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + sed -i "2s/.*/bep_timestamp=${bep_time}/" ${ci_shell_dir}/bep_env.conf + sed -i "3s/.*/second=${bep_time}/" ${ci_shell_dir}/bep_env.conf + sed -i "4s/.*/third=${bep_time}/" ${ci_shell_dir}/bep_env.conf + source ${bep_home}/bep_env.sh -s ${ci_shell_dir}/bep_env.conf + fi +} + +unlock_bep_time() { + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + source ${bep_home}/bep_env.sh -u + fi +} + +collect_linux_build_result() { + # $1:file name + pushd ${install_path} + tar -xf ${1}.tar.gz + rm ${1}.tar.gz + chmod 750 $install_path/${1} -R + tar --format=gnu -czf ${1}.tar.gz ${1} + cp $install_path/${1}.tar.gz ${output_install} + popd +} + +collect_mac_build_result() { + # $1:file name + pushd ${install_path} + tar -xf ${1}.tar.gz + rm ${1}.tar.gz + find ${1}/* -exec touch -h -m -d "$bep_time" {} \; + touch -h -m -d "$bep_time" ${1} + chmod -R 750 ${1} + tar -cf ${1}.tar ${1} + gzip -n ${1}.tar + rm -rf ${1} + cp ${install_path}/${1}.tar.gz ${output_install} + popd +} + +collect_build_result() { + mkdir -p ${output_install} + + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + collect_linux_build_result rust-nightly-x86_64-unknown-linux-gnu + collect_linux_build_result rust-std-nightly-x86_64-pc-windows-gnullvm + collect_linux_build_result rust-std-nightly-aarch64-unknown-linux-ohos + collect_linux_build_result rust-std-nightly-armv7-unknown-linux-ohos + collect_linux_build_result rust-std-nightly-x86_64-unknown-linux-ohos + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "x86_64" ]; then + collect_mac_build_result rust-nightly-x86_64-apple-darwin + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "arm64" ]; then + collect_mac_build_result rust-nightly-aarch64-apple-darwin + fi +} + +main() { + parse_options "$@" + detect_platform + download_rust + copy_config + update_config_clang ${oh_tools} ${mingw_tools} + update_version + move_static_rust_source "${root_build_dir}/opensource/Rust/1.72.0/" ${rust_source_dir} + + rm -rf ${rust_source_dir}/src/llvm-project/* + cp -r ${root_build_dir}/llvm/* ${rust_source_dir}/src/llvm-project + + if [ ${bep_build} = true ]; then + bep_prepare + fi + + pushd ${rust_source_dir} + python3 ./x.py dist + popd + + if [ ${bep_build} = true ]; then + lock_bep_time + collect_build_result + unlock_bep_time + fi + + echo "Building the rust toolchain Completed" +} + +main "$@" diff --git a/rust-build/ohos_ci_build.sh b/rust-build/ohos_ci_build.sh new file mode 100644 index 00000000000..79c9eda9f66 --- /dev/null +++ b/rust-build/ohos_ci_build.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -e + +readonly shell_path=$(cd $(dirname $0); pwd) +readonly root_build_dir="${shell_path}/../../../.." +readonly rust_source_dir="${shell_path}/.." +readonly rust_tools="${root_build_dir}/prebuilts" +readonly output_install="${root_build_dir}/output/" +readonly rust_static_dir="${root_build_dir}/rust_download" +readonly oh_tools="${rust_tools}/ohos-sdk/linux/12/native/llvm/bin" +readonly mingw_tools="${rust_tools}/mingw-w64/ohos/linux-x86_64/clang-mingw/bin" +readonly old_version="xxxxx" +new_version="xxxxx" + +source ${shell_path}/function.sh + +download_rust_static_source() { + local pre_rust_date="2023-07-13" + mkdir -p ${rust_static_dir} ${rust_source_dir}/build/cache/${pre_rust_date} + local rust_down_net="https://mirrors.ustc.edu.cn/rust-static/dist" + pushd ${rust_static_dir} + local platform=$1 + local cpu=$2 + local files=("rust-std-1.71.0-${cpu}-${platform}.tar.xz" "rustc-1.71.0-${cpu}-${platform}.tar.xz" "cargo-1.71.0-${cpu}-${platform}.tar.xz") + + for file in "${files[@]}"; do + curl -O -k -m 300 ${rust_down_net}/${pre_rust_date}/${file} & + done + curl -O -k -m 300 ${rust_down_net}/rustc-1.72.0-src.tar.gz & + wait + popd + cp ${rust_static_dir}/*.tar.xz ${rust_source_dir}/build/cache/${pre_rust_date}/ + cp ${rust_static_dir}/rustc-1.72.0-src.tar.gz ${root_build_dir} +} + +download_rust() { + if [ "${host_platform}" = "linux" ] && [ "${host_cpu}" = "x86_64" ]; then + download_rust_static_source "unknown-linux-gnu" "x86_64" + elif [ "${host_platform}" = "darwin" ]; then + if [ "${host_cpu}" = "x86_64" ]; then + download_rust_static_source "apple-darwin" "x86_64" + else + download_rust_static_source "apple-darwin" "aarch64" + fi + else + echo "Unsupported platform: $(uname -s) $(uname -m)" + exit 1 + fi +} + +get_new_version() { + pushd ${root_build_dir}/../harmony/llvm/ + local commit_id_full=$(git rev-parse HEAD) + local commit_id_short=$(expr substr "$commit_id_full" 1 10) + new_version="OHOS llvm-preject $commit_id_short" + popd +} + +collect_build_result() { + mkdir -p ${output_install} + if [ "${host_platform}" = "linux" ] && [ ${host_cpu} = "x86_64" ]; then + cp ${rust_source_dir}/build/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz ${output_install}/ + cp ${rust_source_dir}/build/dist/rust-std-nightly-x86_64-pc-windows-gnullvm.tar.gz ${output_install}/ + cp ${rust_source_dir}/build/dist/rust-std-nightly-aarch64-unknown-linux-ohos.tar.gz ${output_install}/ + cp ${rust_source_dir}/build/dist/rust-std-nightly-armv7-unknown-linux-ohos.tar.gz ${output_install}/ + cp ${rust_source_dir}/build/dist/rust-std-nightly-x86_64-unknown-linux-ohos.tar.gz ${output_install}/ + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "x86_64" ]; then + cp ${rust_source_dir}/build/dist/rust-nightly-x86_64-apple-darwin.tar.gz ${output_install}/ + elif [ "${host_platform}" = "darwin" ] && [ ${host_cpu} = "arm64" ]; then + cp ${rust_source_dir}/build/dist/rust-nightly-aarch64-apple-darwin.tar.gz ${output_install}/ + fi +} + +main() { + detect_platform + download_rust + copy_config + update_config_clang ${oh_tools} ${mingw_tools} + get_new_version + update_version + move_static_rust_source ${rust_static_dir} ${rust_source_dir} + + rm -rf ${rust_source_dir}/src/llvm-project/* + cp -r ${root_build_dir}/../harmony/llvm/* ${rust_source_dir}/src/llvm-project/ + + pushd ${rust_source_dir} + if [ "${host_platform}" = "linux" ]; then + export PATH=${rust_tools}/cmake/linux-x86/bin:${rust_tools}/clang/ohos/linux-x86_64/llvm/bin:$PATH + fi + python3 ./x.py dist + popd + + collect_build_result + + echo "Building the rust toolchain Completed" +} + +main -- Gitee