From 103df7cc7a3cb0e47abe6eb81dfeb4ca10d2a55a Mon Sep 17 00:00:00 2001 From: cc <18856836718@163.com> Date: Tue, 27 Feb 2024 18:42:35 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4install=5Fdependency,=20downl?= =?UTF-8?q?oad=5Fdependency=E5=B7=A5=E5=85=B7=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 4 + .../BiShengCompiler/check_install_result.sh | 8 + component/BiShengCompiler/install.sh | 33 ++++ .../BiShengJDK17/check_install_result.sh | 10 ++ component/BiShengJDK17/install.sh | 30 ++++ component/BiShengJDK8/check_install_result.sh | 10 ++ component/BiShengJDK8/install.sh | 30 ++++ .../GCCforOpenEuler/check_install_result.sh | 8 + component/GCCforOpenEuler/install.sh | 33 ++++ component/bisheng_compiler/install.sh | 0 component/bisheng_jdk17/install.sh | 0 component/bisheng_jdk8/install.sh | 0 component/gcc_for_openeuler/install.sh | 0 requirements.txt | 5 + tools/download_dependency/src/download.py | 120 ++++++++++++++ .../src/download_config.py | 16 ++ tools/install_dependency/src/command_line.py | 6 +- tools/install_dependency/src/constant.py | 4 +- .../install_dependency/src/devkitpipeline.py | 10 ++ .../src/exception/connect_exception.py | 20 +++ .../src/handler/base_yaml_check.py | 72 ++++++++ .../src/handler/connect_check.py | 47 ++++++ .../src/handler/gather_package.py | 58 +++++++ .../src/handler/handler_and_node.py | 34 ++++ .../src/handler/install_package.py | 57 +++++++ .../src/handler/pipeline.py | 24 +++ .../src/machine/builder_machine.py | 8 + .../src/machine/executor_machine.py | 8 + .../install_dependency/src/machine/machine.py | 155 ++++++++++++++++++ .../src/machine/scanner_machine.py | 8 + 30 files changed, 813 insertions(+), 5 deletions(-) create mode 100644 component/BiShengCompiler/check_install_result.sh create mode 100644 component/BiShengCompiler/install.sh create mode 100644 component/BiShengJDK17/check_install_result.sh create mode 100644 component/BiShengJDK17/install.sh create mode 100644 component/BiShengJDK8/check_install_result.sh create mode 100644 component/BiShengJDK8/install.sh create mode 100644 component/GCCforOpenEuler/check_install_result.sh create mode 100644 component/GCCforOpenEuler/install.sh delete mode 100644 component/bisheng_compiler/install.sh delete mode 100644 component/bisheng_jdk17/install.sh delete mode 100644 component/bisheng_jdk8/install.sh delete mode 100644 component/gcc_for_openeuler/install.sh create mode 100644 requirements.txt create mode 100644 tools/install_dependency/src/exception/connect_exception.py create mode 100644 tools/install_dependency/src/handler/base_yaml_check.py create mode 100644 tools/install_dependency/src/handler/connect_check.py create mode 100644 tools/install_dependency/src/handler/gather_package.py create mode 100644 tools/install_dependency/src/handler/handler_and_node.py create mode 100644 tools/install_dependency/src/handler/install_package.py create mode 100644 tools/install_dependency/src/machine/builder_machine.py create mode 100644 tools/install_dependency/src/machine/executor_machine.py create mode 100644 tools/install_dependency/src/machine/machine.py create mode 100644 tools/install_dependency/src/machine/scanner_machine.py diff --git a/build.sh b/build.sh index 9679080..b6378db 100644 --- a/build.sh +++ b/build.sh @@ -13,3 +13,7 @@ pyinstaller -F ./src/download.py -p ./ cd $current_dir/tools/install_dependency pyinstaller -F ./src/devkitpipeline.py -p ./ + +cp $current_dir/tools/install_dependency/config/machine.yaml $current_dir/tools/install_dependency/dist/machine.yaml +cp -rf $current_dir/component $current_dir/tools/install_dependency/dist/ +cp $current_dir/tools/download_dependency/dist/download $current_dir/tools/install_dependency/dist/ diff --git a/component/BiShengCompiler/check_install_result.sh b/component/BiShengCompiler/check_install_result.sh new file mode 100644 index 0000000..802f67f --- /dev/null +++ b/component/BiShengCompiler/check_install_result.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +clang_path=$(which clang) +if [[ ${clang_path} == ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengCompiler/install.sh b/component/BiShengCompiler/install.sh new file mode 100644 index 0000000..cff4bf5 --- /dev/null +++ b/component/BiShengCompiler/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/BiShengCompiler-3.2.0-aarch64-linux.tar.gz -C ${HOME}/compilers + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/compilers finished." + fi + + clang_path=$(which clang) + if [[ ${clang_path} != ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then + sed -i '/#*export BISHENG_COMPILER_HOME=${HOME}\/compilers/d' ${HOME}/.bashrc + sed -i '/#*export PATH=${BISHENG_COMPILER_HOME}:${PATH}/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' +export BISHENG_COMPILER_HOME=${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin +export PATH=${BISHENG_COMPILER_HOME}:${PATH} +EOF + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc + else + echo "install BiShengCompiler-3.2.0 success." + fi + +else + echo "Failed to verify the signature of the BiShengCompiler-3.2.0-aarch64-linux.tar.gz installation package." +fi \ No newline at end of file diff --git a/component/BiShengJDK17/check_install_result.sh b/component/BiShengJDK17/check_install_result.sh new file mode 100644 index 0000000..7bf7252 --- /dev/null +++ b/component/BiShengJDK17/check_install_result.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#java_path=$(which java) +#if [[ ${java_path} == ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + +if [[ -f ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengJDK17/install.sh b/component/BiShengJDK17/install.sh new file mode 100644 index 0000000..c1ad1b8 --- /dev/null +++ b/component/BiShengJDK17/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/bisheng-jdk-17.0.10 ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/bisheng-jdk-17.0.10-linux-aarch64.tar.gz -C ${HOME}/compilers + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/compilers finished." + fi + + java_path=$(which java) + if [[ ${java_path} != ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + # 配置alternatives + echo "do update-alternatives install something to /usr/bin/" + for BinFilePath in ${HOME}/compilers/bisheng-jdk-17.0.10/bin/*; do + if [[ -x ${BinFilePath} ]]; then + BinFileName=$(basename ${BinFilePath}) + sudo update-alternatives --install /usr/bin/${BinFileName} ${BinFileName} ${BinFilePath} 50 + fi + done + else + echo "install bisheng-jdk-17.0.10 success." + fi + +else + echo "Failed to verify the signature of the bisheng-jdk-17.0.10-linux-aarch64.tar.gz installation package." +fi diff --git a/component/BiShengJDK8/check_install_result.sh b/component/BiShengJDK8/check_install_result.sh new file mode 100644 index 0000000..f3f7e3e --- /dev/null +++ b/component/BiShengJDK8/check_install_result.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#java_path=$(which java) +#if [[ ${java_path} == ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + +if [[ -f ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengJDK8/install.sh b/component/BiShengJDK8/install.sh new file mode 100644 index 0000000..0b4b976 --- /dev/null +++ b/component/BiShengJDK8/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/bisheng-jdk1.8.0_402 ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/bisheng-jdk-8u402-linux-aarch64.tar.gz -C ${HOME}/compilers + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/compilers finished." + fi + + java_path=$(which java) + if [[ ${java_path} != ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + # 配置alternatives + echo "do update-alternatives install something to /usr/bin/" + for BinFilePath in ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/*; do + if [[ -x ${BinFilePath} ]]; then + BinFileName=$(basename ${BinFilePath}) + sudo update-alternatives --install /usr/bin/${BinFileName} ${BinFileName} ${BinFilePath} 100 + fi + done + else + echo "install bisheng-jdk-8u402 success." + fi + +else + echo "Failed to verify the signature of the bisheng-jdk-8u402-linux-aarch64.tar.gz installation package." +fi diff --git a/component/GCCforOpenEuler/check_install_result.sh b/component/GCCforOpenEuler/check_install_result.sh new file mode 100644 index 0000000..54c8f21 --- /dev/null +++ b/component/GCCforOpenEuler/check_install_result.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +gcc_path=$(which gcc) +if [[ ${gcc_path} == ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/GCCforOpenEuler/install.sh b/component/GCCforOpenEuler/install.sh new file mode 100644 index 0000000..03bf6a9 --- /dev/null +++ b/component/GCCforOpenEuler/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/gcc-10.3.1-2023.12-aarch64-linux.tar.gz -C ${HOME}/compilers + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/compilers finished." + fi + + gcc_path=$(which gcc) + if [[ ${gcc_path} != ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then + sed -i '/#*export GCC_HOME=${HOME}\/compilers/d' ${HOME}/.bashrc + sed -i '/#*export PATH=${GCC_HOME}:${PATH}/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' +export GCC_HOME=${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin +export PATH=${GCC_HOME}:${PATH} +EOF + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc + else + echo "install gcc-10.3.1-2023.12 success." + fi + +else + echo "Failed to verify the signature of the gcc-10.3.1-2023.12-aarch64-linux.tar.gz installation package." +fi \ No newline at end of file diff --git a/component/bisheng_compiler/install.sh b/component/bisheng_compiler/install.sh deleted file mode 100644 index e69de29..0000000 diff --git a/component/bisheng_jdk17/install.sh b/component/bisheng_jdk17/install.sh deleted file mode 100644 index e69de29..0000000 diff --git a/component/bisheng_jdk8/install.sh b/component/bisheng_jdk8/install.sh deleted file mode 100644 index e69de29..0000000 diff --git a/component/gcc_for_openeuler/install.sh b/component/gcc_for_openeuler/install.sh deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..47592ab --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pyinstaller +paramiko +PyYAML +wget +timeout_decorator \ No newline at end of file diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index e69de29..b675de3 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -0,0 +1,120 @@ +import os +import sys +import shutil +import tarfile +import urllib.error +import wget + +from download_config import BiShengCompiler, GCCforOpenEuler, BiShengJDK8, BiShengJDK17 + +FILE = "file" +SHA256 = "sha256" +URL = "url" +SAVE_PATH = "save_path" +DEFAULT_PATH = "./devkitdependencies" +DEPENDENCY_FILE = "devkitdependencies.tar.gz" + +component_collection_map = { + "BiShengCompiler": { + f"wget -c {BiShengCompiler.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengCompiler.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengCompiler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengCompiler.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengCompiler.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengCompiler.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengCompiler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengCompiler.get(SHA256).split('/')[-1])}" + }, + }, + + "GCCforOpenEuler": { + f"wget -c {GCCforOpenEuler.get(FILE)} -O {os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(FILE).split('/')[-1])}": + { + URL: f"{GCCforOpenEuler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(FILE).split('/')[-1])}" + }, + f"wget -c {GCCforOpenEuler.get(SHA256)} -O {os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(SHA256).split('/')[-1])}": + { + URL: f"{GCCforOpenEuler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(SHA256).split('/')[-1])}" + } + }, + + "BiShengJDK8": { + f"wget -c {BiShengJDK8.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengJDK8.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengJDK8.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK8.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengJDK8.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengJDK8.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengJDK8.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK8.get(SHA256).split('/')[-1])}" + } + }, + + "BiShengJDK17": { + f"wget -c {BiShengJDK17.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengJDK17.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengJDK17.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK17.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengJDK17.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengJDK17.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengJDK17.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK17.get(SHA256).split('/')[-1])}" + } + }, +} + + +def download_dependence(): + if not os.path.exists(DEFAULT_PATH): + os.mkdir(DEFAULT_PATH) + elif os.path.isfile(DEFAULT_PATH): + print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") + return False + else: + pass + + ret = True + for component_name in component_collection_map: + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + try: + print(f"Downloading from {url_and_save_path.get(URL)}") + download_result = wget.download( + url_and_save_path.get(URL), url_and_save_path.get(SAVE_PATH) + ) + print() + except (TimeoutError, urllib.error.URLError, OSError) as e: + print(f"[ERROR] download error occurs: {str(e)}") + return False + + if not os.path.isfile(download_result): + print(f"[ERROR] Download dependencies failed. " + f"Please visit following url and download dependencies to default directory." + f"\n\t{url_and_save_path.get(URL)}" + ) + ret = False + return ret + + +if __name__ == '__main__': + try: + ret = download_dependence() + if ret: + print(f"Now compress dependencies to {DEPENDENCY_FILE}...") + with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: + tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) + + print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") + shutil.rmtree(DEFAULT_PATH) + print("-- Delete dependencies directory. --") + else: + print("-- Download dependencies failed. Please try execute download tool again. --") + except Exception as e: + print(f"Download dependencies failed. {str(e)} Please try execute download tool again.") + sys.exit(1) diff --git a/tools/download_dependency/src/download_config.py b/tools/download_dependency/src/download_config.py index e69de29..b170a53 100644 --- a/tools/download_dependency/src/download_config.py +++ b/tools/download_dependency/src/download_config.py @@ -0,0 +1,16 @@ +BiShengCompiler = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256", +} +GCCforOpenEuler = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256", +} +BiShengJDK8 = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256", +} +BiShengJDK17 = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256", +} diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/command_line.py index 6d62b97..177bdd9 100644 --- a/tools/install_dependency/src/command_line.py +++ b/tools/install_dependency/src/command_line.py @@ -1,6 +1,6 @@ import argparse -DEFAULT_YAML_PATH = './machine.yaml' +DEFAULT_YAML_PATH = "./machine.yaml" class CommandLine: @@ -8,7 +8,7 @@ class CommandLine: debug = False @classmethod - def add_option(cls, parser): + def add_options(cls, parser): parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, help="Assign yaml config file path.") parser.add_argument("--debug", action="store_true", dest="debug", default=False, help="Open debug log.") @@ -23,7 +23,7 @@ class CommandLine: def process_command_line(program, description, class_list): parser = argparse.ArgumentParser(prog=program, description=description, add_help=True) for klass in class_list: - klass.add_option(parser) + klass.add_options(parser) args = parser.parse_args() for klass in class_list: diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index 840d428..49283cb 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -5,5 +5,5 @@ SCANNER = "scanner" BUILDER = "builder" EXECUTOR = "executor" MACHINE = "machine" -DEPENDENCE_FILE = "devkitdependencies.tar.gz" -DEPENDENCY_DIR = "devkitdependencies" \ No newline at end of file +DEPENDENCY_FILE = "devkitdependencies.tar.gz" +DEPENDENCY_DIR = "devkitdependencies" diff --git a/tools/install_dependency/src/devkitpipeline.py b/tools/install_dependency/src/devkitpipeline.py index 712b07f..57305cf 100644 --- a/tools/install_dependency/src/devkitpipeline.py +++ b/tools/install_dependency/src/devkitpipeline.py @@ -5,7 +5,14 @@ import yaml from log import config_logging from command_line import process_command_line, CommandLine +from handler.pipeline import PipeLine +from handler.base_yaml_check import BaseCheck +from handler.connect_check import ConnectCheck +from handler.gather_package import GatherPackage +from handler.install_package import InstallPackage + LOGGER = logging.getLogger("install_dependency") +PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] def read_yaml_file(yaml_path): @@ -32,5 +39,8 @@ if __name__ == '__main__': config_dict = read_yaml_file(CommandLine.yaml_path) LOGGER.debug(f"-- config_dict: {config_dict}") + pipe = PipeLine(config_dict) + pipe.add_tail(*PIPELINE) + pipe.start() except (KeyboardInterrupt, Exception) as e: print(f"[warning] Program Exited. {str(e)}") diff --git a/tools/install_dependency/src/exception/connect_exception.py b/tools/install_dependency/src/exception/connect_exception.py new file mode 100644 index 0000000..96fd3af --- /dev/null +++ b/tools/install_dependency/src/exception/connect_exception.py @@ -0,0 +1,20 @@ +class ConnectException(Exception): + def __init__(self): + super(ConnectException, self).__init__() + self.status = "" + self.value = "" + + +class ConnectRemoteException(ConnectException): + def __init__(self): + super(ConnectRemoteException, self).__init__() + + +class CreatePkeyFailedException(ConnectException): + def __init__(self): + super(CreatePkeyFailedException, self).__init__() + + +class NotMatchedMachineTypeException(ConnectException): + def __init__(self): + super(NotMatchedMachineTypeException, self).__init__() diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py new file mode 100644 index 0000000..7b07d22 --- /dev/null +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -0,0 +1,72 @@ +import os +import re +import logging +import constant +from handler.handler_and_node import Handler + +LOGGER = logging.getLogger("install_dependency") +MIN_SET = (constant.USER, constant.PKEY, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) +MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) + + +class BaseCheck(Handler): + IPV4_REG = r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" + + def handle(self, data) -> bool: + LOGGER.debug("BaseCheck start!") + key_set = set(data.keys()) + if not key_set.issuperset(MIN_SET) or not key_set.issubset(MAX_SET): + LOGGER.error("Yaml file content not correct. Wrong yaml mappings.") + return False + + if not BaseCheck.check_user(data): + return False + if not BaseCheck.check_pkey(data): + return False + if not BaseCheck.check_machine_ip(data): + return False + + LOGGER.debug(f"After Base Check, data: {data}") + return True + + @staticmethod + def check_user(data): + user_name = data.get(constant.USER, "") + if not user_name: + LOGGER.error("Yaml file content not correct. Empty user name.") + return False + return True + + @staticmethod + def check_pkey(data): + pkey_path = data.get(constant.PKEY, "") + if not pkey_path: + LOGGER.error("Yaml file content not correct. Empty pkey.") + return False + + if not BaseCheck.validate_path(pkey_path) or not os.path.isfile(pkey_path): + LOGGER.error("Yaml file content not correct. Given pkey not exists.") + return False + return True + + @staticmethod + def validate_path(path: str) -> bool: + return path.startswith('/') and path.find('../') == -1 and path.find('./') == -1 + + @staticmethod + def check_machine_ip(data): + machine_type_list = [constant.SCANNER, constant.BUILDER, constant.EXECUTOR] + for machine_type in machine_type_list: + if not data.get(machine_type) or not isinstance(data.get(machine_type), list): + LOGGER.error(f"Yaml file content not correct. Yaml file {machine_type} value not sequence.") + return False + for ip in data.get(machine_type): + if not BaseCheck.validate_ip(ip): + LOGGER.error(f"Yaml file content not correct. Given ip: {ip} not correct.") + return False + return True + + @staticmethod + def validate_ip(ip_address: str): + return re.match(BaseCheck.IPV4_REG, ip_address) + diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py new file mode 100644 index 0000000..8bc6c7a --- /dev/null +++ b/tools/install_dependency/src/handler/connect_check.py @@ -0,0 +1,47 @@ +import sys +import os + +import logging +import constant +from handler.handler_and_node import Handler +from machine.scanner_machine import ScannerMachine +from machine.builder_machine import BuilderMachine +from machine.executor_machine import ExecutorMachine +from exception.connect_exception import ConnectException + +LOGGER = logging.getLogger("install_dependency") + + +class ConnectCheck(Handler): + klass_dict = { + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, + constant.EXECUTOR: ExecutorMachine, + } + + def handle(self, data) -> bool: + LOGGER.debug("ConnectCheck start!") + ret = True + for role in ConnectCheck.klass_dict: + ret = ret and ConnectCheck.machine_role_check(data, role) + return ret + + @staticmethod + def machine_role_check(data, role): + builder_list = data.get(role) + klass = ConnectCheck.klass_dict.get(role) + data[role + constant.MACHINE] = dict() + for ip in builder_list: + try: + machine_instance = klass(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None)) + data[role + constant.MACHINE][ip] = machine_instance + except ConnectException: + LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") + del data[role + constant.MACHINE] + return False + except Exception as e: + LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") + del data[role + constant.MACHINE] + return False + return True diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py new file mode 100644 index 0000000..6e616a3 --- /dev/null +++ b/tools/install_dependency/src/handler/gather_package.py @@ -0,0 +1,58 @@ +import logging +import os +import subprocess +import constant +from download import download_dependence, component_collection_map +from handler.handler_and_node import Handler + +LOGGER = logging.getLogger("install_dependency") + + +class GatherPackage(Handler): + def handle(self, data) -> bool: + LOGGER.debug("GatherPackage start!") + if GatherPackage.check_default_path_available(): + LOGGER.info("Dependencies ready.") + return True + + if os.path.isfile(constant.DEPENDENCY_DIR): + LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + return False + + try: + ret = download_dependence() + except Exception as e: + LOGGER.error(f"Download dependencies failed. {str(e)}. Please execute download tool.") + return False + if not ret: + LOGGER.error("Download dependencies failed. Please execute download tool.") + return False + LOGGER.info("Download dependencies success.") + return True + + @staticmethod + def check_default_path_available(): + if os.path.exists(constant.DEPENDENCY_FILE): + try: + print(f"Now extract files from {constant.DEPENDENCY_FILE}:") + result = subprocess.run(f"tar -zxvf {constant.DEPENDENCY_FILE}".split(' '), + capture_output=False, shell=False, stderr=subprocess.STDOUT) + print(f"{result.stdout}" if result.stdout else "") + except (FileExistsError, ) as e: + LOGGER.warning(f"{constant.DEPENDENCY_FILE} may already extracted.") + except Exception as e: + LOGGER.error(f"Extract {constant.DEPENDENCY_FILE} failed. {str(e)}") + return False + + if not os.path.isdir(constant.DEPENDENCY_DIR): + LOGGER.warning(f"The directory {constant.DEPENDENCY_DIR} not exists.") + return False + for component_name in component_collection_map: + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + component = url_and_save_path.get("save_path") + if not os.path.isfile(component): + LOGGER.warning(f"The file {component} not exists.") + return False + return True diff --git a/tools/install_dependency/src/handler/handler_and_node.py b/tools/install_dependency/src/handler/handler_and_node.py new file mode 100644 index 0000000..5d5f3dc --- /dev/null +++ b/tools/install_dependency/src/handler/handler_and_node.py @@ -0,0 +1,34 @@ +from abc import abstractmethod + + +class Handler: + """处理器基类""" + + def __init__(self): + pass + + @abstractmethod + def handle(self, data) -> bool: + pass + + +class Node: + """链表节点""" + + def __init__(self, handler=None): + self.handler: Handler = handler + self.next_node: Node = None + + def get_next_node(self): + return self.next_node + + def set_next_node(self, node): + self.next_node = node + + def execute(self, data): + ret: bool = self.handler.handle(data) + if not ret: + return False + if self.next_node: + return self.next_node.execute(data) + return True diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py new file mode 100644 index 0000000..1444f73 --- /dev/null +++ b/tools/install_dependency/src/handler/install_package.py @@ -0,0 +1,57 @@ +import logging +import multiprocessing + +import constant +from handler.handler_and_node import Handler +from machine.machine import Machine +from machine.scanner_machine import ScannerMachine +from machine.builder_machine import BuilderMachine +from machine.executor_machine import ExecutorMachine + +LOGGER = logging.getLogger("install_dependency") + + +class InstallPackage(Handler): + klass_dict = { + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, + constant.EXECUTOR: ExecutorMachine, + } + + def handle(self, data) -> bool: + LOGGER.debug("Install Package start!") + ip_set = set() + jobs = [] + + for role in InstallPackage.klass_dict: + machine_dict = data[role + constant.MACHINE] + LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") + for machine_ip in machine_dict: + if machine_ip in ip_set: + continue + ip_set.add(machine_ip) + LOGGER.debug(f"ip_set: {ip_set}") + machine = machine_dict.get(machine_ip) + process = multiprocessing.Process( + target=process_work, + args=(machine, + "GCCforOpenEuler", + "BiShengCompiler", + "BiShengJDK17", + "BiShengJDK8", + ), + ) + jobs.append(process) + process.start() + + for job in jobs: + job.join() + return True + + +def process_work(machine: Machine, *components: str): + try: + for component in components: + machine.install_component(component) + except (OSError, IOError) as e: + LOGGER.error(f"Remote machine {machine.ip} occur Error: {str(e)}") diff --git a/tools/install_dependency/src/handler/pipeline.py b/tools/install_dependency/src/handler/pipeline.py index e69de29..d1db8d0 100644 --- a/tools/install_dependency/src/handler/pipeline.py +++ b/tools/install_dependency/src/handler/pipeline.py @@ -0,0 +1,24 @@ +import logging + +from handler.handler_and_node import Node + +LOGGER = logging.getLogger("install_dependency") + + +class PipeLine: + """维护一个链表""" + + def __init__(self, data): + self.head: Node = Node() + self.tail: Node = self.head + self.data = data + + def start(self): + if self.head.get_next_node() and self.head.get_next_node().execute(self.data): + print("-- Congratulations! Install package success in all machines. --") + + def add_tail(self, *handlers): + for handler in handlers: + node = Node(handler) + self.tail.set_next_node(node) + self.tail = node diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py new file mode 100644 index 0000000..d10db3e --- /dev/null +++ b/tools/install_dependency/src/machine/builder_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class BuilderMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(BuilderMachine, self).__init__(ip, user, pkey, password) + self.role = constant.BUILDER \ No newline at end of file diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py new file mode 100644 index 0000000..d0db47b --- /dev/null +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class ExecutorMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(ExecutorMachine, self).__init__(ip, user, pkey, password) + self.role = constant.EXECUTOR \ No newline at end of file diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py new file mode 100644 index 0000000..73e31ac --- /dev/null +++ b/tools/install_dependency/src/machine/machine.py @@ -0,0 +1,155 @@ +import os +import paramiko +import socket +import logging +import timeout_decorator + +import constant +from exception.connect_exception import CreatePkeyFailedException, ConnectRemoteException, \ + NotMatchedMachineTypeException +from download import component_collection_map + +LOGGER = logging.getLogger("install_dependency") + + +class Machine: + def __init__(self, ip, user, pkey, password=None): + self.ip = ip + self.user = user + self.pkey = pkey + self.password = password + self.check_is_aarch64() + + def check_is_aarch64(self): + machine_type = self.get_machine_type() + LOGGER.info(f"{self.ip} machine type: {machine_type}") + if machine_type != "aarch64": + LOGGER.error(f"Machine type of {self.ip} is {machine_type}, not aarch64. Please replace this machine.") + raise NotMatchedMachineTypeException() + + def get_machine_type(self): + try: + ssh_client = self.ssh_client() + stdin, stdout, stderr = ssh_client.exec_command("uname -m", timeout=10) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + stdout_output = stdout.read().decode().strip() + ssh_client.close() + return stdout_output + + def ssh_client(self): + ssh = paramiko.SSHClient() + ssh._transport = self.transport_connect(self.user, self.pkey, self.password) + return ssh + + def sftp_client(self): + sftp = paramiko.SFTPClient.from_transport(self.transport_connect(self.user, self.pkey, self.password)) + return sftp + + def transport_connect(self, user, pkey_path, password=None): + try: + # 指定本地的RSA私钥文件。如果建立密钥对时设置的有密码，password为设定的密码，如无不用指定password参数 + pkey = paramiko.RSAKey.from_private_key_file(pkey_path, password=password) + except (IOError,) as e: + LOGGER.error(f"Pkey file not exists. {str(e)}") + raise CreatePkeyFailedException() + except (paramiko.ssh_exception.PasswordRequiredException, paramiko.ssh_exception.AuthenticationException) as e: + LOGGER.warning(f"Pkey password is required. {str(e)}") + password = input(f"Press Enter to input password of {pkey_path}: ") + self.password = password + return self.transport_connect(user, pkey_path, password) + except (paramiko.ssh_exception.SSHException,) as e: + LOGGER.error(f"Connect remote {self.ip} failed because of wrong pkey. {str(e)}") + raise CreatePkeyFailedException() + + transport = paramiko.Transport((self.ip, 22)) + + try: + self.transport_connect_with_timeout(transport, user, pkey) + except (paramiko.ssh_exception.AuthenticationException, + paramiko.ssh_exception.SSHException, + timeout_decorator.TimeoutError, + socket.gaierror, + socket.timeout, + socket.error) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + return transport + + @timeout_decorator.timeout(10) + def transport_connect_with_timeout(self, transport, user, pkey): + transport.connect(username=user, pkey=pkey) + + def install_component(self, component_name): + ssh_client = self.ssh_client() + sftp_client = self.sftp_client() + try: + self.install_component_handler(component_name, sftp_client, ssh_client) + except (FileNotFoundError, PermissionError, NotADirectoryError, OSError, IOError) as e: + LOGGER.error(f"Remote machine {self.ip} occur Error: {str(e)}") + finally: + ssh_client.close() + sftp_client.close() + + def install_component_handler(self, component_name, sftp_client, ssh_client): + try: + stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + exit_status = stdout.channel.recv_exit_status() + LOGGER.debug(f"Remote machine {self.ip} mkdir -p /tmp/{constant.DEPENDENCY_DIR} result: " + f"{'success' if not exit_status else 'failed'}") + if exit_status: + raise NotADirectoryError(f"Remote machine {self.ip} " + f"directory {os.path.join('/tmp/', constant.DEPENDENCY_DIR)} not exist.") + + # 上传 组件压缩包和校验文件 + LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") + remote_file_list = [] + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + component = url_and_save_path.get("save_path") + LOGGER.debug(f"Transport component file to remote machine {self.ip}: {component}") + remote_file = os.path.abspath(os.path.join('/tmp', component)) + remote_file_list.append(remote_file) + sftp_client.put(localpath=f"{component}", remotepath=f"{remote_file}") + + # 上传并执行 安装脚本, 校验安装结果脚本 + shell_file_list = ["install.sh", "check_install_result.sh"] + install_result = "" + for shell_file in shell_file_list: + execute_output, sh_file_remote_path = ( + self.transport_shell_file_and_execute(ssh_client, sftp_client, component_name, shell_file)) + remote_file_list.append(sh_file_remote_path) + if shell_file == shell_file_list[1]: + install_result = execute_output + + if install_result == "true": + LOGGER.info(f"Remote machine {self.ip} install {component_name} success.") + else: + LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") + # 清理tmp临时文件 + self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) + + def transport_shell_file_and_execute(self, ssh_client, sftp_client, component_name, shell_file): + sh_file_local_path = os.path.join("./component", component_name, shell_file) + if not os.path.exists(sh_file_local_path): + LOGGER.error(f"{sh_file_local_path} not exists.") + raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + LOGGER.debug(f"Transport local_file: {sh_file_local_path} to remote machine {self.ip} " + f"remote_file: {sh_file_remote_path}") + sftp_client.put(localpath=sh_file_local_path, remotepath=sh_file_remote_path) + stdin, stdout, stderr = ssh_client.exec_command(f"bash {sh_file_remote_path}") + output = stdout.read().decode().strip() + LOGGER.info(f"Remote machine {self.ip} bash {component_name}{shell_file} file output: {output}") + return output, sh_file_remote_path + + def clear_tmp_file_at_remote_machine(self, ssh_client, remote_file_list): + LOGGER.debug(f"Clear tmp file at remote machine {self.ip}") + for remote_file in remote_file_list: + LOGGER.debug(f"Delete tmp file at remote machine {self.ip}: {remote_file}") + ssh_client.exec_command(f"rm -f {remote_file}") diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py new file mode 100644 index 0000000..07daec8 --- /dev/null +++ b/tools/install_dependency/src/machine/scanner_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class ScannerMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(ScannerMachine, self).__init__(ip, user, pkey, password) + self.role = constant.SCANNER \ No newline at end of file -- Gitee