diff --git a/build.sh b/build.sh index 96790801bbde729300e7c937a6839e13cb69a9d4..b6378db31da5ff9fe33fb5dd473d63f36b98e5ca 100644 --- a/build.sh +++ b/build.sh @@ -13,3 +13,7 @@ pyinstaller -F ./src/download.py -p ./ cd $current_dir/tools/install_dependency pyinstaller -F ./src/devkitpipeline.py -p ./ + +cp $current_dir/tools/install_dependency/config/machine.yaml $current_dir/tools/install_dependency/dist/machine.yaml +cp -rf $current_dir/component $current_dir/tools/install_dependency/dist/ +cp $current_dir/tools/download_dependency/dist/download $current_dir/tools/install_dependency/dist/ diff --git a/component/BiShengCompiler/check_install_result.sh b/component/BiShengCompiler/check_install_result.sh new file mode 100644 index 0000000000000000000000000000000000000000..802f67fe5a646709dcea4e7a4db9c764270ef282 --- /dev/null +++ b/component/BiShengCompiler/check_install_result.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +clang_path=$(which clang) +if [[ ${clang_path} == ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengCompiler/install.sh b/component/BiShengCompiler/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..cff4bf596c65adbe194755e86ce0fc01d08cb511 --- /dev/null +++ b/component/BiShengCompiler/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/BiShengCompiler-3.2.0-aarch64-linux.tar.gz -C ${HOME}/compilers + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/compilers finished." + fi + + clang_path=$(which clang) + if [[ ${clang_path} != ${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then + sed -i '/#*export BISHENG_COMPILER_HOME=${HOME}\/compilers/d' ${HOME}/.bashrc + sed -i '/#*export PATH=${BISHENG_COMPILER_HOME}:${PATH}/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' +export BISHENG_COMPILER_HOME=${HOME}/compilers/BiShengCompiler-3.2.0-aarch64-linux/bin +export PATH=${BISHENG_COMPILER_HOME}:${PATH} +EOF + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc + else + echo "install BiShengCompiler-3.2.0 success." + fi + +else + echo "Failed to verify the signature of the BiShengCompiler-3.2.0-aarch64-linux.tar.gz installation package." +fi \ No newline at end of file diff --git a/component/BiShengJDK17/check_install_result.sh b/component/BiShengJDK17/check_install_result.sh new file mode 100644 index 0000000000000000000000000000000000000000..7bf72523c0d465b93a8fcfdbb8fc54f4c22e3c82 --- /dev/null +++ b/component/BiShengJDK17/check_install_result.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#java_path=$(which java) +#if [[ ${java_path} == ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + +if [[ -f ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengJDK17/install.sh b/component/BiShengJDK17/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..c1ad1b866a8d035f8f7da27386326e65b2b89d2e --- /dev/null +++ b/component/BiShengJDK17/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/bisheng-jdk-17.0.10 ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/bisheng-jdk-17.0.10-linux-aarch64.tar.gz -C ${HOME}/compilers + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/compilers finished." + fi + + java_path=$(which java) + if [[ ${java_path} != ${HOME}/compilers/bisheng-jdk-17.0.10/bin/java ]]; then + # 配置alternatives + echo "do update-alternatives install something to /usr/bin/" + for BinFilePath in ${HOME}/compilers/bisheng-jdk-17.0.10/bin/*; do + if [[ -x ${BinFilePath} ]]; then + BinFileName=$(basename ${BinFilePath}) + sudo update-alternatives --install /usr/bin/${BinFileName} ${BinFileName} ${BinFilePath} 50 + fi + done + else + echo "install bisheng-jdk-17.0.10 success." + fi + +else + echo "Failed to verify the signature of the bisheng-jdk-17.0.10-linux-aarch64.tar.gz installation package." +fi diff --git a/component/BiShengJDK8/check_install_result.sh b/component/BiShengJDK8/check_install_result.sh new file mode 100644 index 0000000000000000000000000000000000000000..f3f7e3ee3438620528323e3ddd4111c8f46e1a51 --- /dev/null +++ b/component/BiShengJDK8/check_install_result.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#java_path=$(which java) +#if [[ ${java_path} == ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + +if [[ -f ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/BiShengJDK8/install.sh b/component/BiShengJDK8/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..0b4b976d8141c9ce7646f486b518805dbcaeed38 --- /dev/null +++ b/component/BiShengJDK8/install.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/bisheng-jdk1.8.0_402 ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/bisheng-jdk-8u402-linux-aarch64.tar.gz -C ${HOME}/compilers + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/compilers finished." + fi + + java_path=$(which java) + if [[ ${java_path} != ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/java ]]; then + # 配置alternatives + echo "do update-alternatives install something to /usr/bin/" + for BinFilePath in ${HOME}/compilers/bisheng-jdk1.8.0_402/bin/*; do + if [[ -x ${BinFilePath} ]]; then + BinFileName=$(basename ${BinFilePath}) + sudo update-alternatives --install /usr/bin/${BinFileName} ${BinFileName} ${BinFilePath} 100 + fi + done + else + echo "install bisheng-jdk-8u402 success." + fi + +else + echo "Failed to verify the signature of the bisheng-jdk-8u402-linux-aarch64.tar.gz installation package." +fi diff --git a/component/GCCforOpenEuler/check_install_result.sh b/component/GCCforOpenEuler/check_install_result.sh new file mode 100644 index 0000000000000000000000000000000000000000..54c8f219791967a3d8ebef8ec5c849f1fa4499ad --- /dev/null +++ b/component/GCCforOpenEuler/check_install_result.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +gcc_path=$(which gcc) +if [[ ${gcc_path} == ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then + echo "true" +else + echo "false" +fi diff --git a/component/GCCforOpenEuler/install.sh b/component/GCCforOpenEuler/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..03bf6a9e08836b6534fcdcfb3e9300bab0245776 --- /dev/null +++ b/component/GCCforOpenEuler/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cd /tmp/devkitdependencies/ +verify_signature=$(sha256sum -c gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) +if [[ ${verify_signature} -eq "0" ]]; then + if [[ ! -d ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux ]]; then + mkdir -p ${HOME}/compilers + + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/compilers." + tar -zxf /tmp/devkitdependencies/gcc-10.3.1-2023.12-aarch64-linux.tar.gz -C ${HOME}/compilers + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/compilers finished." + fi + + gcc_path=$(which gcc) + if [[ ${gcc_path} != ${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then + sed -i '/#*export GCC_HOME=${HOME}\/compilers/d' ${HOME}/.bashrc + sed -i '/#*export PATH=${GCC_HOME}:${PATH}/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' +export GCC_HOME=${HOME}/compilers/gcc-10.3.1-2023.12-aarch64-linux/bin +export PATH=${GCC_HOME}:${PATH} +EOF + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc + else + echo "install gcc-10.3.1-2023.12 success." + fi + +else + echo "Failed to verify the signature of the gcc-10.3.1-2023.12-aarch64-linux.tar.gz installation package." +fi \ No newline at end of file diff --git a/component/bisheng_compiler/install.sh b/component/bisheng_compiler/install.sh deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/component/bisheng_jdk17/install.sh b/component/bisheng_jdk17/install.sh deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/component/bisheng_jdk8/install.sh b/component/bisheng_jdk8/install.sh deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/component/gcc_for_openeuler/install.sh b/component/gcc_for_openeuler/install.sh deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..47592aba2dfbb5c7426589dcdd9d1c513ab3a095 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pyinstaller +paramiko +PyYAML +wget +timeout_decorator \ No newline at end of file diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b675de320d8ad9f75d66991c5af24ce9c95022b0 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -0,0 +1,120 @@ +import os +import sys +import shutil +import tarfile +import urllib.error +import wget + +from download_config import BiShengCompiler, GCCforOpenEuler, BiShengJDK8, BiShengJDK17 + +FILE = "file" +SHA256 = "sha256" +URL = "url" +SAVE_PATH = "save_path" +DEFAULT_PATH = "./devkitdependencies" +DEPENDENCY_FILE = "devkitdependencies.tar.gz" + +component_collection_map = { + "BiShengCompiler": { + f"wget -c {BiShengCompiler.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengCompiler.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengCompiler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengCompiler.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengCompiler.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengCompiler.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengCompiler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengCompiler.get(SHA256).split('/')[-1])}" + }, + }, + + "GCCforOpenEuler": { + f"wget -c {GCCforOpenEuler.get(FILE)} -O {os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(FILE).split('/')[-1])}": + { + URL: f"{GCCforOpenEuler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(FILE).split('/')[-1])}" + }, + f"wget -c {GCCforOpenEuler.get(SHA256)} -O {os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(SHA256).split('/')[-1])}": + { + URL: f"{GCCforOpenEuler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, GCCforOpenEuler.get(SHA256).split('/')[-1])}" + } + }, + + "BiShengJDK8": { + f"wget -c {BiShengJDK8.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengJDK8.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengJDK8.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK8.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengJDK8.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengJDK8.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengJDK8.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK8.get(SHA256).split('/')[-1])}" + } + }, + + "BiShengJDK17": { + f"wget -c {BiShengJDK17.get(FILE)} -O {os.path.join(DEFAULT_PATH, BiShengJDK17.get(FILE).split('/')[-1])}": + { + URL: f"{BiShengJDK17.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK17.get(FILE).split('/')[-1])}" + }, + f"wget -c {BiShengJDK17.get(SHA256)} -O {os.path.join(DEFAULT_PATH, BiShengJDK17.get(SHA256).split('/')[-1])}": + { + URL: f"{BiShengJDK17.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, BiShengJDK17.get(SHA256).split('/')[-1])}" + } + }, +} + + +def download_dependence(): + if not os.path.exists(DEFAULT_PATH): + os.mkdir(DEFAULT_PATH) + elif os.path.isfile(DEFAULT_PATH): + print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") + return False + else: + pass + + ret = True + for component_name in component_collection_map: + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + try: + print(f"Downloading from {url_and_save_path.get(URL)}") + download_result = wget.download( + url_and_save_path.get(URL), url_and_save_path.get(SAVE_PATH) + ) + print() + except (TimeoutError, urllib.error.URLError, OSError) as e: + print(f"[ERROR] download error occurs: {str(e)}") + return False + + if not os.path.isfile(download_result): + print(f"[ERROR] Download dependencies failed. " + f"Please visit following url and download dependencies to default directory." + f"\n\t{url_and_save_path.get(URL)}" + ) + ret = False + return ret + + +if __name__ == '__main__': + try: + ret = download_dependence() + if ret: + print(f"Now compress dependencies to {DEPENDENCY_FILE}...") + with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: + tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) + + print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") + shutil.rmtree(DEFAULT_PATH) + print("-- Delete dependencies directory. --") + else: + print("-- Download dependencies failed. Please try execute download tool again. --") + except Exception as e: + print(f"Download dependencies failed. {str(e)} Please try execute download tool again.") + sys.exit(1) diff --git a/tools/download_dependency/src/download_config.py b/tools/download_dependency/src/download_config.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b170a53e146d7383ea374e6babefa91c546c039e 100644 --- a/tools/download_dependency/src/download_config.py +++ b/tools/download_dependency/src/download_config.py @@ -0,0 +1,16 @@ +BiShengCompiler = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256", +} +GCCforOpenEuler = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256", +} +BiShengJDK8 = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256", +} +BiShengJDK17 = { + "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz", + "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256", +} diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/command_line.py index 6d62b97e635146b58c18561dbe580822a59ff7d9..177bdd96fdc477f5760a121acc4dea894bed07e8 100644 --- a/tools/install_dependency/src/command_line.py +++ b/tools/install_dependency/src/command_line.py @@ -1,6 +1,6 @@ import argparse -DEFAULT_YAML_PATH = './machine.yaml' +DEFAULT_YAML_PATH = "./machine.yaml" class CommandLine: @@ -8,7 +8,7 @@ class CommandLine: debug = False @classmethod - def add_option(cls, parser): + def add_options(cls, parser): parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, help="Assign yaml config file path.") parser.add_argument("--debug", action="store_true", dest="debug", default=False, help="Open debug log.") @@ -23,7 +23,7 @@ class CommandLine: def process_command_line(program, description, class_list): parser = argparse.ArgumentParser(prog=program, description=description, add_help=True) for klass in class_list: - klass.add_option(parser) + klass.add_options(parser) args = parser.parse_args() for klass in class_list: diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index 840d428e2942ca7337af2890039cabc750503674..49283cbdc29fe38cbc57a612a53eb01651e5397e 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -5,5 +5,5 @@ SCANNER = "scanner" BUILDER = "builder" EXECUTOR = "executor" MACHINE = "machine" -DEPENDENCE_FILE = "devkitdependencies.tar.gz" -DEPENDENCY_DIR = "devkitdependencies" \ No newline at end of file +DEPENDENCY_FILE = "devkitdependencies.tar.gz" +DEPENDENCY_DIR = "devkitdependencies" diff --git a/tools/install_dependency/src/devkitpipeline.py b/tools/install_dependency/src/devkitpipeline.py index 712b07fcb22e11cff43318f2a3fa89f96af2d56c..57305cf3b63210b4b222ad90571dc593930043bd 100644 --- a/tools/install_dependency/src/devkitpipeline.py +++ b/tools/install_dependency/src/devkitpipeline.py @@ -5,7 +5,14 @@ import yaml from log import config_logging from command_line import process_command_line, CommandLine +from handler.pipeline import PipeLine +from handler.base_yaml_check import BaseCheck +from handler.connect_check import ConnectCheck +from handler.gather_package import GatherPackage +from handler.install_package import InstallPackage + LOGGER = logging.getLogger("install_dependency") +PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] def read_yaml_file(yaml_path): @@ -32,5 +39,8 @@ if __name__ == '__main__': config_dict = read_yaml_file(CommandLine.yaml_path) LOGGER.debug(f"-- config_dict: {config_dict}") + pipe = PipeLine(config_dict) + pipe.add_tail(*PIPELINE) + pipe.start() except (KeyboardInterrupt, Exception) as e: print(f"[warning] Program Exited. {str(e)}") diff --git a/tools/install_dependency/src/exception/connect_exception.py b/tools/install_dependency/src/exception/connect_exception.py new file mode 100644 index 0000000000000000000000000000000000000000..96fd3af99fcbba32f90ebfec4ff02783901c9f80 --- /dev/null +++ b/tools/install_dependency/src/exception/connect_exception.py @@ -0,0 +1,20 @@ +class ConnectException(Exception): + def __init__(self): + super(ConnectException, self).__init__() + self.status = "" + self.value = "" + + +class ConnectRemoteException(ConnectException): + def __init__(self): + super(ConnectRemoteException, self).__init__() + + +class CreatePkeyFailedException(ConnectException): + def __init__(self): + super(CreatePkeyFailedException, self).__init__() + + +class NotMatchedMachineTypeException(ConnectException): + def __init__(self): + super(NotMatchedMachineTypeException, self).__init__() diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py new file mode 100644 index 0000000000000000000000000000000000000000..7b07d226bb2d839e2e4ab11a2da35c9e490123e2 --- /dev/null +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -0,0 +1,72 @@ +import os +import re +import logging +import constant +from handler.handler_and_node import Handler + +LOGGER = logging.getLogger("install_dependency") +MIN_SET = (constant.USER, constant.PKEY, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) +MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) + + +class BaseCheck(Handler): + IPV4_REG = r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$" + + def handle(self, data) -> bool: + LOGGER.debug("BaseCheck start!") + key_set = set(data.keys()) + if not key_set.issuperset(MIN_SET) or not key_set.issubset(MAX_SET): + LOGGER.error("Yaml file content not correct. Wrong yaml mappings.") + return False + + if not BaseCheck.check_user(data): + return False + if not BaseCheck.check_pkey(data): + return False + if not BaseCheck.check_machine_ip(data): + return False + + LOGGER.debug(f"After Base Check, data: {data}") + return True + + @staticmethod + def check_user(data): + user_name = data.get(constant.USER, "") + if not user_name: + LOGGER.error("Yaml file content not correct. Empty user name.") + return False + return True + + @staticmethod + def check_pkey(data): + pkey_path = data.get(constant.PKEY, "") + if not pkey_path: + LOGGER.error("Yaml file content not correct. Empty pkey.") + return False + + if not BaseCheck.validate_path(pkey_path) or not os.path.isfile(pkey_path): + LOGGER.error("Yaml file content not correct. Given pkey not exists.") + return False + return True + + @staticmethod + def validate_path(path: str) -> bool: + return path.startswith('/') and path.find('../') == -1 and path.find('./') == -1 + + @staticmethod + def check_machine_ip(data): + machine_type_list = [constant.SCANNER, constant.BUILDER, constant.EXECUTOR] + for machine_type in machine_type_list: + if not data.get(machine_type) or not isinstance(data.get(machine_type), list): + LOGGER.error(f"Yaml file content not correct. Yaml file {machine_type} value not sequence.") + return False + for ip in data.get(machine_type): + if not BaseCheck.validate_ip(ip): + LOGGER.error(f"Yaml file content not correct. Given ip: {ip} not correct.") + return False + return True + + @staticmethod + def validate_ip(ip_address: str): + return re.match(BaseCheck.IPV4_REG, ip_address) + diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc6c7a40c954324d0bdbbdb96d316295b2039b5 --- /dev/null +++ b/tools/install_dependency/src/handler/connect_check.py @@ -0,0 +1,47 @@ +import sys +import os + +import logging +import constant +from handler.handler_and_node import Handler +from machine.scanner_machine import ScannerMachine +from machine.builder_machine import BuilderMachine +from machine.executor_machine import ExecutorMachine +from exception.connect_exception import ConnectException + +LOGGER = logging.getLogger("install_dependency") + + +class ConnectCheck(Handler): + klass_dict = { + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, + constant.EXECUTOR: ExecutorMachine, + } + + def handle(self, data) -> bool: + LOGGER.debug("ConnectCheck start!") + ret = True + for role in ConnectCheck.klass_dict: + ret = ret and ConnectCheck.machine_role_check(data, role) + return ret + + @staticmethod + def machine_role_check(data, role): + builder_list = data.get(role) + klass = ConnectCheck.klass_dict.get(role) + data[role + constant.MACHINE] = dict() + for ip in builder_list: + try: + machine_instance = klass(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None)) + data[role + constant.MACHINE][ip] = machine_instance + except ConnectException: + LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") + del data[role + constant.MACHINE] + return False + except Exception as e: + LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") + del data[role + constant.MACHINE] + return False + return True diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py new file mode 100644 index 0000000000000000000000000000000000000000..6e616a3fbdf28b53532435640b530f431091cf9f --- /dev/null +++ b/tools/install_dependency/src/handler/gather_package.py @@ -0,0 +1,58 @@ +import logging +import os +import subprocess +import constant +from download import download_dependence, component_collection_map +from handler.handler_and_node import Handler + +LOGGER = logging.getLogger("install_dependency") + + +class GatherPackage(Handler): + def handle(self, data) -> bool: + LOGGER.debug("GatherPackage start!") + if GatherPackage.check_default_path_available(): + LOGGER.info("Dependencies ready.") + return True + + if os.path.isfile(constant.DEPENDENCY_DIR): + LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + return False + + try: + ret = download_dependence() + except Exception as e: + LOGGER.error(f"Download dependencies failed. {str(e)}. Please execute download tool.") + return False + if not ret: + LOGGER.error("Download dependencies failed. Please execute download tool.") + return False + LOGGER.info("Download dependencies success.") + return True + + @staticmethod + def check_default_path_available(): + if os.path.exists(constant.DEPENDENCY_FILE): + try: + print(f"Now extract files from {constant.DEPENDENCY_FILE}:") + result = subprocess.run(f"tar -zxvf {constant.DEPENDENCY_FILE}".split(' '), + capture_output=False, shell=False, stderr=subprocess.STDOUT) + print(f"{result.stdout}" if result.stdout else "") + except (FileExistsError, ) as e: + LOGGER.warning(f"{constant.DEPENDENCY_FILE} may already extracted.") + except Exception as e: + LOGGER.error(f"Extract {constant.DEPENDENCY_FILE} failed. {str(e)}") + return False + + if not os.path.isdir(constant.DEPENDENCY_DIR): + LOGGER.warning(f"The directory {constant.DEPENDENCY_DIR} not exists.") + return False + for component_name in component_collection_map: + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + component = url_and_save_path.get("save_path") + if not os.path.isfile(component): + LOGGER.warning(f"The file {component} not exists.") + return False + return True diff --git a/tools/install_dependency/src/handler/handler_and_node.py b/tools/install_dependency/src/handler/handler_and_node.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5f3dc9fb2da1536bc7198d13b55b5fbafeae0b --- /dev/null +++ b/tools/install_dependency/src/handler/handler_and_node.py @@ -0,0 +1,34 @@ +from abc import abstractmethod + + +class Handler: + """处理器基类""" + + def __init__(self): + pass + + @abstractmethod + def handle(self, data) -> bool: + pass + + +class Node: + """链表节点""" + + def __init__(self, handler=None): + self.handler: Handler = handler + self.next_node: Node = None + + def get_next_node(self): + return self.next_node + + def set_next_node(self, node): + self.next_node = node + + def execute(self, data): + ret: bool = self.handler.handle(data) + if not ret: + return False + if self.next_node: + return self.next_node.execute(data) + return True diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py new file mode 100644 index 0000000000000000000000000000000000000000..1444f73f58bced3df8ef59b03522156e45a5ee15 --- /dev/null +++ b/tools/install_dependency/src/handler/install_package.py @@ -0,0 +1,57 @@ +import logging +import multiprocessing + +import constant +from handler.handler_and_node import Handler +from machine.machine import Machine +from machine.scanner_machine import ScannerMachine +from machine.builder_machine import BuilderMachine +from machine.executor_machine import ExecutorMachine + +LOGGER = logging.getLogger("install_dependency") + + +class InstallPackage(Handler): + klass_dict = { + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, + constant.EXECUTOR: ExecutorMachine, + } + + def handle(self, data) -> bool: + LOGGER.debug("Install Package start!") + ip_set = set() + jobs = [] + + for role in InstallPackage.klass_dict: + machine_dict = data[role + constant.MACHINE] + LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") + for machine_ip in machine_dict: + if machine_ip in ip_set: + continue + ip_set.add(machine_ip) + LOGGER.debug(f"ip_set: {ip_set}") + machine = machine_dict.get(machine_ip) + process = multiprocessing.Process( + target=process_work, + args=(machine, + "GCCforOpenEuler", + "BiShengCompiler", + "BiShengJDK17", + "BiShengJDK8", + ), + ) + jobs.append(process) + process.start() + + for job in jobs: + job.join() + return True + + +def process_work(machine: Machine, *components: str): + try: + for component in components: + machine.install_component(component) + except (OSError, IOError) as e: + LOGGER.error(f"Remote machine {machine.ip} occur Error: {str(e)}") diff --git a/tools/install_dependency/src/handler/pipeline.py b/tools/install_dependency/src/handler/pipeline.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d1db8d034f52a53d49fd812cb6c3024adca6091f 100644 --- a/tools/install_dependency/src/handler/pipeline.py +++ b/tools/install_dependency/src/handler/pipeline.py @@ -0,0 +1,24 @@ +import logging + +from handler.handler_and_node import Node + +LOGGER = logging.getLogger("install_dependency") + + +class PipeLine: + """维护一个链表""" + + def __init__(self, data): + self.head: Node = Node() + self.tail: Node = self.head + self.data = data + + def start(self): + if self.head.get_next_node() and self.head.get_next_node().execute(self.data): + print("-- Congratulations! Install package success in all machines. --") + + def add_tail(self, *handlers): + for handler in handlers: + node = Node(handler) + self.tail.set_next_node(node) + self.tail = node diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py new file mode 100644 index 0000000000000000000000000000000000000000..d10db3ee966bfc902c2dd849fac0dbfddf5e8f58 --- /dev/null +++ b/tools/install_dependency/src/machine/builder_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class BuilderMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(BuilderMachine, self).__init__(ip, user, pkey, password) + self.role = constant.BUILDER \ No newline at end of file diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py new file mode 100644 index 0000000000000000000000000000000000000000..d0db47b8c0b464a95a0f59e8047daa351db7a9bf --- /dev/null +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class ExecutorMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(ExecutorMachine, self).__init__(ip, user, pkey, password) + self.role = constant.EXECUTOR \ No newline at end of file diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py new file mode 100644 index 0000000000000000000000000000000000000000..73e31ac07d57b7a23499500185bb3e741f442450 --- /dev/null +++ b/tools/install_dependency/src/machine/machine.py @@ -0,0 +1,155 @@ +import os +import paramiko +import socket +import logging +import timeout_decorator + +import constant +from exception.connect_exception import CreatePkeyFailedException, ConnectRemoteException, \ + NotMatchedMachineTypeException +from download import component_collection_map + +LOGGER = logging.getLogger("install_dependency") + + +class Machine: + def __init__(self, ip, user, pkey, password=None): + self.ip = ip + self.user = user + self.pkey = pkey + self.password = password + self.check_is_aarch64() + + def check_is_aarch64(self): + machine_type = self.get_machine_type() + LOGGER.info(f"{self.ip} machine type: {machine_type}") + if machine_type != "aarch64": + LOGGER.error(f"Machine type of {self.ip} is {machine_type}, not aarch64. Please replace this machine.") + raise NotMatchedMachineTypeException() + + def get_machine_type(self): + try: + ssh_client = self.ssh_client() + stdin, stdout, stderr = ssh_client.exec_command("uname -m", timeout=10) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + stdout_output = stdout.read().decode().strip() + ssh_client.close() + return stdout_output + + def ssh_client(self): + ssh = paramiko.SSHClient() + ssh._transport = self.transport_connect(self.user, self.pkey, self.password) + return ssh + + def sftp_client(self): + sftp = paramiko.SFTPClient.from_transport(self.transport_connect(self.user, self.pkey, self.password)) + return sftp + + def transport_connect(self, user, pkey_path, password=None): + try: + # 指定本地的RSA私钥文件。如果建立密钥对时设置的有密码,password为设定的密码,如无不用指定password参数 + pkey = paramiko.RSAKey.from_private_key_file(pkey_path, password=password) + except (IOError,) as e: + LOGGER.error(f"Pkey file not exists. {str(e)}") + raise CreatePkeyFailedException() + except (paramiko.ssh_exception.PasswordRequiredException, paramiko.ssh_exception.AuthenticationException) as e: + LOGGER.warning(f"Pkey password is required. {str(e)}") + password = input(f"Press Enter to input password of {pkey_path}: ") + self.password = password + return self.transport_connect(user, pkey_path, password) + except (paramiko.ssh_exception.SSHException,) as e: + LOGGER.error(f"Connect remote {self.ip} failed because of wrong pkey. {str(e)}") + raise CreatePkeyFailedException() + + transport = paramiko.Transport((self.ip, 22)) + + try: + self.transport_connect_with_timeout(transport, user, pkey) + except (paramiko.ssh_exception.AuthenticationException, + paramiko.ssh_exception.SSHException, + timeout_decorator.TimeoutError, + socket.gaierror, + socket.timeout, + socket.error) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + return transport + + @timeout_decorator.timeout(10) + def transport_connect_with_timeout(self, transport, user, pkey): + transport.connect(username=user, pkey=pkey) + + def install_component(self, component_name): + ssh_client = self.ssh_client() + sftp_client = self.sftp_client() + try: + self.install_component_handler(component_name, sftp_client, ssh_client) + except (FileNotFoundError, PermissionError, NotADirectoryError, OSError, IOError) as e: + LOGGER.error(f"Remote machine {self.ip} occur Error: {str(e)}") + finally: + ssh_client.close() + sftp_client.close() + + def install_component_handler(self, component_name, sftp_client, ssh_client): + try: + stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + raise ConnectRemoteException() + exit_status = stdout.channel.recv_exit_status() + LOGGER.debug(f"Remote machine {self.ip} mkdir -p /tmp/{constant.DEPENDENCY_DIR} result: " + f"{'success' if not exit_status else 'failed'}") + if exit_status: + raise NotADirectoryError(f"Remote machine {self.ip} " + f"directory {os.path.join('/tmp/', constant.DEPENDENCY_DIR)} not exist.") + + # 上传 组件压缩包和校验文件 + LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") + remote_file_list = [] + shell_dict = component_collection_map.get(component_name) + for shell_cmd in shell_dict: + url_and_save_path = shell_dict.get(shell_cmd) + component = url_and_save_path.get("save_path") + LOGGER.debug(f"Transport component file to remote machine {self.ip}: {component}") + remote_file = os.path.abspath(os.path.join('/tmp', component)) + remote_file_list.append(remote_file) + sftp_client.put(localpath=f"{component}", remotepath=f"{remote_file}") + + # 上传并执行 安装脚本, 校验安装结果脚本 + shell_file_list = ["install.sh", "check_install_result.sh"] + install_result = "" + for shell_file in shell_file_list: + execute_output, sh_file_remote_path = ( + self.transport_shell_file_and_execute(ssh_client, sftp_client, component_name, shell_file)) + remote_file_list.append(sh_file_remote_path) + if shell_file == shell_file_list[1]: + install_result = execute_output + + if install_result == "true": + LOGGER.info(f"Remote machine {self.ip} install {component_name} success.") + else: + LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") + # 清理tmp临时文件 + self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) + + def transport_shell_file_and_execute(self, ssh_client, sftp_client, component_name, shell_file): + sh_file_local_path = os.path.join("./component", component_name, shell_file) + if not os.path.exists(sh_file_local_path): + LOGGER.error(f"{sh_file_local_path} not exists.") + raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + LOGGER.debug(f"Transport local_file: {sh_file_local_path} to remote machine {self.ip} " + f"remote_file: {sh_file_remote_path}") + sftp_client.put(localpath=sh_file_local_path, remotepath=sh_file_remote_path) + stdin, stdout, stderr = ssh_client.exec_command(f"bash {sh_file_remote_path}") + output = stdout.read().decode().strip() + LOGGER.info(f"Remote machine {self.ip} bash {component_name}{shell_file} file output: {output}") + return output, sh_file_remote_path + + def clear_tmp_file_at_remote_machine(self, ssh_client, remote_file_list): + LOGGER.debug(f"Clear tmp file at remote machine {self.ip}") + for remote_file in remote_file_list: + LOGGER.debug(f"Delete tmp file at remote machine {self.ip}: {remote_file}") + ssh_client.exec_command(f"rm -f {remote_file}") diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py new file mode 100644 index 0000000000000000000000000000000000000000..07daec8b50a9752e9c6ca20a839320d99060e9d6 --- /dev/null +++ b/tools/install_dependency/src/machine/scanner_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class ScannerMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(ScannerMachine, self).__init__(ip, user, pkey, password) + self.role = constant.SCANNER \ No newline at end of file