From 6a6750add7635537f8252148884f5d98b6216e18 Mon Sep 17 00:00:00 2001 From: cc <18856836718@163.com> Date: Wed, 13 Mar 2024 16:05:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=85=8D=E7=BD=AE=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E9=95=9C=E5=83=8F=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 1 - .../check_install_result.sh | 8 ++ component/OpenEulerMirrorISO/install.sh | 33 ++++++ tools/download_dependency/src/download.py | 104 ++++++++++++++---- .../src/download_command_line.py | 56 ++++++++++ .../src/download_config.py | 80 ++++++++++++++ .../download_dependency/src/download_utils.py | 35 ++++++ tools/install_dependency/config/machine.yaml | 2 + tools/install_dependency/src/command_line.py | 4 + tools/install_dependency/src/constant.py | 3 + .../install_dependency/src/devkitpipeline.py | 26 +++++ .../src/handler/base_yaml_check.py | 6 +- .../src/handler/connect_check.py | 13 +-- .../src/handler/gather_package.py | 14 +++ .../src/handler/install_package.py | 47 ++++++-- .../src/handler/pipeline.py | 2 +- .../src/machine/devkit_machine.py | 8 ++ .../src/machine/executor_machine.py | 2 +- .../src/machine/klass_dict.py | 12 ++ .../src/machine/local_machine.py | 70 +++++++++--- .../install_dependency/src/machine/machine.py | 75 +++++++++++-- 21 files changed, 530 insertions(+), 71 deletions(-) create mode 100644 component/OpenEulerMirrorISO/check_install_result.sh create mode 100644 component/OpenEulerMirrorISO/install.sh create mode 100644 tools/download_dependency/src/download_command_line.py create mode 100644 tools/download_dependency/src/download_utils.py create mode 100644 tools/install_dependency/src/machine/devkit_machine.py create mode 100644 tools/install_dependency/src/machine/klass_dict.py diff --git a/build.sh b/build.sh index 634ef2b..bd11477 100644 --- a/build.sh +++ b/build.sh @@ -15,7 +15,6 @@ cd ${current_dir}/tools/install_dependency pyinstaller -F ./src/devkitpipeline.py -p ./ --add-data "${current_dir}/component:component" cp ${current_dir}/tools/install_dependency/config/machine.yaml ${current_dir}/tools/install_dependency/dist/machine.yaml -cp -rf ${current_dir}/component ${current_dir}/tools/install_dependency/dist/ cp ${current_dir}/tools/download_dependency/dist/download ${current_dir}/tools/install_dependency/dist/ mkdir -p ${current_dir}/tools/install_dependency/v1.0/tools/ diff --git a/component/OpenEulerMirrorISO/check_install_result.sh b/component/OpenEulerMirrorISO/check_install_result.sh new file mode 100644 index 0000000..9ad9b44 --- /dev/null +++ b/component/OpenEulerMirrorISO/check_install_result.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +yum install git -y >/dev/null 2>&1 +if [[ "$?" == 0 ]]; then + echo "true" +else + echo "false" +fi \ No newline at end of file diff --git a/component/OpenEulerMirrorISO/install.sh b/component/OpenEulerMirrorISO/install.sh new file mode 100644 index 0000000..a879826 --- /dev/null +++ b/component/OpenEulerMirrorISO/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +function configure_local_mirror() { + local iso_file_path=$1 + mount ${iso_file_path} /mnt -o loop + if [[ -d /etc/yum.repos.d/yum.repos.backup ]]; then + mv -rf /etc/yum.repos.d/yum.repos.backup /etc/yum.repos.backup + else + mkdir -p /etc/yum.repos.backup + fi + mv -f /etc/yum.repos.d/* /etc/yum.repos.backup + mv -f /etc/yum.repos.backup /etc/yum.repos.d/ + cat > /etc/yum.repos.d/local.repo <<'EOF' +[local] +name=local.repo +baseurl=file:///mnt +enabled=1 +gpgcheck=0 +EOF + yum clean all + yum makecache +} + +function main() { + iso_file_path=$1 + if [[ -f ${iso_file_path} ]]; then + configure_local_mirror ${iso_file_path} + else + echo "Failed to find iso file." + fi +} + +main "$@" diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index de344a1..2e45460 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -1,11 +1,14 @@ import os +import platform +import subprocess import sys import shutil import tarfile -import urllib.error -import wget +import time -from download_config import BiShengCompiler, GCCforOpenEuler, BiShengJDK8, BiShengJDK17 +import download_config +from download_utils import download_dependence_handler, download_dependence_file +from download_command_line import process_command_line, CommandLine FILE = "file" SHA256 = "sha256" @@ -27,7 +30,10 @@ component_collection_map = { SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" } } for component in ( - BiShengCompiler, GCCforOpenEuler, BiShengJDK8, BiShengJDK17 + download_config.BiShengCompiler, + download_config.GCCforOpenEuler, + download_config.BiShengJDK8, + download_config.BiShengJDK17 ) } @@ -44,29 +50,81 @@ def download_dependence(): ret = True for component_name in component_collection_map: shell_dict = component_collection_map.get(component_name) - for shell_cmd in shell_dict: - url_and_save_path = shell_dict.get(shell_cmd) - try: - print(f"Downloading from {url_and_save_path.get(URL)}") - download_result = wget.download( - url_and_save_path.get(URL), url_and_save_path.get(SAVE_PATH) - ) - print() - except (TimeoutError, urllib.error.URLError, OSError) as e: - print(f"[ERROR] download error occurs: {str(e)}") - return False - - if not os.path.isfile(download_result): - print(f"[ERROR] Download dependencies failed. " - f"Please visit following url and download dependencies to default directory." - f"\n\t{url_and_save_path.get(URL)}" - ) - ret = False + ret = ret and download_dependence_handler(shell_dict) return ret +iso_collection_map = { + component.get("component_name"): { + "download file": + { + URL: f"{component.get(FILE)}", + SAVE_PATH: f"{os.path.join('./', component.get(FILE).split('/')[-1])}" + }, + "download sha256": + { + URL: f"{component.get(SHA256)}", + SAVE_PATH: f"{os.path.join('./', component.get(SHA256).split('/')[-1])}" + } + } for component in ( + download_config.OpenEuler_2003_LTS, + download_config.OpenEuler_2003_LTS_SP1, + download_config.OpenEuler_2003_LTS_SP2, + download_config.OpenEuler_2003_LTS_SP3, + download_config.OpenEuler_2003_LTS_SP4, + download_config.OpenEuler_2009, + download_config.OpenEuler_2103, + download_config.OpenEuler_2109, + download_config.OpenEuler_2203_LTS, + download_config.OpenEuler_2203_LTS_SP1, + download_config.OpenEuler_2203_LTS_SP2, + download_config.OpenEuler_2203_LTS_SP3, + download_config.OpenEuler_2209, + download_config.OpenEuler_2303, + download_config.OpenEuler_2309, + ) +} + + +def download_iso(): + if platform.system() == "Windows" and CommandLine.download_iso == "auto": + print("Please use '-iso' option in Linux machine if iso version is not specified. " + "OpenEuler Operating System is recommended.") + time.sleep(10) + sys.exit(1) + if CommandLine.download_iso == "auto": + result = subprocess.run("grep PRETTY_NAME /etc/os-release".split(' '), + capture_output=True, shell=False) + output = result.stdout.decode().strip() + print(f"Get os-release output: {output}") + + CommandLine.download_iso = (output.split("=")[1] + .replace("(", "") + .replace(")", "") + .replace(".", "") + .replace("\"", "") + .replace(" ", "_") + .replace("-", "_")) + print(f"Auto detect operating system version: {CommandLine.download_iso}") + + shell_dict = iso_collection_map.get(CommandLine.download_iso, "") + if not shell_dict: + print("Please check /etc/os-release is changed or not.") + return False + return download_dependence_file("download file", shell_dict) + + if __name__ == '__main__': try: + process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", + class_list=[CommandLine]) + if CommandLine.download_iso: + if download_iso(): + print("-- Download iso success. --") + else: + print("Download iso failed.") + sys.exit(0) + ret = download_dependence() if ret: print(f"Now compress dependencies to {DEPENDENCY_FILE}...") @@ -79,5 +137,5 @@ if __name__ == '__main__': else: print("-- Download dependencies failed. Please try execute download tool again. --") except (KeyboardInterrupt, Exception) as e: - print(f"Download dependencies failed. {str(e)} Please try execute download tool again.") + print(f"\nDownload dependencies failed. {str(e)} Please try execute download tool again.") sys.exit(1) diff --git a/tools/download_dependency/src/download_command_line.py b/tools/download_dependency/src/download_command_line.py new file mode 100644 index 0000000..12cd1f5 --- /dev/null +++ b/tools/download_dependency/src/download_command_line.py @@ -0,0 +1,56 @@ +import argparse +import download_config + + +class CommandLine: + download_iso = None + + @classmethod + def add_options(cls, parser): + parser.add_argument("-iso", action="store", dest="download_iso", default="", + choices=[ + component.get("component_name") for component in ( + download_config.OpenEuler_2003_LTS, + download_config.OpenEuler_2003_LTS_SP1, + download_config.OpenEuler_2003_LTS_SP2, + download_config.OpenEuler_2003_LTS_SP3, + download_config.OpenEuler_2003_LTS_SP4, + download_config.OpenEuler_2009, + download_config.OpenEuler_2103, + download_config.OpenEuler_2109, + download_config.OpenEuler_2203_LTS, + download_config.OpenEuler_2203_LTS_SP1, + download_config.OpenEuler_2203_LTS_SP2, + download_config.OpenEuler_2203_LTS_SP3, + download_config.OpenEuler_2209, + download_config.OpenEuler_2303, + download_config.OpenEuler_2309, + {"component_name": "auto"}, + ) + ], + metavar="SPECIFY_DOWNLOADING_ISO_VERSION", + help="Specify downloading iso version. " + "Candidate iso versions: " + "openEuler_2003_LTS, openEuler_2003_LTS_SP1, openEuler_2003_LTS_SP2, " + "openEuler_2003_LTS_SP3, openEuler_2003_LTS_SP4, " + "openEuler_2009, openEuler_2103, openEuler_2109, " + "openEuler_2203_LTS, openEuler_2203_LTS_SP1, openEuler_2203_LTS_SP2, " + "openEuler_2203_LTS_SP3, " + "openEuler_2209, openEuler_2303, openEuler_2309. " + "Input 'auto' will auto detect operating system version in Linux if iso version is not specified." + ) + + @classmethod + def process_args(cls, args): + cls.download_iso = args.download_iso + return cls.download_iso + + +def process_command_line(program, description, class_list): + parser = argparse.ArgumentParser(prog=program, description=description, add_help=True) + for klass in class_list: + klass.add_options(parser) + + args = parser.parse_args() + for klass in class_list: + klass.process_args(args) diff --git a/tools/download_dependency/src/download_config.py b/tools/download_dependency/src/download_config.py index e7e1b15..6bda486 100644 --- a/tools/download_dependency/src/download_config.py +++ b/tools/download_dependency/src/download_config.py @@ -18,3 +18,83 @@ BiShengJDK17 = { "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz", "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256", } + + +OpenEuler_2003_LTS = { + "component_name": "openEuler_2003_LTS", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS/ISO/aarch64/openEuler-20.03-LTS-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS/ISO/aarch64/openEuler-20.03-LTS-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2003_LTS_SP1 = { + "component_name": "openEuler_2003_LTS_SP1", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP1/ISO/aarch64/openEuler-20.03-LTS-SP1-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP1/ISO/aarch64/openEuler-20.03-LTS-SP1-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2003_LTS_SP2 = { + "component_name": "openEuler_2003_LTS_SP2", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP2/ISO/aarch64/openEuler-20.03-LTS-SP2-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP2/ISO/aarch64/openEuler-20.03-LTS-SP2-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2003_LTS_SP3 = { + "component_name": "openEuler_2003_LTS_SP3", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP3/ISO/aarch64/openEuler-20.03-LTS-SP3-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP3/ISO/aarch64/openEuler-20.03-LTS-SP3-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2003_LTS_SP4 = { + "component_name": "openEuler_2003_LTS_SP4", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP4/ISO/aarch64/openEuler-20.03-LTS-SP4-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS-SP4/ISO/aarch64/openEuler-20.03-LTS-SP4-everything-aarch64-dvd.iso.sha256sum", +} + +OpenEuler_2009 = { + "component_name": "openEuler_2009", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.09/ISO/aarch64/openEuler-20.09-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.09/ISO/aarch64/openEuler-20.09-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2103 = { + "component_name": "openEuler_2103", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.03/ISO/aarch64/openEuler-21.03-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.03/ISO/aarch64/openEuler-21.03-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2109 = { + "component_name": "openEuler_2109", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.09/ISO/aarch64/openEuler-21.09-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-21.09/ISO/aarch64/openEuler-21.09-everything-aarch64-dvd.iso.sha256sum", +} + +OpenEuler_2203_LTS = { + "component_name": "openEuler_2203_LTS", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS/ISO/aarch64/openEuler-22.03-LTS-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS/ISO/aarch64/openEuler-22.03-LTS-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2203_LTS_SP1 = { + "component_name": "openEuler_2203_LTS_SP1", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP1/ISO/aarch64/openEuler-22.03-LTS-SP1-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP1/ISO/aarch64/openEuler-22.03-LTS-SP1-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2203_LTS_SP2 = { + "component_name": "openEuler_2203_LTS_SP2", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP2/ISO/aarch64/openEuler-22.03-LTS-SP2-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP2/ISO/aarch64/openEuler-22.03-LTS-SP2-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2203_LTS_SP3 = { + "component_name": "openEuler_2203_LTS_SP3", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/ISO/aarch64/openEuler-22.03-LTS-SP3-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.03-LTS-SP3/ISO/aarch64/openEuler-22.03-LTS-SP3-everything-aarch64-dvd.iso.sha256sum", +} + +OpenEuler_2209 = { + "component_name": "openEuler_2209", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.09/ISO/aarch64/openEuler-22.09-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-22.09/ISO/aarch64/openEuler-22.09-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2303 = { + "component_name": "openEuler_2303", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.03/ISO/aarch64/openEuler-23.03-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.03/ISO/aarch64/openEuler-23.03-everything-aarch64-dvd.iso.sha256sum", +} +OpenEuler_2309 = { + "component_name": "openEuler_2309", + "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.09/ISO/aarch64/openEuler-23.09-everything-aarch64-dvd.iso", + "sha256": "https://mirrors.huaweicloud.com/openeuler/openEuler-23.09/ISO/aarch64/openEuler-23.09-everything-aarch64-dvd.iso.sha256sum", +} diff --git a/tools/download_dependency/src/download_utils.py b/tools/download_dependency/src/download_utils.py new file mode 100644 index 0000000..d6fb7e7 --- /dev/null +++ b/tools/download_dependency/src/download_utils.py @@ -0,0 +1,35 @@ +import os +import urllib.error +import wget + + +def download_dependence_handler(shell_dict): + ret = True + for shell_cmd in shell_dict: + ret = download_dependence_file(shell_cmd, shell_dict) + return ret + + +def download_dependence_file(shell_cmd, shell_dict): + ret = True + url_and_save_path = shell_dict.get(shell_cmd) + try: + print(f"Downloading from {url_and_save_path.get('url')}") + download_result = wget.download( + url_and_save_path.get('url'), url_and_save_path.get('save_path') + ) + print() + except (TimeoutError, urllib.error.URLError, OSError) as e: + print(f"[ERROR] download error occurs: {str(e)} " + f"\nPlease visit following url and download dependencies to default directory." + f"\n\t{url_and_save_path.get('url')}" + ) + raise OSError(f"download error occurs: {str(e)}") + + if not os.path.isfile(download_result): + print(f"[ERROR] Download dependencies failed. " + f"Please visit following url and download dependencies to default directory." + f"\n\t{url_and_save_path.get('url')}" + ) + ret = False + return ret diff --git a/tools/install_dependency/config/machine.yaml b/tools/install_dependency/config/machine.yaml index f2a2ecc..6566911 100644 --- a/tools/install_dependency/config/machine.yaml +++ b/tools/install_dependency/config/machine.yaml @@ -8,3 +8,5 @@ builder: - 192.168.0.1 executor: - 192.168.0.1 +devkit: + - 192.168.0.4 \ No newline at end of file diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/command_line.py index 177bdd9..4c6dca7 100644 --- a/tools/install_dependency/src/command_line.py +++ b/tools/install_dependency/src/command_line.py @@ -5,17 +5,21 @@ DEFAULT_YAML_PATH = "./machine.yaml" class CommandLine: yaml_path = DEFAULT_YAML_PATH + iso_path = None debug = False @classmethod def add_options(cls, parser): parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, help="Assign yaml config file path.") + parser.add_argument("-iso", action="store", dest="iso_path", default=None, + help="Assign downloaded iso file path and deploy iso only.") parser.add_argument("--debug", action="store_true", dest="debug", default=False, help="Open debug log.") @classmethod def process_args(cls, args): cls.yaml_path = args.yaml_path if args.yaml_path and args.yaml_path != "./" else DEFAULT_YAML_PATH + cls.iso_path = args.iso_path cls.debug = args.debug return cls.yaml_path diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index 49283cb..5bc23c5 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -4,6 +4,9 @@ PASSWORD = "password" SCANNER = "scanner" BUILDER = "builder" EXECUTOR = "executor" +DEVKIT = "devkit" MACHINE = "machine" DEPENDENCY_FILE = "devkitdependencies.tar.gz" DEPENDENCY_DIR = "devkitdependencies" + +INSTRUCTION = "instruction" diff --git a/tools/install_dependency/src/devkitpipeline.py b/tools/install_dependency/src/devkitpipeline.py index 57305cf..62b77b5 100644 --- a/tools/install_dependency/src/devkitpipeline.py +++ b/tools/install_dependency/src/devkitpipeline.py @@ -1,7 +1,10 @@ +import os +import subprocess import sys import logging import yaml +import constant from log import config_logging from command_line import process_command_line, CommandLine @@ -13,6 +16,7 @@ from handler.install_package import InstallPackage LOGGER = logging.getLogger("install_dependency") PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] +ISO_VERIFY_FLAG_STRING = "ISO 9660 CD-ROM filesystem data" def read_yaml_file(yaml_path): @@ -31,12 +35,34 @@ def read_yaml_file(yaml_path): return config_dict +def check_iso_available(iso_path): + if not os.path.isfile(iso_path): + LOGGER.error(f"ISO file is not in specified path. Error: {iso_path} file not found.") + sys.exit(1) + try: + result = subprocess.run(f"file -b {iso_path}".split(' '), + capture_output=True, shell=False) + output = result.stdout.decode().strip() + if output.find(ISO_VERIFY_FLAG_STRING) == -1: + LOGGER.error(f"Verify iso result: Not available. Please re-download iso file.") + sys.exit(1) + except (FileNotFoundError, IsADirectoryError, PermissionError, Exception) as e: + LOGGER.error(f"Verify iso file integrity occur error: {str(e)}") + sys.exit(1) + + if __name__ == '__main__': try: process_command_line(program="install_dependency", description="devkit-pipeline install_dependency tool", class_list=[CommandLine]) config_logging(CommandLine.debug) config_dict = read_yaml_file(CommandLine.yaml_path) + + if CommandLine.iso_path: + config_dict[constant.INSTRUCTION] = "deploy_iso" + check_iso_available(CommandLine.iso_path) + else: + config_dict[constant.INSTRUCTION] = "default" LOGGER.debug(f"-- config_dict: {config_dict}") pipe = PipeLine(config_dict) diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 7b07d22..101c9c9 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -5,8 +5,10 @@ import constant from handler.handler_and_node import Handler LOGGER = logging.getLogger("install_dependency") -MIN_SET = (constant.USER, constant.PKEY, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) -MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, constant.SCANNER, constant.BUILDER, constant.EXECUTOR) +MIN_SET = (constant.USER, constant.PKEY, + constant.SCANNER, constant.BUILDER, constant.EXECUTOR, constant.DEVKIT, constant.INSTRUCTION) +MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, + constant.SCANNER, constant.BUILDER, constant.EXECUTOR, constant.DEVKIT, constant.INSTRUCTION) class BaseCheck(Handler): diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index eb321f8..f3e656b 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -3,35 +3,28 @@ import socket import constant from handler.handler_and_node import Handler -from machine.scanner_machine import ScannerMachine -from machine.builder_machine import BuilderMachine -from machine.executor_machine import ExecutorMachine from machine.local_machine import LocalMachine +from machine.klass_dict import KLASS_DICT from exception.connect_exception import ConnectException LOGGER = logging.getLogger("install_dependency") class ConnectCheck(Handler): - klass_dict = { - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, - constant.EXECUTOR: ExecutorMachine, - } def handle(self, data) -> bool: LOGGER.debug("ConnectCheck start!") local_ip = ConnectCheck.get_local_ip() ret = True - for role in ConnectCheck.klass_dict: + for role in (set(KLASS_DICT.keys()) & set(data.keys())): ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) return ret @staticmethod def machine_role_check(data, role, local_ip): builder_list = data.get(role) - klass = ConnectCheck.klass_dict.get(role) + klass = KLASS_DICT.get(role) data[role + constant.MACHINE] = dict() for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index 90f812f..e71aaca 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -9,7 +9,21 @@ LOGGER = logging.getLogger("install_dependency") class GatherPackage(Handler): + def handle(self, data) -> bool: + instruction_to_func_dict = { + "deploy_iso": GatherPackage.deploy_iso_handle, + "default": GatherPackage.default_handle, + } + return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))() + + @staticmethod + def deploy_iso_handle(): + LOGGER.info("Iso file already checked.") + return True + + @staticmethod + def default_handle(): LOGGER.debug("GatherPackage start!") if GatherPackage.check_default_path_available(): LOGGER.info("Dependencies ready.") diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py index fc88628..3a1ddab 100644 --- a/tools/install_dependency/src/handler/install_package.py +++ b/tools/install_dependency/src/handler/install_package.py @@ -3,26 +3,55 @@ import multiprocessing import constant from handler.handler_and_node import Handler -from machine.scanner_machine import ScannerMachine -from machine.builder_machine import BuilderMachine -from machine.executor_machine import ExecutorMachine +from machine.klass_dict import KLASS_DICT LOGGER = logging.getLogger("install_dependency") class InstallPackage(Handler): - klass_dict = { - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, - constant.EXECUTOR: ExecutorMachine, - } def handle(self, data) -> bool: + instruction_to_func_dict = { + "deploy_iso": InstallPackage.deploy_iso_handle, + "default": InstallPackage.default_handle, + } + return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))(data) + + @staticmethod + def deploy_iso_handle(data): + LOGGER.debug("Deploy iso start!") + ip_set = set() + jobs = [] + + for role in ({constant.EXECUTOR, constant.DEVKIT} & set(data.keys())): + machine_dict = data[role + constant.MACHINE] + LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") + for machine_ip in machine_dict: + if machine_ip in ip_set: + continue + ip_set.add(machine_ip) + LOGGER.debug(f"ip_set: {ip_set}") + machine = machine_dict.get(machine_ip) + process = multiprocessing.Process( + target=process_work, + args=(machine, + "OpenEulerMirrorISO", + ), + ) + jobs.append(process) + process.start() + + for job in jobs: + job.join() + return True + + @staticmethod + def default_handle(data): LOGGER.debug("Install Package start!") ip_set = set() jobs = [] - for role in InstallPackage.klass_dict: + for role in (set(KLASS_DICT.keys()) & set(data.keys())): machine_dict = data[role + constant.MACHINE] LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") for machine_ip in machine_dict: diff --git a/tools/install_dependency/src/handler/pipeline.py b/tools/install_dependency/src/handler/pipeline.py index d1db8d0..dbeaca6 100644 --- a/tools/install_dependency/src/handler/pipeline.py +++ b/tools/install_dependency/src/handler/pipeline.py @@ -15,7 +15,7 @@ class PipeLine: def start(self): if self.head.get_next_node() and self.head.get_next_node().execute(self.data): - print("-- Congratulations! Install package success in all machines. --") + print("-- Program finished. --") def add_tail(self, *handlers): for handler in handlers: diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py new file mode 100644 index 0000000..35147f4 --- /dev/null +++ b/tools/install_dependency/src/machine/devkit_machine.py @@ -0,0 +1,8 @@ +import constant +from machine.machine import Machine + + +class DevkitMachine(Machine): + def __init__(self, ip, user, pkey, password=None): + super(DevkitMachine, self).__init__(ip, user, pkey, password) + self.role = constant.DEVKIT diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py index d0db47b..923c7a1 100644 --- a/tools/install_dependency/src/machine/executor_machine.py +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -5,4 +5,4 @@ from machine.machine import Machine class ExecutorMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ExecutorMachine, self).__init__(ip, user, pkey, password) - self.role = constant.EXECUTOR \ No newline at end of file + self.role = constant.EXECUTOR diff --git a/tools/install_dependency/src/machine/klass_dict.py b/tools/install_dependency/src/machine/klass_dict.py new file mode 100644 index 0000000..d9e82c5 --- /dev/null +++ b/tools/install_dependency/src/machine/klass_dict.py @@ -0,0 +1,12 @@ +import constant +from machine.scanner_machine import ScannerMachine +from machine.builder_machine import BuilderMachine +from machine.executor_machine import ExecutorMachine +from machine.devkit_machine import DevkitMachine + +KLASS_DICT = { + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, + constant.EXECUTOR: ExecutorMachine, + constant.DEVKIT: DevkitMachine, +} diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 86a572f..1815b1a 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -3,11 +3,13 @@ import logging import subprocess import constant +from command_line import CommandLine from exception.connect_exception import NotMatchedMachineTypeException from download import component_collection_map from utils import base_path LOGGER = logging.getLogger("install_dependency") +SHELL_FILE_LIST = ["install.sh", "check_install_result.sh"] class LocalMachine: @@ -29,6 +31,42 @@ class LocalMachine: LOGGER.error(f"Local machine {self.ip} occur Error: {str(e)}") def install_component_handler(self, component_name): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "OpenEulerMirrorISO": self.deploy_iso_handle, + } + return component_name_to_func_dict.get(component_name)(component_name) + + def deploy_iso_handle(self, component_name): + # 复制镜像文件 + LOGGER.info(f"Deploy component in local machine {self.ip}: {component_name}") + local_path = os.path.abspath(CommandLine.iso_path) + + # 执行 安装脚本, 校验安装结果脚本 + install_result = "" + for shell_file in SHELL_FILE_LIST: + sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + sh_cmd = f"bash {sh_file_local_path} {local_path}" + if not os.path.exists(sh_file_local_path): + LOGGER.error(f"{sh_file_local_path} not exists.") + raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") + + result = subprocess.run(f"{sh_cmd}".split(' '), + capture_output=True, shell=False) + output = result.stdout.decode().strip() + LOGGER.info(f"Local machine {self.ip} '{sh_cmd}' file output: {output}") + if shell_file == SHELL_FILE_LIST[1]: + install_result = output + + if install_result == "true": + LOGGER.info(f"Remote machine {self.ip} deploy {component_name} success.") + else: + LOGGER.info(f"Remote machine {self.ip} deploy {component_name} failed.") + + def default_install_component_handle(self, component_name): result = subprocess.run(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}".split(' '), capture_output=False, shell=False, stderr=subprocess.STDOUT) if result.returncode == 0: @@ -48,17 +86,23 @@ class LocalMachine: LOGGER.debug(f"Copy component file to local machine {self.ip}: {component}") remote_file = os.path.abspath(os.path.join('/tmp', component)) remote_file_list.append(remote_file) - subprocess.run(f"/bin/cp -rf {component} {remote_file}".split(' '), + subprocess.run(f"/bin/cp -f {component} {remote_file}".split(' '), capture_output=False, shell=False, stderr=subprocess.STDOUT) # 上传并执行 安装脚本, 校验安装结果脚本 - shell_file_list = ["install.sh", "check_install_result.sh"] install_result = "" - for shell_file in shell_file_list: - execute_output, sh_file_remote_path = ( - self.transport_shell_file_and_execute(component_name, shell_file)) + for shell_file in SHELL_FILE_LIST: + sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + sh_cmd = f"bash {sh_file_remote_path}" + execute_output = ( + self.transport_shell_file_and_execute( + sh_file_local_path=sh_file_local_path, + sh_file_remote_path=sh_file_remote_path, + sh_cmd=sh_cmd + )) remote_file_list.append(sh_file_remote_path) - if shell_file == shell_file_list[1]: + if shell_file == SHELL_FILE_LIST[1]: install_result = execute_output if install_result == "true": @@ -68,21 +112,21 @@ class LocalMachine: # 清理tmp临时文件 self.clear_tmp_file_at_local_machine(remote_file_list) - def transport_shell_file_and_execute(self, component_name, shell_file): - sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + def transport_shell_file_and_execute(self, sh_file_local_path, sh_file_remote_path, sh_cmd): if not os.path.exists(sh_file_local_path): LOGGER.error(f"{sh_file_local_path} not exists.") raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") - sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + LOGGER.debug(f"Copy local_file: {sh_file_local_path} to local machine {self.ip} " f"remote_file: {sh_file_remote_path}") - subprocess.run(f"/bin/cp -rf {sh_file_local_path} {sh_file_remote_path}".split(' '), + subprocess.run(f"/bin/cp -f {sh_file_local_path} {sh_file_remote_path}".split(' '), capture_output=False, shell=False, stderr=subprocess.STDOUT) - result = subprocess.run(f"bash {sh_file_remote_path}".split(' '), + + result = subprocess.run(f"{sh_cmd}".split(' '), capture_output=True, shell=False) output = result.stdout.decode().strip() - LOGGER.info(f"Local machine {self.ip} bash {component_name}{shell_file} file output: {output}") - return output, sh_file_remote_path + LOGGER.info(f"Local machine {self.ip} '{sh_cmd}' file output: {output}") + return output def clear_tmp_file_at_local_machine(self, remote_file_list): LOGGER.debug(f"Clear tmp file at local machine {self.ip}") diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index e75f485..e1b3147 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -5,12 +5,14 @@ import logging import timeout_decorator import constant +from command_line import CommandLine from exception.connect_exception import CreatePkeyFailedException, ConnectRemoteException, \ NotMatchedMachineTypeException from download import component_collection_map from utils import base_path LOGGER = logging.getLogger("install_dependency") +SHELL_FILE_LIST = ["install.sh", "check_install_result.sh"] class Machine: @@ -94,6 +96,50 @@ class Machine: sftp_client.close() def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "OpenEulerMirrorISO": self.deploy_iso_handle, + } + return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) + + def deploy_iso_handle(self, component_name, sftp_client, ssh_client): + # 上传 镜像文件 + LOGGER.info(f"Deploy component in remote machine {self.ip}: {component_name}") + local_path = os.path.abspath(CommandLine.iso_path) + remote_path = os.path.join("/home", local_path.split('/')[-1]) + LOGGER.debug(f"Transport local_file: {local_path} to remote machine {self.ip} " + f"remote_file: {remote_path}") + sftp_client.put(localpath=local_path, remotepath=remote_path) + + # 上传并执行 安装脚本, 校验安装结果脚本 + install_result = "" + remote_file_list = [] + for shell_file in SHELL_FILE_LIST: + sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + sh_file_remote_path = os.path.join("/tmp/", component_name + shell_file) + sh_cmd = f"bash {sh_file_remote_path} {remote_path}" + execute_output = ( + self.transport_shell_file_and_execute( + ssh_client, sftp_client, + sh_file_local_path=sh_file_local_path, + sh_file_remote_path=sh_file_remote_path, + sh_cmd=sh_cmd + )) + remote_file_list.append(sh_file_remote_path) + if shell_file == SHELL_FILE_LIST[1]: + install_result = execute_output + + if install_result == "true": + LOGGER.info(f"Remote machine {self.ip} deploy {component_name} success.") + else: + LOGGER.info(f"Remote machine {self.ip} deploy {component_name} failed.") + # 清理tmp临时文件 + self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) + + def default_install_component_handle(self, component_name, sftp_client, ssh_client): try: stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) except (paramiko.ssh_exception.SSHException, socket.timeout) as e: @@ -119,13 +165,20 @@ class Machine: sftp_client.put(localpath=f"{component}", remotepath=f"{remote_file}") # 上传并执行 安装脚本, 校验安装结果脚本 - shell_file_list = ["install.sh", "check_install_result.sh"] install_result = "" - for shell_file in shell_file_list: - execute_output, sh_file_remote_path = ( - self.transport_shell_file_and_execute(ssh_client, sftp_client, component_name, shell_file)) + for shell_file in SHELL_FILE_LIST: + sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + sh_cmd = f"bash {sh_file_remote_path}" + execute_output = ( + self.transport_shell_file_and_execute( + ssh_client, sftp_client, + sh_file_local_path=sh_file_local_path, + sh_file_remote_path=sh_file_remote_path, + sh_cmd=sh_cmd + )) remote_file_list.append(sh_file_remote_path) - if shell_file == shell_file_list[1]: + if shell_file == SHELL_FILE_LIST[1]: install_result = execute_output if install_result == "true": @@ -135,19 +188,19 @@ class Machine: # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) - def transport_shell_file_and_execute(self, ssh_client, sftp_client, component_name, shell_file): - sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) + def transport_shell_file_and_execute(self, ssh_client, sftp_client, sh_file_local_path, sh_file_remote_path, sh_cmd): if not os.path.exists(sh_file_local_path): LOGGER.error(f"{sh_file_local_path} not exists.") raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") - sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + LOGGER.debug(f"Transport local_file: {sh_file_local_path} to remote machine {self.ip} " f"remote_file: {sh_file_remote_path}") sftp_client.put(localpath=sh_file_local_path, remotepath=sh_file_remote_path) - stdin, stdout, stderr = ssh_client.exec_command(f"bash {sh_file_remote_path}") + + stdin, stdout, stderr = ssh_client.exec_command(sh_cmd) output = stdout.read().decode().strip() - LOGGER.info(f"Remote machine {self.ip} bash {component_name}{shell_file} file output: {output}") - return output, sh_file_remote_path + LOGGER.info(f"Remote machine {self.ip} '{sh_cmd}' file output: {output}") + return output def clear_tmp_file_at_remote_machine(self, ssh_client, remote_file_list): LOGGER.debug(f"Clear tmp file at remote machine {self.ip}") -- Gitee