From 7b13ad197cf0d719d3a8b642881034d053b0b912 Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Wed, 27 Mar 2024 16:59:28 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=88=AA=E9=99=A4machine=E5=AD=90=E9=A1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/handler/base_yaml_check.py | 12 +++++++----- .../install_dependency/src/handler/connect_check.py | 11 +++++------ .../src/machine/builder_machine.py | 8 -------- .../install_dependency/src/machine/devkit_machine.py | 8 -------- .../src/machine/executor_machine.py | 8 -------- tools/install_dependency/src/machine/klass_dict.py | 12 ------------ .../src/machine/scanner_machine.py | 8 -------- 7 files changed, 12 insertions(+), 55 deletions(-) delete mode 100644 tools/install_dependency/src/machine/builder_machine.py delete mode 100644 tools/install_dependency/src/machine/devkit_machine.py delete mode 100644 tools/install_dependency/src/machine/executor_machine.py delete mode 100644 tools/install_dependency/src/machine/klass_dict.py delete mode 100644 tools/install_dependency/src/machine/scanner_machine.py diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 748a8b3..82fb714 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -2,7 +2,8 @@ import re import logging import constant from handler.handler_and_node import Handler -from machine.klass_dict import KLASS_DICT +from download import ROLE_LIST + LOGGER = logging.getLogger("install_dependency") MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) @@ -48,11 +49,12 @@ class BaseCheck(Handler): @staticmethod def check_machine_ip(data): - for machine_type in (set(KLASS_DICT.keys()) & set(data.keys())): - if not data.get(machine_type) or not isinstance(data.get(machine_type), list): - LOGGER.error(f"Yaml file content not correct. Yaml file {machine_type} value not sequence.") + for role in (ROLE_LIST & data.keys()): + ip_list = data.get(role, []) + if not isinstance(ip_list, list): + LOGGER.error(f"Yaml file content not correct. Yaml file {role} value not sequence.") return False - for ip in data.get(machine_type): + for ip in ip_list: if not BaseCheck.validate_ip(ip): LOGGER.error(f"Yaml file content not correct. Given ip: {ip} not correct.") return False diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index e1e852f..2b32c46 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -4,9 +4,9 @@ import socket import constant from handler.handler_and_node import Handler from machine.local_machine import LocalMachine -from machine.klass_dict import KLASS_DICT +from machine.machine import Machine from exception.connect_exception import ConnectException -from download import ROLE_COMPONENT +from download import ROLE_COMPONENT, ROLE_LIST LOGGER = logging.getLogger("install_dependency") @@ -17,14 +17,13 @@ class ConnectCheck(Handler): local_ip = ConnectCheck.get_local_ip() data[constant.MACHINE] = dict() ret = True - for role in (set(KLASS_DICT.keys()) & set(data.keys())): + for role in (ROLE_LIST & data.keys()): ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) return ret @staticmethod def machine_role_check(data, role, local_ip): builder_list = data.get(role) - klass = KLASS_DICT.get(role) for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" @@ -33,8 +32,8 @@ class ConnectCheck(Handler): data[constant.MACHINE][ip] = machine_instance continue try: - machine_instance = data[constant.MACHINE].get(ip, klass(ip, data[constant.USER], data[constant.PKEY], - data.get(constant.PASSWORD, None))) + machine_instance = data[constant.MACHINE].get(ip, Machine(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None))) machine_instance.add_component(ROLE_COMPONENT[role]) data[constant.MACHINE][ip] = machine_instance except ConnectException: diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py deleted file mode 100644 index 8ee4479..0000000 --- a/tools/install_dependency/src/machine/builder_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class BuilderMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(BuilderMachine, self).__init__(ip, user, pkey, password) - self.role = constant.BUILDER diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py deleted file mode 100644 index 35147f4..0000000 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class DevkitMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(DevkitMachine, self).__init__(ip, user, pkey, password) - self.role = constant.DEVKIT diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py deleted file mode 100644 index 923c7a1..0000000 --- a/tools/install_dependency/src/machine/executor_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ExecutorMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ExecutorMachine, self).__init__(ip, user, pkey, password) - self.role = constant.EXECUTOR diff --git a/tools/install_dependency/src/machine/klass_dict.py b/tools/install_dependency/src/machine/klass_dict.py deleted file mode 100644 index 835b999..0000000 --- a/tools/install_dependency/src/machine/klass_dict.py +++ /dev/null @@ -1,12 +0,0 @@ -import constant -from machine.scanner_machine import ScannerMachine -from machine.builder_machine import BuilderMachine -from machine.executor_machine import ExecutorMachine -from machine.devkit_machine import DevkitMachine - -KLASS_DICT = { - constant.EXECUTOR: ExecutorMachine, - constant.DEVKIT: DevkitMachine, - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, -} diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py deleted file mode 100644 index 9377b52..0000000 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ScannerMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ScannerMachine, self).__init__(ip, user, pkey, password) - self.role = constant.SCANNER -- Gitee From f5b6ca1bad4feca70fead8ba5ab7014bede6dcba Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Wed, 27 Mar 2024 17:18:33 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E8=AA=BF=E6=95=B4=E4=BE=9D=E8=B3=B4?= =?UTF-8?q?=E9=97=9C=E4=BF=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../build_download.sh | 2 +- tools/install_dependency/build_install.sh | 2 +- tools/install_dependency/src/__init__.py | 0 tools/install_dependency/src/constant.py | 14 +++ .../install_dependency/src/deploy/__init__.py | 0 .../src/{ => deploy}/command_line.py | 0 .../src/{ => deploy}/lkp_collect_map.py | 52 ++++----- .../src/{install.py => deploy_main.py} | 4 +- .../src/download/__init__.py | 0 .../src/download}/download_command_line.py | 2 +- .../src/download}/download_config.py | 0 .../src/download}/download_utils.py | 58 +++++++++- .../src/download_main.py} | 104 ++---------------- .../src/handler/base_yaml_check.py | 2 +- .../src/handler/connect_check.py | 2 +- .../src/handler/gather_package.py | 2 +- .../src/machine/local_machine.py | 6 +- .../install_dependency/src/machine/machine.py | 6 +- tools/install_dependency/src/utils.py | 17 +++ 19 files changed, 137 insertions(+), 136 deletions(-) rename tools/{download_dependency => install_dependency}/build_download.sh (77%) create mode 100644 tools/install_dependency/src/__init__.py create mode 100644 tools/install_dependency/src/deploy/__init__.py rename tools/install_dependency/src/{ => deploy}/command_line.py (100%) rename tools/install_dependency/src/{ => deploy}/lkp_collect_map.py (96%) rename tools/install_dependency/src/{install.py => deploy_main.py} (95%) create mode 100644 tools/install_dependency/src/download/__init__.py rename tools/{download_dependency/src => install_dependency/src/download}/download_command_line.py (98%) rename tools/{download_dependency/src => install_dependency/src/download}/download_config.py (100%) rename tools/{download_dependency/src => install_dependency/src/download}/download_utils.py (39%) rename tools/{download_dependency/src/download.py => install_dependency/src/download_main.py} (59%) diff --git a/tools/download_dependency/build_download.sh b/tools/install_dependency/build_download.sh similarity index 77% rename from tools/download_dependency/build_download.sh rename to tools/install_dependency/build_download.sh index 8f141cf..0201b05 100644 --- a/tools/download_dependency/build_download.sh +++ b/tools/install_dependency/build_download.sh @@ -14,4 +14,4 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src --name "download_tool" +pyinstaller -F "${current_dir}"/src/download_main.py -p "${current_dir}"/src --name "download_tool" diff --git a/tools/install_dependency/build_install.sh b/tools/install_dependency/build_install.sh index e9fcc7b..f2b7a7f 100644 --- a/tools/install_dependency/build_install.sh +++ b/tools/install_dependency/build_install.sh @@ -14,6 +14,6 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/install.py -p "${current_dir}/src:${project_dir}/tools/download_dependency/src" --add-data "../../build/component:component" --name "deploy_tool" +pyinstaller -F "${current_dir}"/src/deploy_main.py -p "${current_dir}/src" --add-data "../../build/component:component" --name "deploy_tool" cp "${current_dir}"/config/machine.yaml "${build_dir}"/dist/machine.yaml diff --git a/tools/install_dependency/src/__init__.py b/tools/install_dependency/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index 5bc23c5..44aa346 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -10,3 +10,17 @@ DEPENDENCY_FILE = "devkitdependencies.tar.gz" DEPENDENCY_DIR = "devkitdependencies" INSTRUCTION = "instruction" + +ROLE_COMPONENT = { + SCANNER: ["BiShengJDK17"], + BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + EXECUTOR: ["BiShengJDK17", "LkpTests"] +} + +ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] + +FILE = "file" +SHA256 = "sha256" +URL = "url" +SAVE_PATH = "save_path" +DEFAULT_PATH = "./devkitdependencies" diff --git a/tools/install_dependency/src/deploy/__init__.py b/tools/install_dependency/src/deploy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/deploy/command_line.py similarity index 100% rename from tools/install_dependency/src/command_line.py rename to tools/install_dependency/src/deploy/command_line.py diff --git a/tools/install_dependency/src/lkp_collect_map.py b/tools/install_dependency/src/deploy/lkp_collect_map.py similarity index 96% rename from tools/install_dependency/src/lkp_collect_map.py rename to tools/install_dependency/src/deploy/lkp_collect_map.py index 2303e2c..e06a4f0 100644 --- a/tools/install_dependency/src/lkp_collect_map.py +++ b/tools/install_dependency/src/deploy/lkp_collect_map.py @@ -1,26 +1,26 @@ -import os.path - -from utils import base_path - -CURRENT_DEFAULT_PATH = "./devkitdependencies" - -lkp_collection_map = { - "LkpTests": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - }, - "CompatibilityTesting": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, - "DevkitDistribute": { - "download file": { - "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", - } - } -} +import os.path + +from utils import base_path + +CURRENT_DEFAULT_PATH = "./devkitdependencies" + +lkp_collection_map = { + "LkpTests": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", + }, + "download gem dependency": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", + }, + }, + "CompatibilityTesting": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + } + }, + "DevkitDistribute": { + "download file": { + "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", + } + } +} diff --git a/tools/install_dependency/src/install.py b/tools/install_dependency/src/deploy_main.py similarity index 95% rename from tools/install_dependency/src/install.py rename to tools/install_dependency/src/deploy_main.py index ca320da..a406e3a 100644 --- a/tools/install_dependency/src/install.py +++ b/tools/install_dependency/src/deploy_main.py @@ -6,14 +6,14 @@ import logging import constant from log import config_logging -from command_line import process_command_line, CommandLine +from deploy.command_line import process_command_line, CommandLine from handler.pipeline import PipeLine from handler.base_yaml_check import BaseCheck from handler.connect_check import ConnectCheck from handler.gather_package import GatherPackage from handler.install_package import InstallPackage -from download import read_yaml_file +from utils import read_yaml_file LOGGER = logging.getLogger("install_dependency") PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] diff --git a/tools/install_dependency/src/download/__init__.py b/tools/install_dependency/src/download/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/download_dependency/src/download_command_line.py b/tools/install_dependency/src/download/download_command_line.py similarity index 98% rename from tools/download_dependency/src/download_command_line.py rename to tools/install_dependency/src/download/download_command_line.py index 0140818..d4b1cc8 100644 --- a/tools/download_dependency/src/download_command_line.py +++ b/tools/install_dependency/src/download/download_command_line.py @@ -1,5 +1,5 @@ import argparse -import download_config +from download import download_config DEFAULT_YAML_PATH = "./machine.yaml" diff --git a/tools/download_dependency/src/download_config.py b/tools/install_dependency/src/download/download_config.py similarity index 100% rename from tools/download_dependency/src/download_config.py rename to tools/install_dependency/src/download/download_config.py diff --git a/tools/download_dependency/src/download_utils.py b/tools/install_dependency/src/download/download_utils.py similarity index 39% rename from tools/download_dependency/src/download_utils.py rename to tools/install_dependency/src/download/download_utils.py index b51ca8b..f3f6510 100644 --- a/tools/download_dependency/src/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -1,7 +1,63 @@ import os -import sys import urllib.error import wget +from download import download_config +from constant import URL, FILE, SAVE_PATH, SHA256, DEFAULT_PATH + + +component_collection_map = { + component.get("component_name"): { + "download file": + { + URL: f"{component.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}" + }, + "download sha256": + { + URL: f"{component.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" + } + } for component in ( + download_config.BiShengCompiler, + download_config.GCCforOpenEuler, + download_config.BiShengJDK8, + download_config.BiShengJDK17, + ) +} + +lkp_collection_map = { + "LkpTests": { + "download file": { + URL: f"{download_config.LkpTests.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'lkp-tests.tar.gz')}", + }, + "download gem dependency": { + URL: f"{download_config.LkpTests.get('gem dependency')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", + }, + "download CompatibilityTesting": { + URL: f"{download_config.CompatibilityTesting.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + } + }, +} + + +def download_dependence(component_list): + if not os.path.exists(DEFAULT_PATH): + os.mkdir(DEFAULT_PATH) + elif os.path.isfile(DEFAULT_PATH): + print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") + return False + + ret = True + component_collection_map.update(lkp_collection_map) + for component_name in component_collection_map: + if component_name not in component_list: + continue + shell_dict = component_collection_map.get(component_name) + ret = ret and download_dependence_handler(shell_dict) + return ret def download_dependence_handler(shell_dict): diff --git a/tools/download_dependency/src/download.py b/tools/install_dependency/src/download_main.py similarity index 59% rename from tools/download_dependency/src/download.py rename to tools/install_dependency/src/download_main.py index 6baba46..9199cbb 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/install_dependency/src/download_main.py @@ -5,17 +5,13 @@ import sys import shutil import tarfile import wget -import yaml -import download_config -from download_utils import download_dependence_handler, download_dependence_file -from download_command_line import process_command_line, CommandLine - -FILE = "file" -SHA256 = "sha256" -URL = "url" -SAVE_PATH = "save_path" -DEFAULT_PATH = "./devkitdependencies" -DEPENDENCY_FILE = "devkitdependencies.tar.gz" +from download import download_config +from download.download_utils import download_dependence_file, download_dependence +from download.download_command_line import process_command_line, CommandLine + +from utils import read_yaml_file +from constant import URL, SHA256, SAVE_PATH, DEFAULT_PATH, FILE, DEPENDENCY_FILE, ROLE_LIST, ROLE_COMPONENT + # A-FOT files BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" @@ -28,88 +24,6 @@ SPLIT_JSON_PY = "split_json.py" FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) -component_collection_map = { - component.get("component_name"): { - "download file": - { - URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}" - }, - "download sha256": - { - URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" - } - } for component in ( - download_config.BiShengCompiler, - download_config.GCCforOpenEuler, - download_config.BiShengJDK8, - download_config.BiShengJDK17, - ) -} - -lkp_collection_map = { - "LkpTests": { - "download file": { - URL: f"{download_config.LkpTests.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - URL: f"{download_config.LkpTests.get('gem dependency')}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - "download CompatibilityTesting": { - URL: f"{download_config.CompatibilityTesting.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, -} -SCANNER = "scanner" -BUILDER = "builder" -EXECUTOR = "executor" - -ROLE_COMPONENT = { - SCANNER: ["BiShengJDK17"], - BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], - EXECUTOR: ["BiShengJDK17", "LkpTests"] -} - -ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] - - -def read_yaml_file(yaml_path): - try: - with open(yaml_path, "r") as file: - yaml_dict = yaml.safe_load(file) - except (FileNotFoundError, IsADirectoryError) as e: - print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") - sys.exit(1) - except (yaml.parser.ParserError, - yaml.scanner.ScannerError, - yaml.composer.ComposerError, - yaml.constructor.ConstructorError) as e: - print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") - sys.exit(1) - return yaml_dict - - -def download_dependence(component_list): - if not os.path.exists(DEFAULT_PATH): - os.mkdir(DEFAULT_PATH) - elif os.path.isfile(DEFAULT_PATH): - print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") - return False - - ret = True - component_collection_map.update(lkp_collection_map) - for component_name in component_collection_map: - if component_name not in component_list: - continue - shell_dict = component_collection_map.get(component_name) - ret = ret and download_dependence_handler(shell_dict) - return ret - - def download_a_fot(): saved_path = os.path.join(DEFAULT_PATH, A_FOT) try: @@ -136,12 +50,12 @@ iso_collection_map = { "download file": { URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join('./', component.get(FILE).split('/')[-1])}" + SAVE_PATH: f"{os.path.join('/', component.get(FILE).split('/')[-1])}" }, "download sha256": { URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join('./', component.get(SHA256).split('/')[-1])}" + SAVE_PATH: f"{os.path.join('/', component.get(SHA256).split('/')[-1])}" } } for component in ( download_config.OpenEuler_2003_LTS, diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 82fb714..eac596c 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -2,7 +2,7 @@ import re import logging import constant from handler.handler_and_node import Handler -from download import ROLE_LIST +from constant import ROLE_LIST LOGGER = logging.getLogger("install_dependency") diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index 2b32c46..02cf7c5 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -6,7 +6,7 @@ from handler.handler_and_node import Handler from machine.local_machine import LocalMachine from machine.machine import Machine from exception.connect_exception import ConnectException -from download import ROLE_COMPONENT, ROLE_LIST +from constant import ROLE_COMPONENT, ROLE_LIST LOGGER = logging.getLogger("install_dependency") diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index 714525c..5ee9528 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -2,7 +2,7 @@ import logging import os import subprocess import constant -from download import download_dependence, component_collection_map, lkp_collection_map +from download.download_utils import download_dependence, component_collection_map, lkp_collection_map from handler.handler_and_node import Handler LOGGER = logging.getLogger("install_dependency") diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index f12f9c7..d4e5077 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -3,10 +3,10 @@ import logging import subprocess import constant -from command_line import CommandLine +from deploy.command_line import CommandLine from exception.connect_exception import NotMatchedMachineTypeException -from download import component_collection_map -from lkp_collect_map import lkp_collection_map +from download.download_utils import component_collection_map +from deploy.lkp_collect_map import lkp_collection_map from utils import (base_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, CHECK_MIRROR_INSTALL_STATUS, PROMPT_MAP) diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index a08d9d9..f64835e 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -7,11 +7,11 @@ import paramiko import timeout_decorator import constant -from command_line import CommandLine +from deploy.command_line import CommandLine from exception.connect_exception import (CreatePkeyFailedException, ConnectRemoteException, NotMatchedMachineTypeException) -from download import component_collection_map -from lkp_collect_map import lkp_collection_map +from download.download_utils import component_collection_map +from deploy.lkp_collect_map import lkp_collection_map from utils import (base_path, validate_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, CHECK_MIRROR_INSTALL_STATUS, PROMPT_MAP) diff --git a/tools/install_dependency/src/utils.py b/tools/install_dependency/src/utils.py index d249129..551769b 100644 --- a/tools/install_dependency/src/utils.py +++ b/tools/install_dependency/src/utils.py @@ -1,5 +1,6 @@ import os import sys +import yaml MKDIR_TMP_DEVKITDEPENDENCIES_CMD = "mkdir -p /tmp/devkitdependencies" YUM_INSTALL_LKP_DEPENDENCIES_CMD = "yum install -y git wget rubygems" @@ -34,3 +35,19 @@ def available_role(role_list, data): if role in data: list_after_verification.append(role) return list_after_verification + + +def read_yaml_file(yaml_path): + try: + with open(yaml_path, "r") as file: + yaml_dict = yaml.safe_load(file) + except (FileNotFoundError, IsADirectoryError) as e: + print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") + sys.exit(1) + except (yaml.parser.ParserError, + yaml.scanner.ScannerError, + yaml.composer.ComposerError, + yaml.constructor.ConstructorError) as e: + print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") + sys.exit(1) + return yaml_dict \ No newline at end of file -- Gitee From 7fdf9ef40bede296c8c682ffc382b3369d82ba96 Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Wed, 27 Mar 2024 17:46:56 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E4=B8=8B=E8=BC=89=E5=BE=A9=E7=94=A8?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E7=9A=84handler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 2 +- tools/install_dependency/src/download_main.py | 36 ++++++++----------- .../src/handler/compress_dep.py | 20 +++++++++++ .../src/handler/gather_package.py | 9 ++--- tools/install_dependency/src/utils.py | 11 +++++- 5 files changed, 47 insertions(+), 31 deletions(-) create mode 100644 tools/install_dependency/src/handler/compress_dep.py diff --git a/build.sh b/build.sh index f0fd21d..de3e1ed 100644 --- a/build.sh +++ b/build.sh @@ -11,7 +11,7 @@ mkdir -p "${current_dir}"/build bash "${current_dir}"/component/build_component.sh -bash "${current_dir}"/tools/download_dependency/build_download.sh +bash "${current_dir}"/tools/install_dependency/build_download.sh bash "${current_dir}"/tools/install_dependency/build_install.sh diff --git a/tools/install_dependency/src/download_main.py b/tools/install_dependency/src/download_main.py index 9199cbb..11d27d3 100644 --- a/tools/install_dependency/src/download_main.py +++ b/tools/install_dependency/src/download_main.py @@ -5,13 +5,20 @@ import sys import shutil import tarfile import wget + from download import download_config -from download.download_utils import download_dependence_file, download_dependence +from download.download_utils import download_dependence_file from download.download_command_line import process_command_line, CommandLine +from handler.pipeline import PipeLine +from handler.base_yaml_check import BaseCheck +from handler.gather_package import GatherPackage +from handler.compress_dep import CompressDep from utils import read_yaml_file -from constant import URL, SHA256, SAVE_PATH, DEFAULT_PATH, FILE, DEPENDENCY_FILE, ROLE_LIST, ROLE_COMPONENT +from constant import URL, SHA256, SAVE_PATH, DEFAULT_PATH, FILE, INSTRUCTION + +PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] # A-FOT files BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" @@ -104,15 +111,6 @@ def download_iso(): return download_dependence_file("download file", shell_dict) -def generate_component_list(yaml_dict): - component_list = list() - for role in ROLE_LIST: - if role not in yaml_dict: - continue - component_list.extend(ROLE_COMPONENT[role]) - return list(set(component_list)) - - if __name__ == '__main__': try: process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", @@ -124,17 +122,11 @@ if __name__ == '__main__': print("Download iso failed.") sys.exit(0) config_dict = read_yaml_file(CommandLine.yaml_path) - ret = download_dependence(generate_component_list(config_dict)) - if ret: - print(f"Now compress dependencies to {DEPENDENCY_FILE}...") - with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: - tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) - - print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") - shutil.rmtree(DEFAULT_PATH) - print("-- Delete dependencies directory. --") - else: - print("-- Download dependencies failed. Please try execute download tool again. --") + config_dict[INSTRUCTION] = "default" + pipe = PipeLine(config_dict) + pipe.add_tail(*PIPELINE) + pipe.start() + except (KeyboardInterrupt, Exception) as e: print(f"\nDownload dependencies failed. {str(e)} Please try execute download tool again.") sys.exit(1) diff --git a/tools/install_dependency/src/handler/compress_dep.py b/tools/install_dependency/src/handler/compress_dep.py new file mode 100644 index 0000000..373bd57 --- /dev/null +++ b/tools/install_dependency/src/handler/compress_dep.py @@ -0,0 +1,20 @@ +import tarfile +import os +import shutil +from handler.handler_and_node import Handler +from constant import DEPENDENCY_FILE, DEFAULT_PATH + + +class CompressDep(Handler): + def handle(self, _): + try: + print(f"Now compress dependencies to {DEPENDENCY_FILE}...") + with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: + tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) + print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") + shutil.rmtree(DEFAULT_PATH) + print("-- Delete dependencies directory. --") + return True + except Exception as e: + print(str(e)) + return False diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index 5ee9528..94a6e35 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -4,6 +4,7 @@ import subprocess import constant from download.download_utils import download_dependence, component_collection_map, lkp_collection_map from handler.handler_and_node import Handler +from utils import generate_component_list LOGGER = logging.getLogger("install_dependency") @@ -13,16 +14,10 @@ class GatherPackage(Handler): super(GatherPackage, self).__init__() self.component_list = list() - def generate_component_list(self, data): - component_list = [] - for _, machine in data[constant.MACHINE].items(): - component_list.extend(machine.get_components()) - self.component_list = list(set(component_list)) - def handle(self, data) -> bool: LOGGER.debug("GatherPackage start!") component_collection_map.update(lkp_collection_map) - self.generate_component_list(data) + self.component_list = generate_component_list(data) try: self.check_dependency() diff --git a/tools/install_dependency/src/utils.py b/tools/install_dependency/src/utils.py index 551769b..501928a 100644 --- a/tools/install_dependency/src/utils.py +++ b/tools/install_dependency/src/utils.py @@ -1,6 +1,7 @@ import os import sys import yaml +from constant import ROLE_LIST, ROLE_COMPONENT MKDIR_TMP_DEVKITDEPENDENCIES_CMD = "mkdir -p /tmp/devkitdependencies" YUM_INSTALL_LKP_DEPENDENCIES_CMD = "yum install -y git wget rubygems" @@ -50,4 +51,12 @@ def read_yaml_file(yaml_path): yaml.constructor.ConstructorError) as e: print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") sys.exit(1) - return yaml_dict \ No newline at end of file + return yaml_dict + +def generate_component_list(yaml_dict): + component_list = list() + for role in ROLE_LIST: + if role not in yaml_dict: + continue + component_list.extend(ROLE_COMPONENT[role]) + return list(set(component_list)) -- Gitee