diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index 8a3321187a2f51d165c447310262203f1e17e966..6baba467031a89e85dffdee197ebd71a1e916ec3 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -5,6 +5,7 @@ import sys import shutil import tarfile import wget +import yaml import download_config from download_utils import download_dependence_handler, download_dependence_file from download_command_line import process_command_line, CommandLine @@ -26,6 +27,7 @@ AUTO_PREFETCH = "auto_prefetch.sh" SPLIT_JSON_PY = "split_json.py" FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) + component_collection_map = { component.get("component_name"): { "download file": @@ -56,28 +58,53 @@ lkp_collection_map = { URL: f"{download_config.LkpTests.get('gem dependency')}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", }, - }, - "CompatibilityTesting": { - "download file": { + "download CompatibilityTesting": { URL: f"{download_config.CompatibilityTesting.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", } }, } +SCANNER = "scanner" +BUILDER = "builder" +EXECUTOR = "executor" + +ROLE_COMPONENT = { + SCANNER: ["BiShengJDK17"], + BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + EXECUTOR: ["BiShengJDK17", "LkpTests"] +} + +ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] + + +def read_yaml_file(yaml_path): + try: + with open(yaml_path, "r") as file: + yaml_dict = yaml.safe_load(file) + except (FileNotFoundError, IsADirectoryError) as e: + print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") + sys.exit(1) + except (yaml.parser.ParserError, + yaml.scanner.ScannerError, + yaml.composer.ComposerError, + yaml.constructor.ConstructorError) as e: + print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") + sys.exit(1) + return yaml_dict -def download_dependence(): +def download_dependence(component_list): if not os.path.exists(DEFAULT_PATH): os.mkdir(DEFAULT_PATH) elif os.path.isfile(DEFAULT_PATH): print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") return False - else: - pass ret = True component_collection_map.update(lkp_collection_map) for component_name in component_collection_map: + if component_name not in component_list: + continue shell_dict = component_collection_map.get(component_name) ret = ret and download_dependence_handler(shell_dict) return ret @@ -163,6 +190,15 @@ def download_iso(): return download_dependence_file("download file", shell_dict) +def generate_component_list(yaml_dict): + component_list = list() + for role in ROLE_LIST: + if role not in yaml_dict: + continue + component_list.extend(ROLE_COMPONENT[role]) + return list(set(component_list)) + + if __name__ == '__main__': try: process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", @@ -173,8 +209,8 @@ if __name__ == '__main__': else: print("Download iso failed.") sys.exit(0) - - ret = download_dependence() + config_dict = read_yaml_file(CommandLine.yaml_path) + ret = download_dependence(generate_component_list(config_dict)) if ret: print(f"Now compress dependencies to {DEPENDENCY_FILE}...") with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: diff --git a/tools/download_dependency/src/download_command_line.py b/tools/download_dependency/src/download_command_line.py index 12cd1f5ea82b5fbcb19f9083987284aa2fa9ec52..0140818ab45e0f646156483e5d4d61192200faa7 100644 --- a/tools/download_dependency/src/download_command_line.py +++ b/tools/download_dependency/src/download_command_line.py @@ -1,12 +1,17 @@ import argparse import download_config +DEFAULT_YAML_PATH = "./machine.yaml" + class CommandLine: download_iso = None + yaml_path = DEFAULT_YAML_PATH @classmethod def add_options(cls, parser): + parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, + help="Assign yaml config file path. Default path is 'machine.yaml' in current directory.") parser.add_argument("-iso", action="store", dest="download_iso", default="", choices=[ component.get("component_name") for component in ( diff --git a/tools/download_dependency/src/download_utils.py b/tools/download_dependency/src/download_utils.py index 6f422176b0b2f382f1f478f357521a2a4668f69c..b51ca8b5a1482e838bc81b9328122f19563d6a0a 100644 --- a/tools/download_dependency/src/download_utils.py +++ b/tools/download_dependency/src/download_utils.py @@ -14,6 +14,8 @@ def download_dependence_handler(shell_dict): def download_dependence_file(shell_cmd, shell_dict): ret = True url_and_save_path = shell_dict.get(shell_cmd) + if os.path.exists(url_and_save_path.get('save_path')) and os.path.isfile(url_and_save_path.get('save_path')): + return ret try: print(f"Downloading from {url_and_save_path.get('url')}") download_result = wget.download( diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index fbe8e877e839fd92c71c149b8a8b224aa9507fc0..e1e852f44ee81ff0317304bbaeeb56610abefcc0 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -6,22 +6,16 @@ from handler.handler_and_node import Handler from machine.local_machine import LocalMachine from machine.klass_dict import KLASS_DICT from exception.connect_exception import ConnectException +from download import ROLE_COMPONENT LOGGER = logging.getLogger("install_dependency") -ROLE_COMPONENT = { - "scanner": ["BiShengJDK17"], - "builder": ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], - "executor": ["BiShengJDK17", "LkpTests"] -} - class ConnectCheck(Handler): - def handle(self, data) -> bool: LOGGER.debug("ConnectCheck start!") local_ip = ConnectCheck.get_local_ip() - + data[constant.MACHINE] = dict() ret = True for role in (set(KLASS_DICT.keys()) & set(data.keys())): ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) @@ -31,7 +25,6 @@ class ConnectCheck(Handler): def machine_role_check(data, role, local_ip): builder_list = data.get(role) klass = KLASS_DICT.get(role) - data[constant.MACHINE] = dict() for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index edaed4e79e8cb48b8657e353bbb4a19cc0dcc6dd..714525c1dd55439a91204f36dfbe08bbb8530a83 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -9,32 +9,28 @@ LOGGER = logging.getLogger("install_dependency") class GatherPackage(Handler): + def __init__(self): + super(GatherPackage, self).__init__() + self.component_list = list() - def handle(self, data) -> bool: - instruction_to_func_dict = { - "deploy_iso": GatherPackage.deploy_iso_handle, - "default": GatherPackage.default_handle, - } - return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))() - - @staticmethod - def deploy_iso_handle(): - LOGGER.info("Iso file already checked.") - return GatherPackage.default_handle() + def generate_component_list(self, data): + component_list = [] + for _, machine in data[constant.MACHINE].items(): + component_list.extend(machine.get_components()) + self.component_list = list(set(component_list)) - @staticmethod - def default_handle(): + def handle(self, data) -> bool: LOGGER.debug("GatherPackage start!") - if GatherPackage.check_default_path_available(): - LOGGER.info("Dependencies ready.") - return True + component_collection_map.update(lkp_collection_map) + self.generate_component_list(data) - if os.path.isfile(constant.DEPENDENCY_DIR): - LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + try: + self.check_dependency() + except Exception as e: return False try: - ret = download_dependence() + ret = download_dependence(self.component_list) except Exception as e: LOGGER.error(f"Download dependencies failed. {str(e)}. Please execute download tool.") return False @@ -45,7 +41,7 @@ class GatherPackage(Handler): return True @staticmethod - def check_default_path_available(): + def check_dependency(): if os.path.exists(constant.DEPENDENCY_FILE): try: print(f"Now extract files from {constant.DEPENDENCY_FILE}:") @@ -55,18 +51,7 @@ class GatherPackage(Handler): LOGGER.warning(f"{constant.DEPENDENCY_FILE} may already extracted.") except Exception as e: LOGGER.error(f"Extract {constant.DEPENDENCY_FILE} failed. {str(e)}") - return False - - if not os.path.isdir(constant.DEPENDENCY_DIR): - LOGGER.warning(f"The directory {constant.DEPENDENCY_DIR} not exists.") - return False - component_collection_map.update(lkp_collection_map) - for component_name in component_collection_map: - shell_dict = component_collection_map.get(component_name) - for shell_cmd in shell_dict: - url_and_save_path = shell_dict.get(shell_cmd) - component = url_and_save_path.get("save_path") - if not os.path.isfile(component): - LOGGER.warning(f"The file {component} not exists.") - return False - return True + raise + if os.path.isfile(constant.DEPENDENCY_DIR): + LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + raise Exception diff --git a/tools/install_dependency/src/install.py b/tools/install_dependency/src/install.py index 62b77b5589ffe2e6276bb3c0bfceee0f8912ea16..ca320daf2843c591982d0862071025fc2fcdfee2 100644 --- a/tools/install_dependency/src/install.py +++ b/tools/install_dependency/src/install.py @@ -2,7 +2,7 @@ import os import subprocess import sys import logging -import yaml + import constant from log import config_logging @@ -13,28 +13,13 @@ from handler.base_yaml_check import BaseCheck from handler.connect_check import ConnectCheck from handler.gather_package import GatherPackage from handler.install_package import InstallPackage +from download import read_yaml_file LOGGER = logging.getLogger("install_dependency") PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] ISO_VERIFY_FLAG_STRING = "ISO 9660 CD-ROM filesystem data" -def read_yaml_file(yaml_path): - try: - with open(yaml_path, "r") as file: - config_dict = yaml.safe_load(file) - except (FileNotFoundError, IsADirectoryError) as e: - LOGGER.error(f"Yaml file is not in specified path. Error: {str(e)}") - sys.exit(1) - except (yaml.parser.ParserError, - yaml.scanner.ScannerError, - yaml.composer.ComposerError, - yaml.constructor.ConstructorError) as e: - LOGGER.error(f"Incorrect yaml file. Error: {str(e)}") - sys.exit(1) - return config_dict - - def check_iso_available(iso_path): if not os.path.isfile(iso_path): LOGGER.error(f"ISO file is not in specified path. Error: {iso_path} file not found.") diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 8e46f6f8c72a0d189d85a6ca8e2442b260da97d2..f12f9c79fd9a0835a7971b0c32022ece97037351 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -28,6 +28,9 @@ class LocalMachine: self.component_list.extend(component) self.component_list = list(set(self.component_list)) + def get_components(self): + return self.component_list.copy() + def install_components(self): if self.mirrors: self.install_component("OpenEulerMirrorISO") diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index 31300fc4518c311d0dac04f9d4e7081fe869e30e..a08d9d915d8fec54dfe35669860f67228133ff62 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -34,6 +34,9 @@ class Machine: self.component_list.extend(component) self.component_list = list(set(self.component_list)) + def get_components(self): + return self.component_list.copy() + def set_mirror(self): self.mirrors = True