diff --git a/build.sh b/build.sh index f0fd21d4540b39d451c887f6669f10469c7cb9a8..de3e1edda3ef760806b8c45e26aa426f39dd6e22 100644 --- a/build.sh +++ b/build.sh @@ -11,7 +11,7 @@ mkdir -p "${current_dir}"/build bash "${current_dir}"/component/build_component.sh -bash "${current_dir}"/tools/download_dependency/build_download.sh +bash "${current_dir}"/tools/install_dependency/build_download.sh bash "${current_dir}"/tools/install_dependency/build_install.sh diff --git a/tools/download_dependency/build_download.sh b/tools/install_dependency/build_download.sh similarity index 77% rename from tools/download_dependency/build_download.sh rename to tools/install_dependency/build_download.sh index 8f141cfb039c5b2e7b44367cdc2a74572bae64a1..0201b05bd23da8ef44de2b7f62a2686b813091dd 100644 --- a/tools/download_dependency/build_download.sh +++ b/tools/install_dependency/build_download.sh @@ -14,4 +14,4 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src --name "download_tool" +pyinstaller -F "${current_dir}"/src/download_main.py -p "${current_dir}"/src --name "download_tool" diff --git a/tools/install_dependency/build_install.sh b/tools/install_dependency/build_install.sh index e9fcc7beee372073ae3917dff0da9f8bc0047918..f2b7a7fd13ad0cc80d425e02eaf4e76dfbee4acf 100644 --- a/tools/install_dependency/build_install.sh +++ b/tools/install_dependency/build_install.sh @@ -14,6 +14,6 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/install.py -p "${current_dir}/src:${project_dir}/tools/download_dependency/src" --add-data "../../build/component:component" --name "deploy_tool" +pyinstaller -F "${current_dir}"/src/deploy_main.py -p "${current_dir}/src" --add-data "../../build/component:component" --name "deploy_tool" cp "${current_dir}"/config/machine.yaml "${build_dir}"/dist/machine.yaml diff --git a/tools/install_dependency/src/__init__.py b/tools/install_dependency/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index 5bc23c505d398536376cbd64516c5c432c6f8a7b..44aa346583a5e4f20d6d16daaa2836fdcf03334e 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -10,3 +10,17 @@ DEPENDENCY_FILE = "devkitdependencies.tar.gz" DEPENDENCY_DIR = "devkitdependencies" INSTRUCTION = "instruction" + +ROLE_COMPONENT = { + SCANNER: ["BiShengJDK17"], + BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + EXECUTOR: ["BiShengJDK17", "LkpTests"] +} + +ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] + +FILE = "file" +SHA256 = "sha256" +URL = "url" +SAVE_PATH = "save_path" +DEFAULT_PATH = "./devkitdependencies" diff --git a/tools/install_dependency/src/deploy/__init__.py b/tools/install_dependency/src/deploy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/deploy/command_line.py similarity index 100% rename from tools/install_dependency/src/command_line.py rename to tools/install_dependency/src/deploy/command_line.py diff --git a/tools/install_dependency/src/lkp_collect_map.py b/tools/install_dependency/src/deploy/lkp_collect_map.py similarity index 96% rename from tools/install_dependency/src/lkp_collect_map.py rename to tools/install_dependency/src/deploy/lkp_collect_map.py index 2303e2c1f6b2d3e14cbb0f2f8bbace1bdac5fb92..e06a4f008aad8569bcd67cf985b93d66e5f2a7cd 100644 --- a/tools/install_dependency/src/lkp_collect_map.py +++ b/tools/install_dependency/src/deploy/lkp_collect_map.py @@ -1,26 +1,26 @@ -import os.path - -from utils import base_path - -CURRENT_DEFAULT_PATH = "./devkitdependencies" - -lkp_collection_map = { - "LkpTests": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - }, - "CompatibilityTesting": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, - "DevkitDistribute": { - "download file": { - "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", - } - } -} +import os.path + +from utils import base_path + +CURRENT_DEFAULT_PATH = "./devkitdependencies" + +lkp_collection_map = { + "LkpTests": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", + }, + "download gem dependency": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", + }, + }, + "CompatibilityTesting": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + } + }, + "DevkitDistribute": { + "download file": { + "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", + } + } +} diff --git a/tools/install_dependency/src/install.py b/tools/install_dependency/src/deploy_main.py similarity index 95% rename from tools/install_dependency/src/install.py rename to tools/install_dependency/src/deploy_main.py index ca320daf2843c591982d0862071025fc2fcdfee2..a406e3a56023d757a4c021e9cf4bd122a7341085 100644 --- a/tools/install_dependency/src/install.py +++ b/tools/install_dependency/src/deploy_main.py @@ -6,14 +6,14 @@ import logging import constant from log import config_logging -from command_line import process_command_line, CommandLine +from deploy.command_line import process_command_line, CommandLine from handler.pipeline import PipeLine from handler.base_yaml_check import BaseCheck from handler.connect_check import ConnectCheck from handler.gather_package import GatherPackage from handler.install_package import InstallPackage -from download import read_yaml_file +from utils import read_yaml_file LOGGER = logging.getLogger("install_dependency") PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] diff --git a/tools/install_dependency/src/download/__init__.py b/tools/install_dependency/src/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/download_dependency/src/download_command_line.py b/tools/install_dependency/src/download/download_command_line.py similarity index 98% rename from tools/download_dependency/src/download_command_line.py rename to tools/install_dependency/src/download/download_command_line.py index 0140818ab45e0f646156483e5d4d61192200faa7..d4b1cc88ad6e18d08d2f803d905e63dcf07ba516 100644 --- a/tools/download_dependency/src/download_command_line.py +++ b/tools/install_dependency/src/download/download_command_line.py @@ -1,5 +1,5 @@ import argparse -import download_config +from download import download_config DEFAULT_YAML_PATH = "./machine.yaml" diff --git a/tools/download_dependency/src/download_config.py b/tools/install_dependency/src/download/download_config.py similarity index 100% rename from tools/download_dependency/src/download_config.py rename to tools/install_dependency/src/download/download_config.py diff --git a/tools/download_dependency/src/download_utils.py b/tools/install_dependency/src/download/download_utils.py similarity index 39% rename from tools/download_dependency/src/download_utils.py rename to tools/install_dependency/src/download/download_utils.py index b51ca8b5a1482e838bc81b9328122f19563d6a0a..f3f6510c451dfd38ba1b5a4c63a5cd9fd0c0ee55 100644 --- a/tools/download_dependency/src/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -1,7 +1,63 @@ import os -import sys import urllib.error import wget +from download import download_config +from constant import URL, FILE, SAVE_PATH, SHA256, DEFAULT_PATH + + +component_collection_map = { + component.get("component_name"): { + "download file": + { + URL: f"{component.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}" + }, + "download sha256": + { + URL: f"{component.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" + } + } for component in ( + download_config.BiShengCompiler, + download_config.GCCforOpenEuler, + download_config.BiShengJDK8, + download_config.BiShengJDK17, + ) +} + +lkp_collection_map = { + "LkpTests": { + "download file": { + URL: f"{download_config.LkpTests.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'lkp-tests.tar.gz')}", + }, + "download gem dependency": { + URL: f"{download_config.LkpTests.get('gem dependency')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", + }, + "download CompatibilityTesting": { + URL: f"{download_config.CompatibilityTesting.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + } + }, +} + + +def download_dependence(component_list): + if not os.path.exists(DEFAULT_PATH): + os.mkdir(DEFAULT_PATH) + elif os.path.isfile(DEFAULT_PATH): + print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") + return False + + ret = True + component_collection_map.update(lkp_collection_map) + for component_name in component_collection_map: + if component_name not in component_list: + continue + shell_dict = component_collection_map.get(component_name) + ret = ret and download_dependence_handler(shell_dict) + return ret def download_dependence_handler(shell_dict): diff --git a/tools/download_dependency/src/download.py b/tools/install_dependency/src/download_main.py similarity index 48% rename from tools/download_dependency/src/download.py rename to tools/install_dependency/src/download_main.py index 6baba467031a89e85dffdee197ebd71a1e916ec3..11d27d3506b1bec3a416d489e090816b457aa720 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/install_dependency/src/download_main.py @@ -5,17 +5,20 @@ import sys import shutil import tarfile import wget -import yaml -import download_config -from download_utils import download_dependence_handler, download_dependence_file -from download_command_line import process_command_line, CommandLine - -FILE = "file" -SHA256 = "sha256" -URL = "url" -SAVE_PATH = "save_path" -DEFAULT_PATH = "./devkitdependencies" -DEPENDENCY_FILE = "devkitdependencies.tar.gz" + +from download import download_config +from download.download_utils import download_dependence_file +from download.download_command_line import process_command_line, CommandLine +from handler.pipeline import PipeLine +from handler.base_yaml_check import BaseCheck +from handler.gather_package import GatherPackage +from handler.compress_dep import CompressDep + +from utils import read_yaml_file +from constant import URL, SHA256, SAVE_PATH, DEFAULT_PATH, FILE, INSTRUCTION + + +PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] # A-FOT files BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" @@ -28,88 +31,6 @@ SPLIT_JSON_PY = "split_json.py" FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) -component_collection_map = { - component.get("component_name"): { - "download file": - { - URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}" - }, - "download sha256": - { - URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" - } - } for component in ( - download_config.BiShengCompiler, - download_config.GCCforOpenEuler, - download_config.BiShengJDK8, - download_config.BiShengJDK17, - ) -} - -lkp_collection_map = { - "LkpTests": { - "download file": { - URL: f"{download_config.LkpTests.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - URL: f"{download_config.LkpTests.get('gem dependency')}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - "download CompatibilityTesting": { - URL: f"{download_config.CompatibilityTesting.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, -} -SCANNER = "scanner" -BUILDER = "builder" -EXECUTOR = "executor" - -ROLE_COMPONENT = { - SCANNER: ["BiShengJDK17"], - BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], - EXECUTOR: ["BiShengJDK17", "LkpTests"] -} - -ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] - - -def read_yaml_file(yaml_path): - try: - with open(yaml_path, "r") as file: - yaml_dict = yaml.safe_load(file) - except (FileNotFoundError, IsADirectoryError) as e: - print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") - sys.exit(1) - except (yaml.parser.ParserError, - yaml.scanner.ScannerError, - yaml.composer.ComposerError, - yaml.constructor.ConstructorError) as e: - print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") - sys.exit(1) - return yaml_dict - - -def download_dependence(component_list): - if not os.path.exists(DEFAULT_PATH): - os.mkdir(DEFAULT_PATH) - elif os.path.isfile(DEFAULT_PATH): - print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") - return False - - ret = True - component_collection_map.update(lkp_collection_map) - for component_name in component_collection_map: - if component_name not in component_list: - continue - shell_dict = component_collection_map.get(component_name) - ret = ret and download_dependence_handler(shell_dict) - return ret - - def download_a_fot(): saved_path = os.path.join(DEFAULT_PATH, A_FOT) try: @@ -136,12 +57,12 @@ iso_collection_map = { "download file": { URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join('./', component.get(FILE).split('/')[-1])}" + SAVE_PATH: f"{os.path.join('/', component.get(FILE).split('/')[-1])}" }, "download sha256": { URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join('./', component.get(SHA256).split('/')[-1])}" + SAVE_PATH: f"{os.path.join('/', component.get(SHA256).split('/')[-1])}" } } for component in ( download_config.OpenEuler_2003_LTS, @@ -190,15 +111,6 @@ def download_iso(): return download_dependence_file("download file", shell_dict) -def generate_component_list(yaml_dict): - component_list = list() - for role in ROLE_LIST: - if role not in yaml_dict: - continue - component_list.extend(ROLE_COMPONENT[role]) - return list(set(component_list)) - - if __name__ == '__main__': try: process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", @@ -210,17 +122,11 @@ if __name__ == '__main__': print("Download iso failed.") sys.exit(0) config_dict = read_yaml_file(CommandLine.yaml_path) - ret = download_dependence(generate_component_list(config_dict)) - if ret: - print(f"Now compress dependencies to {DEPENDENCY_FILE}...") - with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: - tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) - - print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") - shutil.rmtree(DEFAULT_PATH) - print("-- Delete dependencies directory. --") - else: - print("-- Download dependencies failed. Please try execute download tool again. --") + config_dict[INSTRUCTION] = "default" + pipe = PipeLine(config_dict) + pipe.add_tail(*PIPELINE) + pipe.start() + except (KeyboardInterrupt, Exception) as e: print(f"\nDownload dependencies failed. {str(e)} Please try execute download tool again.") sys.exit(1) diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 748a8b36c9e48a829e25f6b730ad036a1076b6a5..eac596c14670fc698c23ee18e6928e56e4aa3376 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -2,7 +2,8 @@ import re import logging import constant from handler.handler_and_node import Handler -from machine.klass_dict import KLASS_DICT +from constant import ROLE_LIST + LOGGER = logging.getLogger("install_dependency") MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) @@ -48,11 +49,12 @@ class BaseCheck(Handler): @staticmethod def check_machine_ip(data): - for machine_type in (set(KLASS_DICT.keys()) & set(data.keys())): - if not data.get(machine_type) or not isinstance(data.get(machine_type), list): - LOGGER.error(f"Yaml file content not correct. Yaml file {machine_type} value not sequence.") + for role in (ROLE_LIST & data.keys()): + ip_list = data.get(role, []) + if not isinstance(ip_list, list): + LOGGER.error(f"Yaml file content not correct. Yaml file {role} value not sequence.") return False - for ip in data.get(machine_type): + for ip in ip_list: if not BaseCheck.validate_ip(ip): LOGGER.error(f"Yaml file content not correct. Given ip: {ip} not correct.") return False diff --git a/tools/install_dependency/src/handler/compress_dep.py b/tools/install_dependency/src/handler/compress_dep.py new file mode 100644 index 0000000000000000000000000000000000000000..373bd57882a78e9b0948924299dfb0d433c563c0 --- /dev/null +++ b/tools/install_dependency/src/handler/compress_dep.py @@ -0,0 +1,20 @@ +import tarfile +import os +import shutil +from handler.handler_and_node import Handler +from constant import DEPENDENCY_FILE, DEFAULT_PATH + + +class CompressDep(Handler): + def handle(self, _): + try: + print(f"Now compress dependencies to {DEPENDENCY_FILE}...") + with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: + tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) + print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") + shutil.rmtree(DEFAULT_PATH) + print("-- Delete dependencies directory. --") + return True + except Exception as e: + print(str(e)) + return False diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index e1e852f44ee81ff0317304bbaeeb56610abefcc0..02cf7c5cbeb979ae4b879a678b909fde0e78c9f5 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -4,9 +4,9 @@ import socket import constant from handler.handler_and_node import Handler from machine.local_machine import LocalMachine -from machine.klass_dict import KLASS_DICT +from machine.machine import Machine from exception.connect_exception import ConnectException -from download import ROLE_COMPONENT +from constant import ROLE_COMPONENT, ROLE_LIST LOGGER = logging.getLogger("install_dependency") @@ -17,14 +17,13 @@ class ConnectCheck(Handler): local_ip = ConnectCheck.get_local_ip() data[constant.MACHINE] = dict() ret = True - for role in (set(KLASS_DICT.keys()) & set(data.keys())): + for role in (ROLE_LIST & data.keys()): ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) return ret @staticmethod def machine_role_check(data, role, local_ip): builder_list = data.get(role) - klass = KLASS_DICT.get(role) for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" @@ -33,8 +32,8 @@ class ConnectCheck(Handler): data[constant.MACHINE][ip] = machine_instance continue try: - machine_instance = data[constant.MACHINE].get(ip, klass(ip, data[constant.USER], data[constant.PKEY], - data.get(constant.PASSWORD, None))) + machine_instance = data[constant.MACHINE].get(ip, Machine(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None))) machine_instance.add_component(ROLE_COMPONENT[role]) data[constant.MACHINE][ip] = machine_instance except ConnectException: diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index 714525c1dd55439a91204f36dfbe08bbb8530a83..94a6e354bf6086d26a0ca5fd3dfef92f5ff07ea9 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -2,8 +2,9 @@ import logging import os import subprocess import constant -from download import download_dependence, component_collection_map, lkp_collection_map +from download.download_utils import download_dependence, component_collection_map, lkp_collection_map from handler.handler_and_node import Handler +from utils import generate_component_list LOGGER = logging.getLogger("install_dependency") @@ -13,16 +14,10 @@ class GatherPackage(Handler): super(GatherPackage, self).__init__() self.component_list = list() - def generate_component_list(self, data): - component_list = [] - for _, machine in data[constant.MACHINE].items(): - component_list.extend(machine.get_components()) - self.component_list = list(set(component_list)) - def handle(self, data) -> bool: LOGGER.debug("GatherPackage start!") component_collection_map.update(lkp_collection_map) - self.generate_component_list(data) + self.component_list = generate_component_list(data) try: self.check_dependency() diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py deleted file mode 100644 index 8ee4479e37a5800eaf9f829d1115739e3fb82120..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/builder_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class BuilderMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(BuilderMachine, self).__init__(ip, user, pkey, password) - self.role = constant.BUILDER diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py deleted file mode 100644 index 35147f46c3d95cb5da27fd0c67a577e04b794f53..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class DevkitMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(DevkitMachine, self).__init__(ip, user, pkey, password) - self.role = constant.DEVKIT diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py deleted file mode 100644 index 923c7a133a7a743828700e73da9e49ee1a969874..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/executor_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ExecutorMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ExecutorMachine, self).__init__(ip, user, pkey, password) - self.role = constant.EXECUTOR diff --git a/tools/install_dependency/src/machine/klass_dict.py b/tools/install_dependency/src/machine/klass_dict.py deleted file mode 100644 index 835b9994cdc1a0325d7d137f1f384de095dcc2b8..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/klass_dict.py +++ /dev/null @@ -1,12 +0,0 @@ -import constant -from machine.scanner_machine import ScannerMachine -from machine.builder_machine import BuilderMachine -from machine.executor_machine import ExecutorMachine -from machine.devkit_machine import DevkitMachine - -KLASS_DICT = { - constant.EXECUTOR: ExecutorMachine, - constant.DEVKIT: DevkitMachine, - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, -} diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index f12f9c79fd9a0835a7971b0c32022ece97037351..d4e5077a5900eeeb31fefdb4157e1ac6bb78cc9f 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -3,10 +3,10 @@ import logging import subprocess import constant -from command_line import CommandLine +from deploy.command_line import CommandLine from exception.connect_exception import NotMatchedMachineTypeException -from download import component_collection_map -from lkp_collect_map import lkp_collection_map +from download.download_utils import component_collection_map +from deploy.lkp_collect_map import lkp_collection_map from utils import (base_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, CHECK_MIRROR_INSTALL_STATUS, PROMPT_MAP) diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index a08d9d915d8fec54dfe35669860f67228133ff62..f64835e18ed86b690f883018cb0cbef1f719c44c 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -7,11 +7,11 @@ import paramiko import timeout_decorator import constant -from command_line import CommandLine +from deploy.command_line import CommandLine from exception.connect_exception import (CreatePkeyFailedException, ConnectRemoteException, NotMatchedMachineTypeException) -from download import component_collection_map -from lkp_collect_map import lkp_collection_map +from download.download_utils import component_collection_map +from deploy.lkp_collect_map import lkp_collection_map from utils import (base_path, validate_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, CHECK_MIRROR_INSTALL_STATUS, PROMPT_MAP) diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py deleted file mode 100644 index 9377b527f8b9a0507156b71be16f3ec62d97c2e0..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ScannerMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ScannerMachine, self).__init__(ip, user, pkey, password) - self.role = constant.SCANNER diff --git a/tools/install_dependency/src/utils.py b/tools/install_dependency/src/utils.py index d249129dd2352a28d71133d6292f9d52d39126ae..501928ac2effe0a00cb77f532a400e8e66c51862 100644 --- a/tools/install_dependency/src/utils.py +++ b/tools/install_dependency/src/utils.py @@ -1,5 +1,7 @@ import os import sys +import yaml +from constant import ROLE_LIST, ROLE_COMPONENT MKDIR_TMP_DEVKITDEPENDENCIES_CMD = "mkdir -p /tmp/devkitdependencies" YUM_INSTALL_LKP_DEPENDENCIES_CMD = "yum install -y git wget rubygems" @@ -34,3 +36,27 @@ def available_role(role_list, data): if role in data: list_after_verification.append(role) return list_after_verification + + +def read_yaml_file(yaml_path): + try: + with open(yaml_path, "r") as file: + yaml_dict = yaml.safe_load(file) + except (FileNotFoundError, IsADirectoryError) as e: + print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") + sys.exit(1) + except (yaml.parser.ParserError, + yaml.scanner.ScannerError, + yaml.composer.ComposerError, + yaml.constructor.ConstructorError) as e: + print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") + sys.exit(1) + return yaml_dict + +def generate_component_list(yaml_dict): + component_list = list() + for role in ROLE_LIST: + if role not in yaml_dict: + continue + component_list.extend(ROLE_COMPONENT[role]) + return list(set(component_list))