diff --git a/tools/download_dependency/src/devkitdependencies.tar.gz b/tools/download_dependency/src/devkitdependencies.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..d4cecd9803228b81095b2b638134e9fd242a8dfa Binary files /dev/null and b/tools/download_dependency/src/devkitdependencies.tar.gz differ diff --git a/tools/install_dependency/src/deploy/__init__.py b/tools/install_dependency/src/deploy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/install_dependency/src/command_line.py b/tools/install_dependency/src/deploy/command_line.py similarity index 100% rename from tools/install_dependency/src/command_line.py rename to tools/install_dependency/src/deploy/command_line.py diff --git a/tools/install_dependency/src/lkp_collect_map.py b/tools/install_dependency/src/deploy/lkp_collect_map.py similarity index 96% rename from tools/install_dependency/src/lkp_collect_map.py rename to tools/install_dependency/src/deploy/lkp_collect_map.py index 2303e2c1f6b2d3e14cbb0f2f8bbace1bdac5fb92..e06a4f008aad8569bcd67cf985b93d66e5f2a7cd 100644 --- a/tools/install_dependency/src/lkp_collect_map.py +++ b/tools/install_dependency/src/deploy/lkp_collect_map.py @@ -1,26 +1,26 @@ -import os.path - -from utils import base_path - -CURRENT_DEFAULT_PATH = "./devkitdependencies" - -lkp_collection_map = { - "LkpTests": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - }, - "CompatibilityTesting": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, - "DevkitDistribute": { - "download file": { - "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", - } - } -} +import os.path + +from utils import base_path + +CURRENT_DEFAULT_PATH = "./devkitdependencies" + +lkp_collection_map = { + "LkpTests": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", + }, + "download gem dependency": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", + }, + }, + "CompatibilityTesting": { + "download file": { + "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + } + }, + "DevkitDistribute": { + "download file": { + "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", + } + } +} diff --git a/tools/install_dependency/src/deploy_main.py b/tools/install_dependency/src/deploy_main.py new file mode 100644 index 0000000000000000000000000000000000000000..a406e3a56023d757a4c021e9cf4bd122a7341085 --- /dev/null +++ b/tools/install_dependency/src/deploy_main.py @@ -0,0 +1,57 @@ +import os +import subprocess +import sys +import logging + + +import constant +from log import config_logging +from deploy.command_line import process_command_line, CommandLine + +from handler.pipeline import PipeLine +from handler.base_yaml_check import BaseCheck +from handler.connect_check import ConnectCheck +from handler.gather_package import GatherPackage +from handler.install_package import InstallPackage +from utils import read_yaml_file + +LOGGER = logging.getLogger("install_dependency") +PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] +ISO_VERIFY_FLAG_STRING = "ISO 9660 CD-ROM filesystem data" + + +def check_iso_available(iso_path): + if not os.path.isfile(iso_path): + LOGGER.error(f"ISO file is not in specified path. Error: {iso_path} file not found.") + sys.exit(1) + try: + result = subprocess.run(f"file -b {iso_path}".split(' '), + capture_output=True, shell=False) + output = result.stdout.decode().strip() + if output.find(ISO_VERIFY_FLAG_STRING) == -1: + LOGGER.error(f"Verify iso result: Not available. Please re-download iso file.") + sys.exit(1) + except (FileNotFoundError, IsADirectoryError, PermissionError, Exception) as e: + LOGGER.error(f"Verify iso file integrity occur error: {str(e)}") + sys.exit(1) + + +if __name__ == '__main__': + try: + process_command_line(program="install_dependency", description="devkit-pipeline install_dependency tool", + class_list=[CommandLine]) + config_logging(CommandLine.debug) + config_dict = read_yaml_file(CommandLine.yaml_path) + + if CommandLine.iso_path: + config_dict[constant.INSTRUCTION] = "deploy_iso" + check_iso_available(CommandLine.iso_path) + else: + config_dict[constant.INSTRUCTION] = "default" + LOGGER.debug(f"-- config_dict: {config_dict}") + + pipe = PipeLine(config_dict) + pipe.add_tail(*PIPELINE) + pipe.start() + except (KeyboardInterrupt, Exception) as e: + print(f"[warning] Program Exited. {str(e)}") diff --git a/tools/download_dependency/src/download.py b/tools/install_dependency/src/download.py similarity index 100% rename from tools/download_dependency/src/download.py rename to tools/install_dependency/src/download.py diff --git a/tools/install_dependency/src/download/__init__.py b/tools/install_dependency/src/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/download_dependency/src/download_command_line.py b/tools/install_dependency/src/download/download_command_line.py similarity index 100% rename from tools/download_dependency/src/download_command_line.py rename to tools/install_dependency/src/download/download_command_line.py diff --git a/tools/download_dependency/src/download_config.py b/tools/install_dependency/src/download/download_config.py similarity index 100% rename from tools/download_dependency/src/download_config.py rename to tools/install_dependency/src/download/download_config.py diff --git a/tools/download_dependency/src/download_utils.py b/tools/install_dependency/src/download/download_utils.py similarity index 100% rename from tools/download_dependency/src/download_utils.py rename to tools/install_dependency/src/download/download_utils.py diff --git a/tools/install_dependency/src/download_main.py b/tools/install_dependency/src/download_main.py new file mode 100644 index 0000000000000000000000000000000000000000..48819ba5ac255f586fa71009f389fef9e9b53d1a --- /dev/null +++ b/tools/install_dependency/src/download_main.py @@ -0,0 +1,70 @@ +import os +import sys +import shutil +import tarfile +import wget +from download.download_command_line import process_command_line, CommandLine +from download.download_utils import download_iso, download_dependence +from utils import read_yaml_file, generate_component_list + + +DEFAULT_PATH = "./devkitdependencies" +DEPENDENCY_FILE = "devkitdependencies.tar.gz" + +# A-FOT files +BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" +A_FOT = "a-fot" +A_FOT_INI = "a-fot.ini" +AUTO_FDO_SH = "auto_fdo.sh" +AUTO_BOLT_SH = "auto_bolt.sh" +AUTO_PREFETCH = "auto_prefetch.sh" +SPLIT_JSON_PY = "split_json.py" +FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) + + +def download_a_fot(): + saved_path = os.path.join(DEFAULT_PATH, A_FOT) + try: + os.mkdir(saved_path) + except FileExistsError as e: + pass + + try: + for f in FILE_LIST: + wget.download(BASE_URL.format(f), os.path.join(saved_path, f)) + + with tarfile.open(os.path.join(DEFAULT_PATH, "a-fot.tar.gz"), "w:gz") as t: + t.add(saved_path, arcname="a-fot") + return True + except Exception as e: + print(e) + return False + finally: + shutil.rmtree(saved_path) + + +if __name__ == '__main__': + try: + process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", + class_list=[CommandLine]) + if CommandLine.download_iso: + if download_iso(): + print("-- Download iso success. --") + else: + print("Download iso failed.") + sys.exit(0) + config_dict = read_yaml_file(CommandLine.yaml_path) + ret = download_dependence(generate_component_list(config_dict)) + if ret: + print(f"Now compress dependencies to {DEPENDENCY_FILE}...") + with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: + tar.add(DEFAULT_PATH, arcname=os.path.basename(DEFAULT_PATH)) + + print(f"-- Compress dependencies to {DEPENDENCY_FILE} success. --") + shutil.rmtree(DEFAULT_PATH) + print("-- Delete dependencies directory. --") + else: + print("-- Download dependencies failed. Please try execute download tool again. --") + except (KeyboardInterrupt, Exception) as e: + print(f"\nDownload dependencies failed. {str(e)} Please try execute download tool again.") + sys.exit(1) diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 748a8b36c9e48a829e25f6b730ad036a1076b6a5..57028aefa13973714fb75e856b9d56e7190fe0e9 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -2,7 +2,8 @@ import re import logging import constant from handler.handler_and_node import Handler -from machine.klass_dict import KLASS_DICT +from download import ROLE_LIST + LOGGER = logging.getLogger("install_dependency") MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) @@ -48,11 +49,12 @@ class BaseCheck(Handler): @staticmethod def check_machine_ip(data): - for machine_type in (set(KLASS_DICT.keys()) & set(data.keys())): - if not data.get(machine_type) or not isinstance(data.get(machine_type), list): - LOGGER.error(f"Yaml file content not correct. Yaml file {machine_type} value not sequence.") + for role in ROLE_LIST: + ip_list = data.get(role, []) + if not isinstance(ip_list, list) or not ip_list: + LOGGER.error(f"Yaml file content not correct. Yaml file {role} value not sequence.") return False - for ip in data.get(machine_type): + for ip in ip_list: if not BaseCheck.validate_ip(ip): LOGGER.error(f"Yaml file content not correct. Given ip: {ip} not correct.") return False diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index e1e852f44ee81ff0317304bbaeeb56610abefcc0..66c62726bb1f8bb21486c05a9ee1b1bd9c626a5e 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -4,9 +4,9 @@ import socket import constant from handler.handler_and_node import Handler from machine.local_machine import LocalMachine -from machine.klass_dict import KLASS_DICT +from machine.machine import Machine from exception.connect_exception import ConnectException -from download import ROLE_COMPONENT +from download import ROLE_COMPONENT, ROLE_LIST LOGGER = logging.getLogger("install_dependency") @@ -17,14 +17,13 @@ class ConnectCheck(Handler): local_ip = ConnectCheck.get_local_ip() data[constant.MACHINE] = dict() ret = True - for role in (set(KLASS_DICT.keys()) & set(data.keys())): + for role in ROLE_LIST: ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) return ret @staticmethod def machine_role_check(data, role, local_ip): builder_list = data.get(role) - klass = KLASS_DICT.get(role) for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" @@ -33,8 +32,8 @@ class ConnectCheck(Handler): data[constant.MACHINE][ip] = machine_instance continue try: - machine_instance = data[constant.MACHINE].get(ip, klass(ip, data[constant.USER], data[constant.PKEY], - data.get(constant.PASSWORD, None))) + machine_instance = data[constant.MACHINE].get(ip, Machine(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None))) machine_instance.add_component(ROLE_COMPONENT[role]) data[constant.MACHINE][ip] = machine_instance except ConnectException: diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py deleted file mode 100644 index 8ee4479e37a5800eaf9f829d1115739e3fb82120..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/builder_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class BuilderMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(BuilderMachine, self).__init__(ip, user, pkey, password) - self.role = constant.BUILDER diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py deleted file mode 100644 index 35147f46c3d95cb5da27fd0c67a577e04b794f53..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class DevkitMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(DevkitMachine, self).__init__(ip, user, pkey, password) - self.role = constant.DEVKIT diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py deleted file mode 100644 index 923c7a133a7a743828700e73da9e49ee1a969874..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/executor_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ExecutorMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ExecutorMachine, self).__init__(ip, user, pkey, password) - self.role = constant.EXECUTOR diff --git a/tools/install_dependency/src/machine/klass_dict.py b/tools/install_dependency/src/machine/klass_dict.py deleted file mode 100644 index 835b9994cdc1a0325d7d137f1f384de095dcc2b8..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/klass_dict.py +++ /dev/null @@ -1,12 +0,0 @@ -import constant -from machine.scanner_machine import ScannerMachine -from machine.builder_machine import BuilderMachine -from machine.executor_machine import ExecutorMachine -from machine.devkit_machine import DevkitMachine - -KLASS_DICT = { - constant.EXECUTOR: ExecutorMachine, - constant.DEVKIT: DevkitMachine, - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, -} diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py deleted file mode 100644 index 9377b527f8b9a0507156b71be16f3ec62d97c2e0..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ /dev/null @@ -1,8 +0,0 @@ -import constant -from machine.machine import Machine - - -class ScannerMachine(Machine): - def __init__(self, ip, user, pkey, password=None): - super(ScannerMachine, self).__init__(ip, user, pkey, password) - self.role = constant.SCANNER