diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 95ee72500c49ca84048d95fa5c83411104c7a47b..748a8b36c9e48a829e25f6b730ad036a1076b6a5 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -1,4 +1,3 @@ -import os import re import logging import constant @@ -6,8 +5,7 @@ from handler.handler_and_node import Handler from machine.klass_dict import KLASS_DICT LOGGER = logging.getLogger("install_dependency") -MIN_SET = (constant.USER, constant.PKEY, - constant.EXECUTOR, constant.INSTRUCTION) +MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, constant.SCANNER, constant.BUILDER, constant.EXECUTOR, constant.DEVKIT, constant.INSTRUCTION) diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index f3e656bf87b19f3d40c99aed65f39f41505b756a..fbe8e877e839fd92c71c149b8a8b224aa9507fc0 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -9,6 +9,12 @@ from exception.connect_exception import ConnectException LOGGER = logging.getLogger("install_dependency") +ROLE_COMPONENT = { + "scanner": ["BiShengJDK17"], + "builder": ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + "executor": ["BiShengJDK17", "LkpTests"] +} + class ConnectCheck(Handler): @@ -25,25 +31,29 @@ class ConnectCheck(Handler): def machine_role_check(data, role, local_ip): builder_list = data.get(role) klass = KLASS_DICT.get(role) - data[role + constant.MACHINE] = dict() + data[constant.MACHINE] = dict() for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" - machine_instance = LocalMachine(ip) - data[role + constant.MACHINE][ip] = machine_instance + machine_instance = data[constant.MACHINE].get(ip, LocalMachine(ip)) + machine_instance.add_component(ROLE_COMPONENT[role]) + data[constant.MACHINE][ip] = machine_instance continue try: - machine_instance = klass(ip, data[constant.USER], data[constant.PKEY], - data.get(constant.PASSWORD, None)) - data[role + constant.MACHINE][ip] = machine_instance + machine_instance = data[constant.MACHINE].get(ip, klass(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None))) + machine_instance.add_component(ROLE_COMPONENT[role]) + data[constant.MACHINE][ip] = machine_instance except ConnectException: LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") - del data[role + constant.MACHINE] + del data[constant.MACHINE] return False except Exception as e: LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") - del data[role + constant.MACHINE] + del data[constant.MACHINE] return False + if data.get(constant.INSTRUCTION) == "deploy_iso" and role in ("devkit", "executor"): + machine_instance.set_mirror() return True @staticmethod diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py index 7abb0c6c171b67d11e5721106e93446d71701a09..ab81daa10a043a219b85ff65c8e11d7514aa167a 100644 --- a/tools/install_dependency/src/handler/install_package.py +++ b/tools/install_dependency/src/handler/install_package.py @@ -3,118 +3,26 @@ import multiprocessing import constant from handler.handler_and_node import Handler -from machine.klass_dict import KLASS_DICT -from utils import available_role LOGGER = logging.getLogger("install_dependency") class InstallPackage(Handler): - def handle(self, data) -> bool: - instruction_to_func_dict = { - "deploy_iso": InstallPackage.deploy_iso_all_handle, - "default": InstallPackage.default_handle, - } - return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))(data) - - @staticmethod - def deploy_iso_all_handle(data): - InstallPackage.deploy_iso_handle(data) - InstallPackage.default_handle(data) - InstallPackage.undeploy_iso_handle(data) - return True - - @staticmethod - def deploy_iso_handle(data): - LOGGER.debug("Deploy iso and install Package start!") - ip_set = set() - jobs = [] - - for role in available_role([constant.EXECUTOR, constant.DEVKIT], data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list to deploy iso: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to deploy iso: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "OpenEulerMirrorISO", - ), - ) - jobs.append(process) - process.start() - - for job in jobs: - job.join() - - @staticmethod - def default_handle(data): + def handle(self, data): LOGGER.debug("Install Package start!") - ip_set = set() jobs = [] - - for role in available_role(KLASS_DICT, data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to install package: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "GCCforOpenEuler", - "BiShengCompiler", - "BiShengJDK17", - "BiShengJDK8", - "LkpTests", - "NonInvasiveSwitching" - ), - ) - jobs.append(process) - process.start() - + for _, machine in data[constant.MACHINE].items(): + process = multiprocessing.Process(target=process_work, args=(machine,)) + jobs.append(process) + process.start() for job in jobs: job.join() return True - @staticmethod - def undeploy_iso_handle(data): - ip_set = set() - jobs = [] - - for role in available_role([constant.EXECUTOR, constant.DEVKIT], data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list to un-deploy iso: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to un-deploy iso: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "UnOpenEulerMirrorISO", - ), - ) - jobs.append(process) - process.start() - - for job in jobs: - job.join() - -def process_work(machine, *components: str): +def process_work(machine): try: - for component in components: - machine.install_component(component) + machine.install_components() except (OSError, IOError) as e: LOGGER.error(f"Remote machine {machine.ip} occur Error: {str(e)}") diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py index ebacd4d7a596c3f20e6c5f6c74bbe70bcad5fdeb..8ee4479e37a5800eaf9f829d1115739e3fb82120 100644 --- a/tools/install_dependency/src/machine/builder_machine.py +++ b/tools/install_dependency/src/machine/builder_machine.py @@ -6,16 +6,3 @@ class BuilderMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(BuilderMachine, self).__init__(ip, user, pkey, password) self.role = constant.BUILDER - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.do_nothing, - "UnOpenEulerMirrorISO": self.do_nothing, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py index be3ffa55036b52addaf3a36dc8649292f4d7f2de..35147f46c3d95cb5da27fd0c67a577e04b794f53 100644 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ b/tools/install_dependency/src/machine/devkit_machine.py @@ -6,16 +6,3 @@ class DevkitMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(DevkitMachine, self).__init__(ip, user, pkey, password) self.role = constant.DEVKIT - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.deploy_iso_handle, - "UnOpenEulerMirrorISO": self.undeploy_iso_handle, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py index 4c10f524b13f3f7d9bce10cf1d142e23ea584992..923c7a133a7a743828700e73da9e49ee1a969874 100644 --- a/tools/install_dependency/src/machine/executor_machine.py +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -6,16 +6,3 @@ class ExecutorMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ExecutorMachine, self).__init__(ip, user, pkey, password) self.role = constant.EXECUTOR - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.lkptest_install_component_handle, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.deploy_iso_handle, - "UnOpenEulerMirrorISO": self.undeploy_iso_handle, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 74959e3650571059b7b29364b65b89a0fe9e99f1..8e46f6f8c72a0d189d85a6ca8e2442b260da97d2 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -18,6 +18,23 @@ class LocalMachine: def __init__(self, ip): self.ip = ip self.check_is_aarch64() + self.component_list = [] + self.mirrors = False + + def set_mirror(self): + self.mirrors = True + + def add_component(self, component): + self.component_list.extend(component) + self.component_list = list(set(self.component_list)) + + def install_components(self): + if self.mirrors: + self.install_component("OpenEulerMirrorISO") + for component in self.component_list: + self.install_component(component) + if self.mirrors: + self.install_component("UnOpenEulerMirrorISO") def check_is_aarch64(self): machine_type = os.uname().machine.lower() diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index a923b812e27ca2ecbdb2f12a5b8ad005e5454874..31300fc4518c311d0dac04f9d4e7081fe869e30e 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -27,6 +27,15 @@ class Machine: self.pkey = pkey self.password = password self.check_is_aarch64() + self.component_list = [] + self.mirrors = False + + def add_component(self, component): + self.component_list.extend(component) + self.component_list = list(set(self.component_list)) + + def set_mirror(self): + self.mirrors = True def check_is_aarch64(self): machine_type = self.get_machine_type() @@ -105,6 +114,14 @@ class Machine: ssh_client.close() sftp_client.close() + def install_components(self): + if self.mirrors: + self.install_component("OpenEulerMirrorISO") + for component in self.component_list: + self.install_component(component) + if self.mirrors: + self.install_component("UnOpenEulerMirrorISO") + def install_component_handler(self, component_name, sftp_client, ssh_client): component_name_to_func_dict: typing.Dict[ str, typing.Callable[[str, paramiko.SFTPClient, paramiko.SSHClient], typing.Any]] = { diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py index f87a6435259cee13b0ed6aca8aa977ffc568dea5..9377b527f8b9a0507156b71be16f3ec62d97c2e0 100644 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ b/tools/install_dependency/src/machine/scanner_machine.py @@ -6,16 +6,3 @@ class ScannerMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ScannerMachine, self).__init__(ip, user, pkey, password) self.role = constant.SCANNER - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.do_nothing, - "UnOpenEulerMirrorISO": self.do_nothing, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file