From ffd0be8a1011986b2fea7a97d542029543bbe97b Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Tue, 26 Mar 2024 10:10:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=96=AE=E4=B8=80=E8=A7=92?= =?UTF-8?q?=E8=89=B2=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/handler/base_yaml_check.py | 4 +- .../src/handler/connect_check.py | 26 +++-- .../src/handler/install_package.py | 106 ++---------------- .../src/machine/builder_machine.py | 13 --- .../src/machine/devkit_machine.py | 13 --- .../src/machine/executor_machine.py | 13 --- .../src/machine/local_machine.py | 17 +++ .../install_dependency/src/machine/machine.py | 17 +++ .../src/machine/scanner_machine.py | 13 --- 9 files changed, 60 insertions(+), 162 deletions(-) diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/install_dependency/src/handler/base_yaml_check.py index 95ee725..748a8b3 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/install_dependency/src/handler/base_yaml_check.py @@ -1,4 +1,3 @@ -import os import re import logging import constant @@ -6,8 +5,7 @@ from handler.handler_and_node import Handler from machine.klass_dict import KLASS_DICT LOGGER = logging.getLogger("install_dependency") -MIN_SET = (constant.USER, constant.PKEY, - constant.EXECUTOR, constant.INSTRUCTION) +MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, constant.SCANNER, constant.BUILDER, constant.EXECUTOR, constant.DEVKIT, constant.INSTRUCTION) diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index f3e656b..fbe8e87 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -9,6 +9,12 @@ from exception.connect_exception import ConnectException LOGGER = logging.getLogger("install_dependency") +ROLE_COMPONENT = { + "scanner": ["BiShengJDK17"], + "builder": ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + "executor": ["BiShengJDK17", "LkpTests"] +} + class ConnectCheck(Handler): @@ -25,25 +31,29 @@ class ConnectCheck(Handler): def machine_role_check(data, role, local_ip): builder_list = data.get(role) klass = KLASS_DICT.get(role) - data[role + constant.MACHINE] = dict() + data[constant.MACHINE] = dict() for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" - machine_instance = LocalMachine(ip) - data[role + constant.MACHINE][ip] = machine_instance + machine_instance = data[constant.MACHINE].get(ip, LocalMachine(ip)) + machine_instance.add_component(ROLE_COMPONENT[role]) + data[constant.MACHINE][ip] = machine_instance continue try: - machine_instance = klass(ip, data[constant.USER], data[constant.PKEY], - data.get(constant.PASSWORD, None)) - data[role + constant.MACHINE][ip] = machine_instance + machine_instance = data[constant.MACHINE].get(ip, klass(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None))) + machine_instance.add_component(ROLE_COMPONENT[role]) + data[constant.MACHINE][ip] = machine_instance except ConnectException: LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") - del data[role + constant.MACHINE] + del data[constant.MACHINE] return False except Exception as e: LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") - del data[role + constant.MACHINE] + del data[constant.MACHINE] return False + if data.get(constant.INSTRUCTION) == "deploy_iso" and role in ("devkit", "executor"): + machine_instance.set_mirror() return True @staticmethod diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py index 7abb0c6..ab81daa 100644 --- a/tools/install_dependency/src/handler/install_package.py +++ b/tools/install_dependency/src/handler/install_package.py @@ -3,118 +3,26 @@ import multiprocessing import constant from handler.handler_and_node import Handler -from machine.klass_dict import KLASS_DICT -from utils import available_role LOGGER = logging.getLogger("install_dependency") class InstallPackage(Handler): - def handle(self, data) -> bool: - instruction_to_func_dict = { - "deploy_iso": InstallPackage.deploy_iso_all_handle, - "default": InstallPackage.default_handle, - } - return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))(data) - - @staticmethod - def deploy_iso_all_handle(data): - InstallPackage.deploy_iso_handle(data) - InstallPackage.default_handle(data) - InstallPackage.undeploy_iso_handle(data) - return True - - @staticmethod - def deploy_iso_handle(data): - LOGGER.debug("Deploy iso and install Package start!") - ip_set = set() - jobs = [] - - for role in available_role([constant.EXECUTOR, constant.DEVKIT], data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list to deploy iso: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to deploy iso: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "OpenEulerMirrorISO", - ), - ) - jobs.append(process) - process.start() - - for job in jobs: - job.join() - - @staticmethod - def default_handle(data): + def handle(self, data): LOGGER.debug("Install Package start!") - ip_set = set() jobs = [] - - for role in available_role(KLASS_DICT, data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to install package: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "GCCforOpenEuler", - "BiShengCompiler", - "BiShengJDK17", - "BiShengJDK8", - "LkpTests", - "NonInvasiveSwitching" - ), - ) - jobs.append(process) - process.start() - + for _, machine in data[constant.MACHINE].items(): + process = multiprocessing.Process(target=process_work, args=(machine,)) + jobs.append(process) + process.start() for job in jobs: job.join() return True - @staticmethod - def undeploy_iso_handle(data): - ip_set = set() - jobs = [] - - for role in available_role([constant.EXECUTOR, constant.DEVKIT], data): - machine_dict = data[role + constant.MACHINE] - LOGGER.debug(f"{role} machine list to un-deploy iso: {list(machine_dict.keys())}") - for machine_ip in machine_dict: - if machine_ip in ip_set: - continue - ip_set.add(machine_ip) - LOGGER.debug(f"ip_set to un-deploy iso: {ip_set}") - machine = machine_dict.get(machine_ip) - process = multiprocessing.Process( - target=process_work, - args=(machine, - "UnOpenEulerMirrorISO", - ), - ) - jobs.append(process) - process.start() - - for job in jobs: - job.join() - -def process_work(machine, *components: str): +def process_work(machine): try: - for component in components: - machine.install_component(component) + machine.install_components() except (OSError, IOError) as e: LOGGER.error(f"Remote machine {machine.ip} occur Error: {str(e)}") diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py index ebacd4d..8ee4479 100644 --- a/tools/install_dependency/src/machine/builder_machine.py +++ b/tools/install_dependency/src/machine/builder_machine.py @@ -6,16 +6,3 @@ class BuilderMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(BuilderMachine, self).__init__(ip, user, pkey, password) self.role = constant.BUILDER - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.do_nothing, - "UnOpenEulerMirrorISO": self.do_nothing, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py index be3ffa5..35147f4 100644 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ b/tools/install_dependency/src/machine/devkit_machine.py @@ -6,16 +6,3 @@ class DevkitMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(DevkitMachine, self).__init__(ip, user, pkey, password) self.role = constant.DEVKIT - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.deploy_iso_handle, - "UnOpenEulerMirrorISO": self.undeploy_iso_handle, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py index 4c10f52..923c7a1 100644 --- a/tools/install_dependency/src/machine/executor_machine.py +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -6,16 +6,3 @@ class ExecutorMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ExecutorMachine, self).__init__(ip, user, pkey, password) self.role = constant.EXECUTOR - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.lkptest_install_component_handle, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.deploy_iso_handle, - "UnOpenEulerMirrorISO": self.undeploy_iso_handle, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 74959e3..8e46f6f 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -18,6 +18,23 @@ class LocalMachine: def __init__(self, ip): self.ip = ip self.check_is_aarch64() + self.component_list = [] + self.mirrors = False + + def set_mirror(self): + self.mirrors = True + + def add_component(self, component): + self.component_list.extend(component) + self.component_list = list(set(self.component_list)) + + def install_components(self): + if self.mirrors: + self.install_component("OpenEulerMirrorISO") + for component in self.component_list: + self.install_component(component) + if self.mirrors: + self.install_component("UnOpenEulerMirrorISO") def check_is_aarch64(self): machine_type = os.uname().machine.lower() diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index a923b81..31300fc 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -27,6 +27,15 @@ class Machine: self.pkey = pkey self.password = password self.check_is_aarch64() + self.component_list = [] + self.mirrors = False + + def add_component(self, component): + self.component_list.extend(component) + self.component_list = list(set(self.component_list)) + + def set_mirror(self): + self.mirrors = True def check_is_aarch64(self): machine_type = self.get_machine_type() @@ -105,6 +114,14 @@ class Machine: ssh_client.close() sftp_client.close() + def install_components(self): + if self.mirrors: + self.install_component("OpenEulerMirrorISO") + for component in self.component_list: + self.install_component(component) + if self.mirrors: + self.install_component("UnOpenEulerMirrorISO") + def install_component_handler(self, component_name, sftp_client, ssh_client): component_name_to_func_dict: typing.Dict[ str, typing.Callable[[str, paramiko.SFTPClient, paramiko.SSHClient], typing.Any]] = { diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py index f87a643..9377b52 100644 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ b/tools/install_dependency/src/machine/scanner_machine.py @@ -6,16 +6,3 @@ class ScannerMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ScannerMachine, self).__init__(ip, user, pkey, password) self.role = constant.SCANNER - - def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { - "GCCforOpenEuler": self.default_install_component_handle, - "BiShengCompiler": self.default_install_component_handle, - "BiShengJDK17": self.default_install_component_handle, - "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.do_nothing, - "NonInvasiveSwitching": self.nis_install_component_handle, - "OpenEulerMirrorISO": self.do_nothing, - "UnOpenEulerMirrorISO": self.do_nothing, - } - return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file -- Gitee