From ac33c691ab76848499f36eccbd6d91189d96d4ff Mon Sep 17 00:00:00 2001 From: cc <18856836718@163.com> Date: Sat, 16 Mar 2024 11:11:09 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AE=89=E8=A3=85lkp-tes?= =?UTF-8?q?ts=E5=B7=A5=E5=85=B7=E7=9A=84=E4=B8=80=E4=BA=9B=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/distribute/build_devkit_distribute.sh | 2 +- tools/download_dependency/build_download.sh | 4 +- .../lkp_help/compatibility-test/metayaml | 2 +- .../src/handler/install_package.py | 4 +- .../src/machine/builder_machine.py | 11 ++++ .../src/machine/devkit_machine.py | 11 ++++ .../src/machine/executor_machine.py | 11 ++++ .../src/machine/klass_dict.py | 4 +- .../install_dependency/src/machine/machine.py | 55 ++++++++++--------- .../src/machine/scanner_machine.py | 11 ++++ 10 files changed, 80 insertions(+), 35 deletions(-) diff --git a/tools/distribute/build_devkit_distribute.sh b/tools/distribute/build_devkit_distribute.sh index 6f7e05e..05e9205 100644 --- a/tools/distribute/build_devkit_distribute.sh +++ b/tools/distribute/build_devkit_distribute.sh @@ -1,4 +1,4 @@ -#！/bin/bash +#!/bin/bash # SourceCode build script # Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. diff --git a/tools/download_dependency/build_download.sh b/tools/download_dependency/build_download.sh index 6970043..5b4ba6a 100644 --- a/tools/download_dependency/build_download.sh +++ b/tools/download_dependency/build_download.sh @@ -1,4 +1,4 @@ -#！/bin/bash +#!/bin/bash # SourceCode build script # Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. @@ -14,4 +14,4 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src --add-data "${current_dir}/lkp_help:lkp_help" --add-data "${current_dir}/compatibility_test_help:compatibility_test_help" +pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src --add-data "../../tools/download_dependency/lkp_help:lkp_help" --add-data "../../tools/download_dependency/compatibility_test_help:compatibility_test_help" diff --git a/tools/download_dependency/lkp_help/compatibility-test/metayaml b/tools/download_dependency/lkp_help/compatibility-test/metayaml index 9787f7f..21dbe73 100644 --- a/tools/download_dependency/lkp_help/compatibility-test/metayaml +++ b/tools/download_dependency/lkp_help/compatibility-test/metayaml @@ -3,7 +3,7 @@ metadata: summary: A program can run some basic tests description: run compatibility test and generate the report homepage: https://gitee.com/openeuler/devkit-pipeline -type: woorkload +type: workload depends: params: results: diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py index 2757c92..a873af5 100644 --- a/tools/install_dependency/src/handler/install_package.py +++ b/tools/install_dependency/src/handler/install_package.py @@ -23,7 +23,7 @@ class InstallPackage(Handler): ip_set = set() jobs = [] - for role in ({constant.EXECUTOR, constant.DEVKIT} & set(data.keys())): + for role in {constant.EXECUTOR, constant.DEVKIT}: machine_dict = data[role + constant.MACHINE] LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") for machine_ip in machine_dict: @@ -53,7 +53,7 @@ class InstallPackage(Handler): ip_set = set() jobs = [] - for role in (set(KLASS_DICT.keys()) & set(data.keys())): + for role in KLASS_DICT: machine_dict = data[role + constant.MACHINE] LOGGER.debug(f"{role} machine list: {list(machine_dict.keys())}") for machine_ip in machine_dict: diff --git a/tools/install_dependency/src/machine/builder_machine.py b/tools/install_dependency/src/machine/builder_machine.py index 8ee4479..3320be3 100644 --- a/tools/install_dependency/src/machine/builder_machine.py +++ b/tools/install_dependency/src/machine/builder_machine.py @@ -6,3 +6,14 @@ class BuilderMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(BuilderMachine, self).__init__(ip, user, pkey, password) self.role = constant.BUILDER + + def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "LkpTests": self.do_nothing, + "OpenEulerMirrorISO": self.do_nothing, + } + return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file diff --git a/tools/install_dependency/src/machine/devkit_machine.py b/tools/install_dependency/src/machine/devkit_machine.py index 35147f4..4c3201e 100644 --- a/tools/install_dependency/src/machine/devkit_machine.py +++ b/tools/install_dependency/src/machine/devkit_machine.py @@ -6,3 +6,14 @@ class DevkitMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(DevkitMachine, self).__init__(ip, user, pkey, password) self.role = constant.DEVKIT + + def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "LkpTests": self.do_nothing, + "OpenEulerMirrorISO": self.deploy_iso_handle, + } + return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/executor_machine.py b/tools/install_dependency/src/machine/executor_machine.py index 923c7a1..20fcd33 100644 --- a/tools/install_dependency/src/machine/executor_machine.py +++ b/tools/install_dependency/src/machine/executor_machine.py @@ -6,3 +6,14 @@ class ExecutorMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ExecutorMachine, self).__init__(ip, user, pkey, password) self.role = constant.EXECUTOR + + def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "LkpTests": self.lkpTest_install_component_handle, + "OpenEulerMirrorISO": self.deploy_iso_handle, + } + return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/machine/klass_dict.py b/tools/install_dependency/src/machine/klass_dict.py index d9e82c5..835b999 100644 --- a/tools/install_dependency/src/machine/klass_dict.py +++ b/tools/install_dependency/src/machine/klass_dict.py @@ -5,8 +5,8 @@ from machine.executor_machine import ExecutorMachine from machine.devkit_machine import DevkitMachine KLASS_DICT = { - constant.SCANNER: ScannerMachine, - constant.BUILDER: BuilderMachine, constant.EXECUTOR: ExecutorMachine, constant.DEVKIT: DevkitMachine, + constant.SCANNER: ScannerMachine, + constant.BUILDER: BuilderMachine, } diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index ba6231c..db016d5 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -102,17 +102,30 @@ class Machine: "BiShengCompiler": self.default_install_component_handle, "BiShengJDK17": self.default_install_component_handle, "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.lkpTestGem_install_component_handle, + "LkpTests": self.lkpTest_install_component_handle, "OpenEulerMirrorISO": self.deploy_iso_handle, } return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) + def lkpTest_install_component_handle(self, component_name, sftp_client, ssh_client): + try: + stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) + stdin, stdout, stderr = ssh_client.exec_command(f"yum install -y git wget rubygems", timeout=100) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"yum install -y git wget rubygems failed plz run this command to install") + LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") + raise ConnectRemoteException() + exit_status = stdout.channel.recv_exit_status() + LOGGER.debug(f"Remote machine {self.ip} mkdir -p /tmp/{constant.DEPENDENCY_DIR} result: " + f"{'success' if not exit_status else 'failed'}") + if exit_status: + raise NotADirectoryError(f"Remote machine {self.ip} " + f"directory {os.path.join('/tmp/', constant.DEPENDENCY_DIR)} not exist.") - def compatibilityTest_install_component_handle(self, component_name, sftp_client, ssh_client): - # 上传 compatibility_testing.tar.gz文件 - LOGGER.info(f"Install component in remote machine {self.ip}: compatibility test ") - shell_dict = lkp_collection_map.get(component_name) + # 上传 lkp-tests.tar.gz文件 + LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") remote_file_list = [] + shell_dict = lkp_collection_map.get(component_name) for shell_cmd in shell_dict: url_and_save_path = shell_dict.get(shell_cmd) local_file = url_and_save_path.get("save_path") @@ -122,13 +135,14 @@ class Machine: remote_file = os.path.abspath(os.path.join('/tmp', local_file)) LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " f"remote_file: {remote_file}") + remote_file_list.append(remote_file) sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") install_result = "" for shell_file in SHELL_FILE_LIST: sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) - sh_cmd = f"bash {sh_file_remote_path} {remote_file_list[0]}" + sh_cmd = f"bash {sh_file_remote_path} {remote_file_list[0]} {remote_file_list[1]}" execute_output = ( self.transport_shell_file_and_execute( ssh_client, sftp_client, @@ -146,26 +160,13 @@ class Machine: LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) + self.compatibilityTest_install_component_handle("CompatibilityTesting", sftp_client, ssh_client) - - def lkpTestGem_install_component_handle(self, component_name, sftp_client, ssh_client): - try: - stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) - stdin, stdout, stderr = ssh_client.exec_command(f"yum install -y git wget rubygems", timeout=100) - except (paramiko.ssh_exception.SSHException, socket.timeout) as e: - LOGGER.error(f"yum install -y git wget rubygems failed plz run this command to install") - raise ConnectRemoteException() - exit_status = stdout.channel.recv_exit_status() - LOGGER.debug(f"Remote machine {self.ip} mkdir -p /tmp/{constant.DEPENDENCY_DIR} result: " - f"{'success' if not exit_status else 'failed'}") - if exit_status: - raise NotADirectoryError(f"Remote machine {self.ip} " - f"directory {os.path.join('/tmp/', constant.DEPENDENCY_DIR)} not exist.") - - # 上传 lkp-tests.tar.gz文件 + def compatibilityTest_install_component_handle(self, component_name, sftp_client, ssh_client): + # 上传 compatibility_testing.tar.gz文件 LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") - remote_file_list = [] shell_dict = lkp_collection_map.get(component_name) + remote_file_list = [] for shell_cmd in shell_dict: url_and_save_path = shell_dict.get(shell_cmd) local_file = url_and_save_path.get("save_path") @@ -175,14 +176,13 @@ class Machine: remote_file = os.path.abspath(os.path.join('/tmp', local_file)) LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " f"remote_file: {remote_file}") - remote_file_list.append(remote_file) sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") install_result = "" for shell_file in SHELL_FILE_LIST: sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) - sh_cmd = f"bash {sh_file_remote_path} {remote_file_list[0]} {remote_file_list[1]}" + sh_cmd = f"bash {sh_file_remote_path} {remote_file_list[0]}" execute_output = ( self.transport_shell_file_and_execute( ssh_client, sftp_client, @@ -200,8 +200,6 @@ class Machine: LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) - self.compatibilityTest_install_component_handle("CompatibilityTesting", sftp_client, ssh_client) - def deploy_iso_handle(self, component_name, sftp_client, ssh_client): # 上传 镜像文件 @@ -305,3 +303,6 @@ class Machine: for remote_file in remote_file_list: LOGGER.debug(f"Delete tmp file at remote machine {self.ip}: {remote_file}") ssh_client.exec_command(f"rm -f {remote_file}") + + def do_nothing(self, component_name, sftp_client, ssh_client): + return diff --git a/tools/install_dependency/src/machine/scanner_machine.py b/tools/install_dependency/src/machine/scanner_machine.py index 9377b52..8a993b8 100644 --- a/tools/install_dependency/src/machine/scanner_machine.py +++ b/tools/install_dependency/src/machine/scanner_machine.py @@ -6,3 +6,14 @@ class ScannerMachine(Machine): def __init__(self, ip, user, pkey, password=None): super(ScannerMachine, self).__init__(ip, user, pkey, password) self.role = constant.SCANNER + + def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": self.default_install_component_handle, + "BiShengCompiler": self.default_install_component_handle, + "BiShengJDK17": self.default_install_component_handle, + "BiShengJDK8": self.default_install_component_handle, + "LkpTests": self.do_nothing, + "OpenEulerMirrorISO": self.do_nothing, + } + return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) \ No newline at end of file -- Gitee From 5e9f9cae9a6c2c89bb13fb1e14089182a9069fef Mon Sep 17 00:00:00 2001 From: cc <18856836718@163.com> Date: Sat, 16 Mar 2024 14:38:25 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E9=A2=84=E5=A1=AB=E5=8D=B8=E8=BD=BD?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E4=BB=A3=E7=A0=81=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 10 +++---- component/OpenEulerMirrorISO/install.sh | 3 +- component/OpenEulerMirrorISO/uninstall.sh | 21 ++++++++++++++ .../src/handler/install_package.py | 28 ++++++++++++++++--- .../src/machine/local_machine.py | 6 ++++ .../install_dependency/src/machine/machine.py | 3 ++ 6 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 component/OpenEulerMirrorISO/uninstall.sh diff --git a/build.sh b/build.sh index 3ea072b..26a8334 100644 --- a/build.sh +++ b/build.sh @@ -15,9 +15,9 @@ sh "${current_dir}"/tools/install_dependency/build_install.sh cd "${current_dir}"/build -mkdir -p "${current_dir}"/build/dekvit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/install_dependency/dist/* "${current_dir}"/build/dekvit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/download_dependency/dist/* "${current_dir}"/build/dekvit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/distribute/devkit_distribute "${current_dir}"/build/dekvit-pipeline-${tag}/linux +mkdir -p "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/install_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/download_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/distribute/devkit_distribute "${current_dir}"/build/devkit-pipeline-${tag}/linux -tar -zcvf dekvit-pipeline-${tag}.tar.gz dekvit-pipeline-${tag} +tar -zcvf devkit-pipeline-${tag}.tar.gz devkit-pipeline-${tag} diff --git a/component/OpenEulerMirrorISO/install.sh b/component/OpenEulerMirrorISO/install.sh index a879826..a6cd2bf 100644 --- a/component/OpenEulerMirrorISO/install.sh +++ b/component/OpenEulerMirrorISO/install.sh @@ -2,7 +2,8 @@ function configure_local_mirror() { local iso_file_path=$1 - mount ${iso_file_path} /mnt -o loop + mkdir -p /devkitmirror + mount ${iso_file_path} /devkitmirror -o loop if [[ -d /etc/yum.repos.d/yum.repos.backup ]]; then mv -rf /etc/yum.repos.d/yum.repos.backup /etc/yum.repos.backup else diff --git a/component/OpenEulerMirrorISO/uninstall.sh b/component/OpenEulerMirrorISO/uninstall.sh new file mode 100644 index 0000000..43233d1 --- /dev/null +++ b/component/OpenEulerMirrorISO/uninstall.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +function resume_original_mirror() { + rm -rf /etc/yum.repos.d/local.repo + mv -rf /etc/yum.repos.d/yum.repos.backup/* /etc/yum.repos.d/ + rm -rf /etc/yum.repos.d/yum.repos.backup + + yum clean all + yum makecache +} + +function main() { + iso_file_path=$1 + rm -rf "${iso_file_path}" + umount /devkitmirror + rm -rf /devkitmirror + + resume_original_mirror +} + +main "$@" \ No newline at end of file diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py index a873af5..af98a27 100644 --- a/tools/install_dependency/src/handler/install_package.py +++ b/tools/install_dependency/src/handler/install_package.py @@ -12,11 +12,18 @@ class InstallPackage(Handler): def handle(self, data) -> bool: instruction_to_func_dict = { - "deploy_iso": InstallPackage.deploy_iso_handle, + "deploy_iso": InstallPackage.deploy_iso_all_handle, "default": InstallPackage.default_handle, } return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))(data) + @staticmethod + def deploy_iso_all_handle(data): + InstallPackage.deploy_iso_handle(data) + InstallPackage.default_handle(data) + InstallPackage.undeploy_iso_handle(data) + return True + @staticmethod def deploy_iso_handle(data): LOGGER.debug("Deploy iso and install Package start!") @@ -44,9 +51,6 @@ class InstallPackage(Handler): for job in jobs: job.join() - InstallPackage.default_handle(data) - return True - @staticmethod def default_handle(data): LOGGER.debug("Install Package start!") @@ -79,6 +83,22 @@ class InstallPackage(Handler): job.join() return True + @staticmethod + def undeploy_iso_handle(data): + ip_set = set() + + for role in ({constant.EXECUTOR, constant.DEVKIT} & set(data.keys())): + machine_dict = data[role + constant.MACHINE] + + for machine_ip in machine_dict: + if machine_ip in ip_set: + continue + ip_set.add(machine_ip) + LOGGER.debug(f"ip_set to un-deploy iso: {ip_set}") + machine = machine_dict.get(machine_ip) + machine.undeploy_iso_work() + LOGGER.debug(f"Resume original mirror in machine: {machine_ip} ") + def process_work(machine, *components: str): try: diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 1815b1a..744267a 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -134,3 +134,9 @@ class LocalMachine: LOGGER.debug(f"Delete tmp file at local machine {self.ip}: {remote_file}") subprocess.run(f"rm -f {remote_file}".split(' '), capture_output=False, shell=False, stderr=subprocess.STDOUT) + + def do_nothing(self, component_name, sftp_client, ssh_client): + return + + def undeploy_iso_work(self): + return diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index db016d5..c14e545 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -306,3 +306,6 @@ class Machine: def do_nothing(self, component_name, sftp_client, ssh_client): return + + def undeploy_iso_work(self): + return -- Gitee