From d7dd7775501d774d30fc2276c7d1c634c380532a Mon Sep 17 00:00:00 2001 From: cc <18856836718@163.com> Date: Mon, 1 Apr 2024 11:02:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9Edistribute=20=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E9=93=BE=E6=8E=A5=20=E5=B9=B6=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../build_devkit_distribute.sh | 1 - tools/install_dependency/src/constant.py | 1 + ...command_line.py => deploy_command_line.py} | 0 .../src/deploy/lkp_collect_map.py | 26 ------ tools/install_dependency/src/deploy_main.py | 2 +- .../src/download/download_config.py | 10 +-- .../src/download/download_utils.py | 86 +++++++++++++------ .../src/handler/gather_package.py | 3 +- .../install_dependency/src/machine/machine.py | 38 ++++---- tools/install_dependency/src/utils.py | 4 +- 10 files changed, 85 insertions(+), 86 deletions(-) rename tools/install_dependency/src/deploy/{command_line.py => deploy_command_line.py} (100%) delete mode 100644 tools/install_dependency/src/deploy/lkp_collect_map.py diff --git a/component/DevkitDistribute/build_devkit_distribute.sh b/component/DevkitDistribute/build_devkit_distribute.sh index 7fe8517..e558e87 100644 --- a/component/DevkitDistribute/build_devkit_distribute.sh +++ b/component/DevkitDistribute/build_devkit_distribute.sh @@ -42,6 +42,5 @@ cp devkit_pipeline_agent.tar.gz devkit_distribute/config tar -czf devkit_distribute.tar.gz devkit_distribute mkdir -p "${project_dir}"/build/component/DevkitDistribute -cp devkit_distribute.tar.gz "${project_dir}"/build/component/DevkitDistribute cp "${current_dir}/check_install_result.sh" "${project_dir}"/build/component/DevkitDistribute cp "${current_dir}/install.sh" "${project_dir}"/build/component/DevkitDistribute diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index f403763..2929bf0 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -26,4 +26,5 @@ FILE = "file" SHA256 = "sha256" URL = "url" SAVE_PATH = "save_path" +FILE_SIZE = "file_size" DEFAULT_PATH = "./devkitdependencies" diff --git a/tools/install_dependency/src/deploy/command_line.py b/tools/install_dependency/src/deploy/deploy_command_line.py similarity index 100% rename from tools/install_dependency/src/deploy/command_line.py rename to tools/install_dependency/src/deploy/deploy_command_line.py diff --git a/tools/install_dependency/src/deploy/lkp_collect_map.py b/tools/install_dependency/src/deploy/lkp_collect_map.py deleted file mode 100644 index e06a4f0..0000000 --- a/tools/install_dependency/src/deploy/lkp_collect_map.py +++ /dev/null @@ -1,26 +0,0 @@ -import os.path - -from utils import base_path - -CURRENT_DEFAULT_PATH = "./devkitdependencies" - -lkp_collection_map = { - "LkpTests": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'lkp-tests.tar.gz')}", - }, - "download gem dependency": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'gem_dependencies.zip')}", - }, - }, - "CompatibilityTesting": { - "download file": { - "save_path": f"{os.path.join(CURRENT_DEFAULT_PATH, 'compatibility_testing.tar.gz')}", - } - }, - "DevkitDistribute": { - "download file": { - "save_path": f"{os.path.join(base_path('component'), 'DevkitDistribute', 'devkit_distribute.tar.gz')}", - } - } -} diff --git a/tools/install_dependency/src/deploy_main.py b/tools/install_dependency/src/deploy_main.py index a406e3a..356297c 100644 --- a/tools/install_dependency/src/deploy_main.py +++ b/tools/install_dependency/src/deploy_main.py @@ -6,7 +6,7 @@ import logging import constant from log import config_logging -from deploy.command_line import process_command_line, CommandLine +from deploy.deploy_command_line import process_command_line, CommandLine from handler.pipeline import PipeLine from handler.base_yaml_check import BaseCheck diff --git a/tools/install_dependency/src/download/download_config.py b/tools/install_dependency/src/download/download_config.py index bd7b2a8..262519e 100644 --- a/tools/install_dependency/src/download/download_config.py +++ b/tools/install_dependency/src/download/download_config.py @@ -23,20 +23,18 @@ BiShengJDK17 = { LkpTests = { "component_name": "LkpTests", "file": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/lkp-tests.tar.gz", - "gem dependency": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/gem_dependencies.zip", + "GemDependency": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/gem_dependencies.zip", + "CompatibilityTesting": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/compatibility_testing.tar.gz", + "DevkitDistribute": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/devkit_distribute.tar.gz", } -CompatibilityTesting = { - "component_name": "CompatibilityTesting", - "file": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/compatibility_testing.tar.gz", -} DevKitWeb = { "component_name": "DevKitWeb", "file": "https://kunpeng-repo.obs.cn-north-4.myhuaweicloud.com/Porting%20Advisor/Porting%20Advisor%2023.0.T30/DevKit-All-23.0.T30-Linux-Kunpeng.tar.gz", - "sha256": "https://kunpeng-repo.obs.cn-north-4.myhuaweicloud.com/Porting%20Advisor/Porting%20Advisor%2023.0.T30/DevKit-All-23.0.T30-Linux-Kunpeng.tar.gz.asc", } + OpenEuler_2003_LTS = { "component_name": "openEuler_2003_LTS", "file": "https://mirrors.huaweicloud.com/openeuler/openEuler-20.03-LTS/ISO/aarch64/openEuler-20.03-LTS-everything-aarch64-dvd.iso", diff --git a/tools/install_dependency/src/download/download_utils.py b/tools/install_dependency/src/download/download_utils.py index 231e1c0..2416e42 100644 --- a/tools/install_dependency/src/download/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -26,41 +26,72 @@ FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_J component_collection_map = { - component.get("component_name"): { - "download file": - { - URL: f"{component.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}" - }, - "download sha256": - { - URL: f"{component.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(SHA256).split('/')[-1])}" - } - } for component in ( - download_config.BiShengCompiler, - download_config.GCCforOpenEuler, - download_config.BiShengJDK8, - download_config.BiShengJDK17, - download_config.DevKitWeb, - ) -} + "BiShengCompiler": { + "download file": { + URL: f"{download_config.BiShengCompiler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(FILE).split('/')[-1])}", + }, + "download sha256": { + URL: f"{download_config.BiShengCompiler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(SHA256).split('/')[-1])}", + }, + }, + "GCCforOpenEuler": { + "download file": { + URL: f"{download_config.GCCforOpenEuler.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(FILE).split('/')[-1])}", + }, + "download sha256": { + URL: f"{download_config.GCCforOpenEuler.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(SHA256).split('/')[-1])}", + }, + }, + "BiShengJDK8": { + "download file": { + URL: f"{download_config.BiShengJDK8.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(FILE).split('/')[-1])}", + }, + "download sha256": { + URL: f"{download_config.BiShengJDK8.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(SHA256).split('/')[-1])}", + }, + }, + "BiShengJDK17": { + "download file": { + URL: f"{download_config.BiShengJDK17.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(FILE).split('/')[-1])}", + }, + "download sha256": { + URL: f"{download_config.BiShengJDK17.get(SHA256)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(SHA256).split('/')[-1])}", + }, + }, -lkp_collection_map = { "LkpTests": { "download file": { URL: f"{download_config.LkpTests.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'lkp-tests.tar.gz')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get(FILE).split('/')[-1])}", + }, + "GemDependency": { + URL: f"{download_config.LkpTests.get('GemDependency')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('GemDependency').split('/')[-1])}", }, - "download gem dependency": { - URL: f"{download_config.LkpTests.get('gem dependency')}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", + "CompatibilityTesting": { + URL: f"{download_config.LkpTests.get('CompatibilityTesting')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('CompatibilityTesting').split('/')[-1])}", }, - "download CompatibilityTesting": { - URL: f"{download_config.CompatibilityTesting.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", + "DevkitDistribute": { + URL: f"{download_config.LkpTests.get('DevkitDistribute')}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('DevkitDistribute').split('/')[-1])}", } }, + + "DevKitWeb": { + "download file": { + URL: f"{download_config.DevKitWeb.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.DevKitWeb.get(FILE).split('/')[-1])}", + }, + } } @@ -98,7 +129,6 @@ def download_dependence(component_list): return False ret = True - component_collection_map.update(lkp_collection_map) for component_name in component_collection_map: if component_name not in component_list: continue diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index 94a6e35..e03e674 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -2,7 +2,7 @@ import logging import os import subprocess import constant -from download.download_utils import download_dependence, component_collection_map, lkp_collection_map +from download.download_utils import download_dependence from handler.handler_and_node import Handler from utils import generate_component_list @@ -16,7 +16,6 @@ class GatherPackage(Handler): def handle(self, data) -> bool: LOGGER.debug("GatherPackage start!") - component_collection_map.update(lkp_collection_map) self.component_list = generate_component_list(data) try: diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index 6eb8a5a..a6d798d 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -8,13 +8,12 @@ import paramiko import timeout_decorator import constant -from deploy.command_line import CommandLine +from deploy.deploy_command_line import CommandLine from exception.connect_exception import (CreatePkeyFailedException, ConnectRemoteException, NotMatchedMachineTypeException) from download.download_utils import component_collection_map -from deploy.lkp_collect_map import lkp_collection_map from utils import (base_path, validate_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, - YUM_INSTALL_TAR_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, + CHECK_TAR_AVAILABLE_CMD, YUM_INSTALL_LKP_DEPENDENCIES_CMD, CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, CHECK_MIRROR_INSTALL_STATUS, PROMPT_MAP) @@ -144,7 +143,7 @@ class Machine: "UnOpenEulerMirrorISO": self.undeploy_iso_handle, "A-FOT": self.install_a_fot, } - self._remote_exec_command(YUM_INSTALL_TAR_CMD, ssh_client) + self._remote_exec_command(CHECK_TAR_AVAILABLE_CMD, ssh_client) self._remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, ssh_client) self._remote_exec_command(CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, ssh_client) return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) @@ -214,7 +213,7 @@ class Machine: # 上传 lkp-tests.tar.gz文件 LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") remote_file_list = [] - shell_dict = lkp_collection_map.get(component_name) + shell_dict = component_collection_map.get(component_name) for shell_cmd in shell_dict: url_and_save_path = shell_dict.get(shell_cmd) local_file = url_and_save_path.get("save_path") @@ -250,24 +249,23 @@ class Machine: self.__install_component_on_lkptest("CompatibilityTesting", sftp_client, ssh_client) self.__install_component_on_lkptest("DevkitDistribute", sftp_client, ssh_client) - def __install_component_on_lkptest(self, component_name, sftp_client, ssh_client): + def __install_component_on_lkptest(self, sub_component_name, sftp_client, ssh_client): # 上传 tar.gz 文件 - LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") + LOGGER.info(f"Install component in remote machine {self.ip}: {sub_component_name}") remote_file_list = [] - shell_dict = lkp_collection_map.get(component_name) - for shell_cmd in shell_dict: - url_and_save_path = shell_dict.get(shell_cmd) - local_file = url_and_save_path.get("save_path") - remote_file = os.path.abspath(os.path.join('/tmp', constant.DEPENDENCY_DIR, local_file.split('/')[-1])) - LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " - f"remote_file: {remote_file}") - remote_file_list.append(remote_file) - sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") + url_and_save_path = component_collection_map.get("LkpTests").get(sub_component_name) + + local_file = url_and_save_path.get("save_path") + remote_file = os.path.abspath(os.path.join('/tmp', constant.DEPENDENCY_DIR, local_file.split('/')[-1])) + LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " + f"remote_file: {remote_file}") + remote_file_list.append(remote_file) + sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") # 上传并执行 安装脚本, 校验安装结果脚本 install_result = "" for shell_file in SHELL_FILE_LIST: - sh_file_local_path = os.path.join(base_path("component"), component_name, shell_file) - sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, component_name + shell_file) + sh_file_local_path = os.path.join(base_path("component"), sub_component_name, shell_file) + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, sub_component_name + shell_file) sh_cmd = f"bash {sh_file_remote_path} {remote_file_list[0]}" execute_output = ( self.transport_shell_file_and_execute( @@ -281,9 +279,9 @@ class Machine: install_result = execute_output if install_result == "true": - LOGGER.info(f"Remote machine {self.ip} install {component_name} success.") + LOGGER.info(f"Remote machine {self.ip} install {sub_component_name} success.") else: - LOGGER.error(f"Remote machine {self.ip} install {component_name} failed.") + LOGGER.error(f"Remote machine {self.ip} install {sub_component_name} failed.") # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) diff --git a/tools/install_dependency/src/utils.py b/tools/install_dependency/src/utils.py index 8efbc48..89df579 100644 --- a/tools/install_dependency/src/utils.py +++ b/tools/install_dependency/src/utils.py @@ -3,7 +3,7 @@ import sys import yaml from constant import ROLE_LIST, ROLE_COMPONENT -YUM_INSTALL_TAR_CMD = "sudo yum install -y tar" +CHECK_TAR_AVAILABLE_CMD = "which tar" MKDIR_TMP_DEVKITDEPENDENCIES_CMD = "mkdir -p /tmp/devkitdependencies" YUM_INSTALL_LKP_DEPENDENCIES_CMD = "sudo yum install -y git wget rubygems" CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR = "[[ $(df -m /home | awk 'NR==2' | awk '{print $4}') -gt 17*1024 ]]" @@ -11,7 +11,7 @@ CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE = "[[ $(df -m /tmp | awk 'NR==2' | awk '{ CHECK_MIRROR_INSTALL_STATUS = "test -d /etc/yum.repos.d/yum.repos.backup" PROMPT_MAP = { - YUM_INSTALL_TAR_CMD: "Yum install 'tar' failed.", + CHECK_TAR_AVAILABLE_CMD: "'tar' command not available.", MKDIR_TMP_DEVKITDEPENDENCIES_CMD: "Directory /tmp/devkitdependencies not exists.", YUM_INSTALL_LKP_DEPENDENCIES_CMD: "Yum install lkp dependencies failed.", CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR: "Machine /home space not sufficient for mirror.", -- Gitee