From 05c7ecb7d64741846d30adaaea86bae38793ccde Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Thu, 28 Mar 2024 10:05:53 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=B8=8B=E8=BD=BDA-FOT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/install_dependency/src/constant.py | 2 +- .../src/download/download_utils.py | 36 +++++++++++++++++++ tools/install_dependency/src/download_main.py | 36 +------------------ 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/tools/install_dependency/src/constant.py b/tools/install_dependency/src/constant.py index f15c00c..1e3ad0e 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/install_dependency/src/constant.py @@ -14,7 +14,7 @@ INSTRUCTION = "instruction" ROLE_COMPONENT = { SCANNER: ["BiShengJDK17"], - C_BUIDLER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17"], + C_BUIDLER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "A-FOT"], JAVA_BUILDER: ["BiShengJDK17", "BiShengJDK8"], EXECUTOR: ["BiShengJDK17", "LkpTests"] } diff --git a/tools/install_dependency/src/download/download_utils.py b/tools/install_dependency/src/download/download_utils.py index d1c86a7..d073861 100644 --- a/tools/install_dependency/src/download/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -1,6 +1,8 @@ import os import platform import subprocess +import shutil +import tarfile import urllib.error import warnings @@ -9,8 +11,19 @@ import wget from download import download_config from constant import URL, FILE, SAVE_PATH, SHA256, DEFAULT_PATH + warnings.filterwarnings("ignore", message='Unverified HTTPS request') +# A-FOT files +BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" +A_FOT = "a-fot" +A_FOT_INI = "a-fot.ini" +AUTO_FDO_SH = "auto_fdo.sh" +AUTO_BOLT_SH = "auto_bolt.sh" +AUTO_PREFETCH = "auto_prefetch.sh" +SPLIT_JSON_PY = "split_json.py" +FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) + component_collection_map = { component.get("component_name"): { @@ -50,6 +63,27 @@ lkp_collection_map = { } +def download_a_fot(): + saved_path = os.path.join(DEFAULT_PATH, A_FOT) + try: + os.mkdir(saved_path) + except FileExistsError as e: + pass + + try: + for f in FILE_LIST: + wget.download(BASE_URL.format(f), os.path.join(saved_path, f)) + + with tarfile.open(os.path.join(DEFAULT_PATH, "a-fot.tar.gz"), "w:gz") as t: + t.add(saved_path, arcname="a-fot") + return True + except Exception as e: + print(e) + return False + finally: + shutil.rmtree(saved_path) + + def download_dependence(component_list): if not os.path.exists(DEFAULT_PATH): os.mkdir(DEFAULT_PATH) @@ -64,6 +98,8 @@ def download_dependence(component_list): continue shell_dict = component_collection_map.get(component_name) ret = ret and download_dependence_handler(shell_dict) + if "A-FOT" in component_list: + ret &= download_a_fot() return ret diff --git a/tools/install_dependency/src/download_main.py b/tools/install_dependency/src/download_main.py index 11d27d3..2c0d0aa 100644 --- a/tools/install_dependency/src/download_main.py +++ b/tools/install_dependency/src/download_main.py @@ -2,9 +2,6 @@ import os import platform import subprocess import sys -import shutil -import tarfile -import wget from download import download_config from download.download_utils import download_dependence_file @@ -15,42 +12,11 @@ from handler.gather_package import GatherPackage from handler.compress_dep import CompressDep from utils import read_yaml_file -from constant import URL, SHA256, SAVE_PATH, DEFAULT_PATH, FILE, INSTRUCTION +from constant import URL, SHA256, SAVE_PATH, FILE, INSTRUCTION PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] -# A-FOT files -BASE_URL = "https://gitee.com/openeuler/A-FOT/raw/master/{}" -A_FOT = "a-fot" -A_FOT_INI = "a-fot.ini" -AUTO_FDO_SH = "auto_fdo.sh" -AUTO_BOLT_SH = "auto_bolt.sh" -AUTO_PREFETCH = "auto_prefetch.sh" -SPLIT_JSON_PY = "split_json.py" -FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) - - -def download_a_fot(): - saved_path = os.path.join(DEFAULT_PATH, A_FOT) - try: - os.mkdir(saved_path) - except FileExistsError as e: - pass - - try: - for f in FILE_LIST: - wget.download(BASE_URL.format(f), os.path.join(saved_path, f)) - - with tarfile.open(os.path.join(DEFAULT_PATH, "a-fot.tar.gz"), "w:gz") as t: - t.add(saved_path, arcname="a-fot") - return True - except Exception as e: - print(e) - return False - finally: - shutil.rmtree(saved_path) - iso_collection_map = { component.get("component_name"): { -- Gitee From 8aec76d60e1185c3bccb7078889f0bac1ff2e458 Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Thu, 28 Mar 2024 14:13:12 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=AE=89=E8=A3=85A-FOT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- component/build_component.sh | 2 +- .../src/download/download_utils.py | 9 +++++++-- .../src/machine/local_machine.py | 11 +++++++++++ tools/install_dependency/src/machine/machine.py | 14 +++++++++++++- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/component/build_component.sh b/component/build_component.sh index 04978f4..ddb166c 100644 --- a/component/build_component.sh +++ b/component/build_component.sh @@ -30,7 +30,7 @@ function main() { mkdir -p "${final_component_dir}" component_arrays=( - "BiShengCompiler" "BiShengJDK8" "BiShengJDK17" "GCCforOpenEuler" "OpenEulerMirrorISO" "CompatibilityTesting" "NonInvasiveSwitching" + "BiShengCompiler" "BiShengJDK8" "BiShengJDK17" "GCCforOpenEuler" "OpenEulerMirrorISO" "CompatibilityTesting" "NonInvasiveSwitching" "A-FOT" ) for element in "${component_arrays[@]}"; do cp -rf "${current_dir}/${element}" "${final_component_dir}" diff --git a/tools/install_dependency/src/download/download_utils.py b/tools/install_dependency/src/download/download_utils.py index d073861..d9960e7 100644 --- a/tools/install_dependency/src/download/download_utils.py +++ b/tools/install_dependency/src/download/download_utils.py @@ -64,6 +64,10 @@ lkp_collection_map = { def download_a_fot(): + tar_file = os.path.join(DEFAULT_PATH, "a-fot.tar.gz") + if os.path.exists(tar_file) and os.path.isfile(tar_file): + return True + saved_path = os.path.join(DEFAULT_PATH, A_FOT) try: os.mkdir(saved_path) @@ -72,9 +76,10 @@ def download_a_fot(): try: for f in FILE_LIST: + print(f"Downloading from {BASE_URL.format(f)}") wget.download(BASE_URL.format(f), os.path.join(saved_path, f)) - - with tarfile.open(os.path.join(DEFAULT_PATH, "a-fot.tar.gz"), "w:gz") as t: + print() + with tarfile.open(tar_file, "w:gz") as t: t.add(saved_path, arcname="a-fot") return True except Exception as e: diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index d4e5077..af34286 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -62,9 +62,20 @@ class LocalMachine: "NonInvasiveSwitching": self.nis_install_component_handle, "OpenEulerMirrorISO": self.deploy_iso_handle, "UnOpenEulerMirrorISO": self.undeploy_iso_handle, + "A-FOT": self.install_a_fot, } + self._local_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD) + self._local_exec_command(YUM_INSTALL_LKP_DEPENDENCIES_CMD) return component_name_to_func_dict.get(component_name)(component_name) + def install_a_fot(self, component_name): + saved_path = os.path.join(constant.DEFAULT_PATH, "a-fot.tar.gz") + remote_file = os.path.abspath(os.path.join('/tmp', saved_path)) + LOGGER.debug(f"Copy local_file: {saved_path} to local machine {self.ip} remote_file: {remote_file}") + self._local_exec_command(f"/bin/cp -f {saved_path} {remote_file}") + self.nis_install_component_handle(component_name) + self.clear_tmp_file_at_local_machine([remote_file]) + def nis_install_component_handle(self, component_name): remote_file_list = [] # 上传并执行 安装脚本, 校验安装结果脚本 diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index f64835e..3fad35c 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -136,9 +136,21 @@ class Machine: "NonInvasiveSwitching": self.nis_install_component_handle, "OpenEulerMirrorISO": self.deploy_iso_handle, "UnOpenEulerMirrorISO": self.undeploy_iso_handle, + "A-FOT": self.install_a_fot, } + self._remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, ssh_client) + self._remote_exec_command(CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, ssh_client) return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) + def install_a_fot(self, component_name, sftp_client, ssh_client): + saved_path = os.path.join(constant.DEFAULT_PATH, "a-fot.tar.gz") + remote_file = os.path.abspath(os.path.join('/tmp', saved_path)) + LOGGER.debug(f"Transport local_file: {saved_path} to remote machine {self.ip} " + f"remote_file: {remote_file}") + sftp_client.put(localpath=f"{saved_path}", remotepath=f"{remote_file}") + self.nis_install_component_handle(component_name, sftp_client, ssh_client) + self.clear_tmp_file_at_remote_machine(ssh_client, [remote_file]) + def nis_install_component_handle(self, component_name, sftp_client, ssh_client): remote_file_list = [] # 上传并执行 安装脚本, 校验安装结果脚本 @@ -163,7 +175,7 @@ class Machine: else: LOGGER.error(f"Remote machine {self.ip} install {component_name} failed.") # 清理tmp临时文件 - self.clear_tmp_file_at_remote_machine(remote_file_list) + self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) def lkptest_install_component_handle(self, component_name, sftp_client, ssh_client): self._remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, ssh_client) -- Gitee