diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c795b054e5ade51b7031abab1581a5b7e2d2f5ba --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build \ No newline at end of file diff --git a/build.sh b/build.sh index 26a8334aa7f5bb852812fd4642b7c8f9dab8708f..5938151bbf51f1bb855a1875d067abac9fb0cd8f 100644 --- a/build.sh +++ b/build.sh @@ -7,17 +7,18 @@ current_dir=$(cd $(dirname "$0"); pwd) tag="v0.1" -sh "${current_dir}"/tools/distribute/build_devkit_distribute.sh +bash "${current_dir}"/component/build_component.sh -sh "${current_dir}"/tools/download_dependency/build_download.sh +bash "${current_dir}"/tools/distribute/build_devkit_distribute.sh -sh "${current_dir}"/tools/install_dependency/build_install.sh +bash "${current_dir}"/tools/download_dependency/build_download.sh + +bash "${current_dir}"/tools/install_dependency/build_install.sh cd "${current_dir}"/build mkdir -p "${current_dir}"/build/devkit-pipeline-${tag}/linux cp -rf "${current_dir}"/build/install_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux cp -rf "${current_dir}"/build/download_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/distribute/devkit_distribute "${current_dir}"/build/devkit-pipeline-${tag}/linux tar -zcvf devkit-pipeline-${tag}.tar.gz devkit-pipeline-${tag} diff --git a/component/DevkitDistribute/check_install_result.sh b/component/DevkitDistribute/check_install_result.sh new file mode 100644 index 0000000000000000000000000000000000000000..c854e0b8883b64c58176ef8c4187798c0ce298d8 --- /dev/null +++ b/component/DevkitDistribute/check_install_result.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "true" \ No newline at end of file diff --git a/component/DevkitDistribute/install.sh b/component/DevkitDistribute/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..83d7ecd4b5c03eaa52f5a440c53921c5933298a6 --- /dev/null +++ b/component/DevkitDistribute/install.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + + +function main (){ + devkit_distribute_tar=$1 + echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local" + tar -zvxf "${devkit_distribute_tar}" -C /"${HOME}"/.local/lkp-tests/programs/devkit_distribute + chmod 777 /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh + ln -s /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh /"${HOME}"/.local/lkp-tests/tests/devkit_distribute + echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local finished." +} + + +main "$@" \ No newline at end of file diff --git a/component/build_component.sh b/component/build_component.sh new file mode 100644 index 0000000000000000000000000000000000000000..1bb9be76bf7698a52bc26e76ad342519564184ff --- /dev/null +++ b/component/build_component.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# SourceCode build script +# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +set -e +current_dir=$(cd $(dirname "$0"); pwd) +project_dir=$(realpath "${current_dir}/..") +umask 077 + +mkdir -p "${project_dir}"/build/component + +component_arrays=( + "BiShengCompiler" "BiShengJDK8" "BiShengJDK17" "CompatibilityTesting" "GCCforOpenEuler" "DevkitDistribute" + "LkpTests" "OpenEulerMirrorISO" +) + +for element in "${component_arrays[@]}"; + do + cp -rf "${project_dir}/component/${element}" "${project_dir}"/build/component + done + + diff --git a/tools/common/devkit_utils/devkit_client.py b/tools/common/devkit_utils/devkit_client.py index ebcef7cfb1f976992f0d209c1fd8387e88fe83c5..1941da2b0a99e7af0bc03506fe44bea6feab43eb 100644 --- a/tools/common/devkit_utils/devkit_client.py +++ b/tools/common/devkit_utils/devkit_client.py @@ -6,6 +6,8 @@ from urllib3 import encode_multipart_formdata class DevKitClient: + NO_PROXY = {"http": None, "https": None} + def __init__(self, ip, port, username, password): self.ip = ip self.port = port @@ -13,6 +15,7 @@ class DevKitClient: self.password = password self.user_id = "" self.token = "" + self.use_proxy = True self.header = dict() self.login() @@ -26,7 +29,17 @@ class DevKitClient: url = f"https://{self.ip}:{self.port}/framework/api/v1.0/users/session/" body = dict({"username": self.username, "password": self.password}) try: - ret = requests.post(url=url, json=body, verify=False, timeout=10) + self.try_to_login(url, body) + except Exception as err: + self.use_proxy = False + self.try_to_login(url, body) + + def try_to_login(self, url, body): + try: + if self.use_proxy: + ret = requests.post(url=url, json=body, verify=False, timeout=10) + else: + ret = requests.post(url=url, json=body, verify=False, timeout=10, proxies=self.NO_PROXY) user_dict = ret.json() self.token = ret.headers["token"] self.user_id = user_dict["data"]["id"] @@ -42,7 +55,10 @@ class DevKitClient: def logout(self): url = f"https://{self.ip}:{self.port}/framework/api/v1.0/users/session/{self.user_id}/" try: - requests.delete(url, headers=self.header, verify=False) + if self.use_proxy: + requests.delete(url, headers=self.header, verify=False) + else: + requests.delete(url, headers=self.header, verify=False, proxies=self.NO_PROXY) except Exception as ex: logging.exception(ex) pass @@ -57,16 +73,25 @@ class DevKitClient: _header = self.header.copy() _header.update({"Content-Type": encoded_data[1]}) url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/actions/upload/" - return requests.post(url=url, headers=_header, data=encoded_data[0], verify=False) + if self.use_proxy: + return requests.post(url=url, headers=_header, data=encoded_data[0], verify=False) + else: + return requests.post(url=url, headers=_header, data=encoded_data[0], verify=False, proxies=self.NO_PROXY) def get_record_list(self): url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/user/" data = {"userId": self.user_id} - return requests.post(url=url, json=data, headers=self.header, verify=False) + if self.use_proxy: + return requests.post(url=url, json=data, headers=self.header, verify=False) + else: + return requests.post(url=url, json=data, headers=self.header, verify=False, proxies=self.NO_PROXY) def delete_report(self, task_id): url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/{task_id}/" - requests.delete(url=url, headers=self.header, verify=False) + if self.use_proxy: + requests.delete(url=url, headers=self.header, verify=False) + else: + requests.delete(url=url, headers=self.header, proxies=self.NO_PROXY) def upload_report_by_force(self, file_path): ret = self.upload_report(file_path) diff --git a/tools/common/devkit_utils/pyinstaller_utils.py b/tools/common/devkit_utils/pyinstaller_utils.py index 604ee668fbe447ed25d55bfebe1f953a6dceabaf..91f578bcd2b2e90b671e9d36e432a031826aceb6 100644 --- a/tools/common/devkit_utils/pyinstaller_utils.py +++ b/tools/common/devkit_utils/pyinstaller_utils.py @@ -14,7 +14,7 @@ def check_is_running_in_pyinstaller_bundle(): def obtain_root_path(root_path): """ - 获取rootpath,当在pyinstaller中时,为父母录,否认为参入的参数 + 获取rootpath,当在pyinstaller中时,为父目录,否则为传入的参数 """ if check_is_running_in_pyinstaller_bundle(): return os.path.dirname(os.path.dirname(sys.executable)) diff --git a/tools/distribute/build_devkit_distribute.sh b/tools/distribute/build_devkit_distribute.sh index 05e9205adb5b6906900bfcbdb7e5d18359e06d71..697704547c0b987663a9e0f1781a92561f21233a 100644 --- a/tools/distribute/build_devkit_distribute.sh +++ b/tools/distribute/build_devkit_distribute.sh @@ -35,6 +35,9 @@ mkdir -p devkit_distribute/data mkdir -p devkit_distribute/log cp "${build_dir}"/dist/entrance devkit_distribute/bin -cp "${current_dir}"/devkit_distribute/script/devkit_pipeline_start.sh devkit_distribute/bin cp -rf "${current_dir}"/devkit_distribute/config devkit_distribute cp devkit_pipeline_agent.tar.gz devkit_distribute/config + +tar -czf devkit_distribute.tar.gz devkit_distribute + +cp devkit_distribute.tar.gz "${project_dir}"/build/component diff --git a/tools/distribute/devkit_distribute/bin/entrance.py b/tools/distribute/devkit_distribute/bin/entrance.py index 0a58251602ee600d4681ce57de375f873407b0fe..57b4d3ce34e05e980d1c8f6cc1c02090ca018901 100644 --- a/tools/distribute/devkit_distribute/bin/entrance.py +++ b/tools/distribute/devkit_distribute/bin/entrance.py @@ -45,12 +45,23 @@ class Distributor: self.obtain_jfrs(local_jfrs, task_id) # 发送至 Devkit client = DevKitClient(self.devkit_ip, self.devkit_port, self.devkit_user, self.devkit_password) + jfr_names = list() for jfr in local_jfrs: - client.upload_report(jfr) + jfr_names.append(os.path.basename(jfr)) + client.upload_report_by_force(jfr) client.logout() # 清空本地jfr文件 file_utils.clear_dir(self.data_path) + def __print_result(self, jfr_names): + print("=============================================================") + print("The following files have been uploaded to the DevKit server:") + for jfr_name in jfr_names: + print(jfr_name) + print(f"Please open the following address to view:\n" + f"https://{self.devkit_ip}:{self.devkit_port}") + print(f"user :{self.devkit_user}, password: ${self.devkit_password}") + def obtain_jfrs(self, local_jfrs, task_id): # 顺序获取 for ip in self.ips_list: @@ -154,11 +165,9 @@ def main(): parser.add_argument("-i", "--ips", required=True, dest="ips_list", help="the machine ips on which the java application is running ") parser.add_argument("-u", "--user", required=True, dest="user", default="root", - help="the user password of the ips") + help="the user of the ips") parser.add_argument("-P", "--port", dest="port", type=int, default=22, help="the ssh port of the ips") - parser.add_argument("-p", "--password", dest="password", - help="the user password of the ips") parser.add_argument("-f", "--pkey-file", dest="pkey_file", help="the file path of the private key") parser.add_argument("-c", "--pkey-content", dest="pkey_content", @@ -166,18 +175,19 @@ def main(): parser.add_argument("-w", "--pkey-password", dest="pkey_password", help="the private key password") parser.add_argument("--devkit-ip", dest="devkit_ip", required=True, - help="the process names that can be multiple, each separated by a comma") + help="the ip of the kunpeng DevKit server") parser.add_argument("--devkit-port", dest="devkit_port", default="8086", - help="the process names that can be multiple, each separated by a comma") + help="the port of the kunpeng DevKit server") parser.add_argument("--devkit-user", dest="devkit_user", default="devadmin", - help="the process names that can be multiple, each separated by a comma") + help="the user of the kunpeng DevKit server") parser.add_argument("--devkit-password", dest="devkit_password", default="admin100", - help="the process names that can be multiple, each separated by a comma") + help="the password of the user of the kunpeng DevKit server") parser.add_argument("-a", "--app", required=True, dest="applications", help="the process names that can be multiple, each separated by a comma") parser.add_argument("-d", "--duration", required=True, dest="duration", type=int, help="the time of the sample") parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) + parser.set_defaults(password="") args = parser.parse_args() config_log_ini(args.root_path, "devkit_distribute") logging.info("devkit_distribute start") diff --git a/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml b/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d706c6f08438cef2a3c07e8dce0a4011b2dc402c --- /dev/null +++ b/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml @@ -0,0 +1,16 @@ +suite: devkit_distribute +category: functional + +root_path: ${root_path} +ips_list: ${ips_list} +user: ${user} +port: ${port} +pkey_file: ${pkey_file} +pkey_password: ${pkey_password} +devkit_ip: ${devkit_ip} +devkit_port: ${devkit_port} +devkit_user: ${devkit_user} +devkit_password: ${devkit_password} +applications: ${applications} +duration: ${duration} + diff --git a/tools/distribute/devkit_distribute/script/devkit_distribute_start.sh b/tools/distribute/devkit_distribute/script/devkit_distribute_start.sh new file mode 100644 index 0000000000000000000000000000000000000000..9347bd3ff9e02c9a1ebbd4b4e983894b2684ac7d --- /dev/null +++ b/tools/distribute/devkit_distribute/script/devkit_distribute_start.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SourceCode build script +# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + + +# shellcheck disable=SC2154 +"${root_path}/bin/entrance" -i "${ips_list}" -u "${user}" -p "${port}" -f "${pkey_file}" --duration "${duration}" --app "${applications}" \ + --devkit-ip "${devkit_ip}" --devkit-port "${devkit_port}" --devkit-password "${devkit_password}" --devkit-user "${devkit_user}" \ + --pkey-password "${pkey_password}" \ No newline at end of file diff --git a/tools/distribute/devkit_distribute/script/devkit_pipeline_start.sh b/tools/distribute/devkit_distribute/script/devkit_pipeline_start.sh deleted file mode 100644 index 880e0d83b6fb1632f5cd78f06ee7b5a0d44e3bf4..0000000000000000000000000000000000000000 --- a/tools/distribute/devkit_distribute/script/devkit_pipeline_start.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# shellcheck disable=SC2164 -current_path=$(cd $(dirname "$0"); pwd) - -nohup "${current_path}"/entrance "$@" >/dev/null 2>&1 & - -echo "start devkit success" \ No newline at end of file diff --git a/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh b/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh new file mode 100644 index 0000000000000000000000000000000000000000..0f284ca612f8d004157ee879d107dc5bff8d8846 --- /dev/null +++ b/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SourceCode build script +# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. + +set -ex +current_dir=$(cd $(dirname "$0"); pwd) +root_path=$(realpath "${current_dir}/..") +umask 077 +cp "${root_path}/config/devkit_distribute_template.yaml" "${root_path}/config/devkit_distribute.yaml" + +function main() { + +local ips_list="" +local user="root" +local port=22 +local pkey_file="" +local pkey_password="" +local devkit_ip="" +local devkit_port=8086 +local devkit_user="devadmin" +local devkit_password="admin100" +local applications="" +local duration=10 + +while getopts "i:u:f:a:d:D:" opts; do + case $opts in + i) + ips_list=$OPTARG ;; + u) + user=$OPTARG ;; + f) + pkey_file=$OPTARG ;; + a) + applications=$OPTARG ;; + d) + duration=$OPTARG ;; + D) + devkit_ip=$OPTARG ;; + ?) + echo "not recogize paramters";; + esac +done + +sed -i "s?\${root_path}?${root_path}?g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${ips_list}/${ips_list}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${user}/${user}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${port}/${port}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s?\${pkey_file}?${pkey_file}?g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${pkey_password}/${pkey_password}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${devkit_ip}/${devkit_ip}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${devkit_port}/${devkit_port}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${devkit_user}/${devkit_user}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${devkit_password}/${devkit_password}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s?\${applications}?${applications}?g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s/\${duration}/${duration}/g" "${root_path}/config/devkit_distribute.yaml" + +} + +main "$@" + + + + + + diff --git a/tools/download_dependency/build_download.sh b/tools/download_dependency/build_download.sh index 5b4ba6a3b4d12aaf9a13d3d8cc2df8130b935b56..500aabdf9ee88cdcab58099a1d3c4e814ad192e5 100644 --- a/tools/download_dependency/build_download.sh +++ b/tools/download_dependency/build_download.sh @@ -14,4 +14,6 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src --add-data "../../tools/download_dependency/lkp_help:lkp_help" --add-data "../../tools/download_dependency/compatibility_test_help:compatibility_test_help" +pyinstaller -F "${current_dir}"/src/download.py -p "${current_dir}"/src \ + --add-data "../../tools/download_dependency/lkp_help:lkp_help" \ + --add-data "../../tools/download_dependency/compatibility_test_help:compatibility_test_help" diff --git a/tools/install_dependency/build_install.sh b/tools/install_dependency/build_install.sh index d7c99be1961adae786cc425f1243e920c3d14433..eb9db8d1e43cf83aeb04d354cb06c2f9aaa317dc 100644 --- a/tools/install_dependency/build_install.sh +++ b/tools/install_dependency/build_install.sh @@ -14,6 +14,8 @@ mkdir -p "${build_dir}" cd "${build_dir}" -pyinstaller -F "${current_dir}"/src/devkitpipeline.py -p "${current_dir}/src:${project_dir}/tools/download_dependency/src" --add-data "../../component:component" --add-data "${project_dir}/tools/download_dependency/lkp_help:lkp_help" --add-data "${project_dir}/tools/download_dependency/compatibility_test_help:compatibility_test_help" +pyinstaller -F "${current_dir}"/src/devkitpipeline.py -p "${current_dir}/src:${project_dir}/tools/download_dependency/src" \ + --add-data "../../build/component:component" --add-data "${project_dir}/tools/download_dependency/lkp_help:lkp_help" \ + --add-data "${project_dir}/tools/download_dependency/compatibility_test_help:compatibility_test_help" cp "${current_dir}"/config/machine.yaml "${build_dir}"/dist/machine.yaml diff --git a/tools/install_dependency/src/lkp_collect_map.py b/tools/install_dependency/src/lkp_collect_map.py index b92f0acc8834ad32e3ca763aa1a3a3a8471e2d78..f3da45e39caea64681085797e7c645a206fc6def 100644 --- a/tools/install_dependency/src/lkp_collect_map.py +++ b/tools/install_dependency/src/lkp_collect_map.py @@ -16,5 +16,10 @@ lkp_collection_map = { "download file": { "save_path": f"{os.path.join('./', constant.DEPENDENCY_DIR, 'compatibility_testing.tar.gz')}", } + }, + "DevkitDistribute": { + "download file": { + "save_path": f"{os.path.join(base_path('component'), 'devkit_distribute.tar.gz')}", + } } } diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index c14e545fa9a4fd69456f31fd15e003de26a6d842..8868df9ad0feead81080680e3cc6f0aa2372b9f6 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -1,14 +1,16 @@ +import logging import os -import paramiko import socket -import logging +import typing + +import paramiko import timeout_decorator import constant from command_line import CommandLine +from download import component_collection_map from exception.connect_exception import CreatePkeyFailedException, ConnectRemoteException, \ NotMatchedMachineTypeException -from download import component_collection_map from lkp_collect_map import lkp_collection_map from utils import base_path @@ -97,17 +99,18 @@ class Machine: sftp_client.close() def install_component_handler(self, component_name, sftp_client, ssh_client): - component_name_to_func_dict = { + component_name_to_func_dict: typing.Dict[ + str, typing.Callable[[str, paramiko.SFTPClient, paramiko.SSHClient], typing.Any]] = { "GCCforOpenEuler": self.default_install_component_handle, "BiShengCompiler": self.default_install_component_handle, "BiShengJDK17": self.default_install_component_handle, "BiShengJDK8": self.default_install_component_handle, - "LkpTests": self.lkpTest_install_component_handle, + "LkpTests": self.lkptest_install_component_handle, "OpenEulerMirrorISO": self.deploy_iso_handle, } return component_name_to_func_dict.get(component_name)(component_name, sftp_client, ssh_client) - def lkpTest_install_component_handle(self, component_name, sftp_client, ssh_client): + def lkptest_install_component_handle(self, component_name, sftp_client, ssh_client): try: stdin, stdout, stderr = ssh_client.exec_command(f"mkdir -p /tmp/{constant.DEPENDENCY_DIR}", timeout=10) stdin, stdout, stderr = ssh_client.exec_command(f"yum install -y git wget rubygems", timeout=100) @@ -160,9 +163,10 @@ class Machine: LOGGER.info(f"Remote machine {self.ip} install {component_name} failed.") # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) - self.compatibilityTest_install_component_handle("CompatibilityTesting", sftp_client, ssh_client) + self.__install_component_on_lkptest("CompatibilityTesting", sftp_client, ssh_client) + self.__install_component_on_lkptest("DevkitDistribute", sftp_client, ssh_client) - def compatibilityTest_install_component_handle(self, component_name, sftp_client, ssh_client): + def __install_component_on_lkptest(self, component_name, sftp_client, ssh_client): # 上传 compatibility_testing.tar.gz文件 LOGGER.info(f"Install component in remote machine {self.ip}: {component_name}") shell_dict = lkp_collection_map.get(component_name) @@ -284,7 +288,8 @@ class Machine: # 清理tmp临时文件 self.clear_tmp_file_at_remote_machine(ssh_client, remote_file_list) - def transport_shell_file_and_execute(self, ssh_client, sftp_client, sh_file_local_path, sh_file_remote_path, sh_cmd): + def transport_shell_file_and_execute(self, ssh_client, sftp_client, sh_file_local_path, sh_file_remote_path, + sh_cmd): if not os.path.exists(sh_file_local_path): LOGGER.error(f"{sh_file_local_path} not exists.") raise FileNotFoundError(f"local file {sh_file_local_path} not exists.")