diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..02db4d00b2c815b72582d251162a74111d7ad8f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build +.vscode +__pycache__ +tools/install_dependency/src/component \ No newline at end of file diff --git a/component/DevkitDistribute/install.sh b/component/DevkitDistribute/install.sh index 83d7ecd4b5c03eaa52f5a440c53921c5933298a6..0080c065b5529e4376ee242fd1c4290be102cb5a 100644 --- a/component/DevkitDistribute/install.sh +++ b/component/DevkitDistribute/install.sh @@ -5,10 +5,10 @@ set -e function main (){ devkit_distribute_tar=$1 - echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local" - tar -zvxf "${devkit_distribute_tar}" -C /"${HOME}"/.local/lkp-tests/programs/devkit_distribute - chmod 777 /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh - ln -s /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh /"${HOME}"/.local/lkp-tests/tests/devkit_distribute + echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local/lkp-tests/programs" + tar -zvxf "${devkit_distribute_tar}" -C /"${HOME}"/.local/lkp-tests/programs + chmod 755 "${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh + ln -s "${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh "${HOME}"/.local/lkp-tests/tests/devkit_distribute echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local finished." } diff --git a/tools/common/devkit_utils/devkit_client.py b/tools/common/devkit_utils/devkit_client.py index 1941da2b0a99e7af0bc03506fe44bea6feab43eb..e508f3889dc75f0ca5584a885e5087d296a4cffe 100644 --- a/tools/common/devkit_utils/devkit_client.py +++ b/tools/common/devkit_utils/devkit_client.py @@ -91,7 +91,7 @@ class DevKitClient: if self.use_proxy: requests.delete(url=url, headers=self.header, verify=False) else: - requests.delete(url=url, headers=self.header, proxies=self.NO_PROXY) + requests.delete(url=url, headers=self.header, verify=False, proxies=self.NO_PROXY) def upload_report_by_force(self, file_path): ret = self.upload_report(file_path) diff --git a/tools/distribute/build_devkit_distribute.sh b/tools/distribute/build_devkit_distribute.sh index 697704547c0b987663a9e0f1781a92561f21233a..f99ce6c464b3a27f233c5997c7effa8ee50dec8d 100644 --- a/tools/distribute/build_devkit_distribute.sh +++ b/tools/distribute/build_devkit_distribute.sh @@ -36,6 +36,8 @@ mkdir -p devkit_distribute/log cp "${build_dir}"/dist/entrance devkit_distribute/bin cp -rf "${current_dir}"/devkit_distribute/config devkit_distribute +cp -rf "${current_dir}"/devkit_distribute/script/start.sh devkit_distribute/bin +cp -rf "${current_dir}"/devkit_distribute/script/generate_lkptest_config.sh devkit_distribute/bin cp devkit_pipeline_agent.tar.gz devkit_distribute/config tar -czf devkit_distribute.tar.gz devkit_distribute diff --git a/tools/distribute/devkit_distribute/bin/entrance.py b/tools/distribute/devkit_distribute/bin/entrance.py index 57b4d3ce34e05e980d1c8f6cc1c02090ca018901..4c3194c6c1a58e6236c840dbc41e7d621a002843 100644 --- a/tools/distribute/devkit_distribute/bin/entrance.py +++ b/tools/distribute/devkit_distribute/bin/entrance.py @@ -33,8 +33,9 @@ class Distributor: self.devkit_port = args.devkit_port self.devkit_user = args.devkit_user self.devkit_password = args.devkit_password - self.report = Report("./") file_utils.create_dir(self.data_path) + self.template_path=os.path.join(self.root_path, "config") + self.git_path = args.git_path def distribute(self): task_id = str(uuid.uuid4()) @@ -52,6 +53,11 @@ class Distributor: client.logout() # 清空本地jfr文件 file_utils.clear_dir(self.data_path) + report = Report(report_path=self.data_path,template_path=self.template_path, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + report.report() + self.__print_result(jfr_names) def __print_result(self, jfr_names): print("=============================================================") @@ -60,7 +66,7 @@ class Distributor: print(jfr_name) print(f"Please open the following address to view:\n" f"https://{self.devkit_ip}:{self.devkit_port}") - print(f"user :{self.devkit_user}, password: ${self.devkit_password}") + print(f"user :{self.devkit_user}, password: {self.devkit_password}") def obtain_jfrs(self, local_jfrs, task_id): # 顺序获取 @@ -186,6 +192,8 @@ def main(): help="the process names that can be multiple, each separated by a comma") parser.add_argument("-d", "--duration", required=True, dest="duration", type=int, help="the time of the sample") + parser.add_argument("--git-path", required=True, dest="git_path", type=str, + help="git path") parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) parser.set_defaults(password="") args = parser.parse_args() diff --git a/tools/distribute/devkit_distribute/bin/report/report.py b/tools/distribute/devkit_distribute/bin/report/report.py index aec913941d331e709180e8a126c278630b984bdd..39eea96abbf562340eeb8c1adaf85da9c5a849f7 100644 --- a/tools/distribute/devkit_distribute/bin/report/report.py +++ b/tools/distribute/devkit_distribute/bin/report/report.py @@ -17,23 +17,47 @@ git --git-dir={}/.git log --since=1.day \ """ JMETER_REPORT_NAME = "result.csv" HTML_TEMPLATE_NAME = "perf_report.html" -JMETER_SUMMARY_TEMPLATE_HOLDER = "/** jmeter-trend-data **/" -GIT_TEMPLATE_HOLDER = "/** git-history **/" +DEVKIT_REPORT_DATA_LINE_NUM = 32 +GIT_REPORT_DATA_LINE_NUM = 41 + class Report: - def __init__(self, report_path): + def __init__(self, report_path="./", template_path="./", git_path="./", jmeter_report_path="./", devkit_tool_ip="", devkit_tool_port="8086", devkit_user_name="devadmin"): if not os.path.isdir(report_path): raise Exception(f"Report path:{report_path} illegal.") - self.report_dir = os.path.join(report_path, str(time.time())) - os.mkdir(self.report_dir) - self.main_page = "" - self.summary_page = "" - self.git_log_page = "" - self.chart_page = "" + self.report_dir = report_path + self.git_path = git_path + self.template_path = template_path + self.jmeter_report_path = jmeter_report_path + self.devkit_tool_ip = devkit_tool_ip + self.devkit_tool_port = devkit_tool_port + self.devkit_user_name = devkit_user_name + + def report(self): + html_lines = self.read_template() + git_log = self.generate_git_log() + devkit_report_json = self.generate_devkit_html() + + html_lines[DEVKIT_REPORT_DATA_LINE_NUM] = "report_tb_data: {}".format(devkit_report_json) + html_lines[GIT_REPORT_DATA_LINE_NUM] = "git_tb_data: {},".format(git_log) + + final_report = os.path.join(self.report_dir, "devkit_performance_report.html") + with open(final_report, "w") as file: + file.writelines(html_lines) + return final_report - def generate_git_log(self, repo_path): - full_cmd = GIT_LOG_RECORD_COMMAND.format(repo_path) - parent_url_cmd = GIT_REMOTE_URL_COMMAND.format(repo_path) + def read_template(self): + html_lines = [] + with open(os.path.join(self.template_path, HTML_TEMPLATE_NAME), "r") as file: + html_lines = file.readlines() + return html_lines + + def generate_devkit_html(self): + return json.dumps(["Devkit URL", "user name", "https://{}:{}/#login".format(self.devkit_tool_ip, self.devkit_tool_port), self.devkit_user_name]) + + def generate_git_log(self): + full_cmd = GIT_LOG_RECORD_COMMAND.format(self.git_path) + parent_url_cmd = GIT_REMOTE_URL_COMMAND.format(self.git_path) parent_url = subprocess.Popen(parent_url_cmd, shell=True, stdout=subprocess.PIPE, encoding="utf-8").stdout.read() git_url = re.sub(PORT_SUB_PATTERN, '', parent_url.replace("ssh://git@", "https://").replace(".git", "")).strip("\n") + "/commit/" git_data = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8').stdout.readlines() @@ -41,13 +65,8 @@ class Report: for x in git_data: data.extend([value.strip("'").rstrip("'") if key != "commit" else git_url + value.strip("'").rstrip("'") for key, value in json.loads(x).items()]) git_log = json.dumps(data) + return git_log - with open(os.path.join(HTML_TEMPLATE_NAME), "r") as f: - html_lines = f.readlines() - res = [sub.replace(GIT_TEMPLATE_HOLDER, git_log) for sub in html_lines] - with open(os.path.join(HTML_TEMPLATE_NAME), "w") as file: - file.writelines(res) - def jmeter_report_to_html(self, jmeter_report_path): all_data= [] jmeter_report = os.path.join(jmeter_report_path, JMETER_REPORT_NAME) @@ -57,8 +76,4 @@ class Report: all_data.extend(row) all_data_json = json.dumps(all_data) - with open(HTML_TEMPLATE_NAME, "r") as file: - html_lines = file.readlines() - res = [sub.replace(JMETER_SUMMARY_TEMPLATE_HOLDER, all_data_json) for sub in html_lines] - with open(HTML_TEMPLATE_NAME, "w") as new_file: - new_file.writelines(res) + return all_data_json diff --git a/tools/distribute/devkit_distribute/bin/report/template/perf_report.html b/tools/distribute/devkit_distribute/bin/report/template/perf_report.html deleted file mode 100644 index 7a79e45e12c8625f05b2c4e549b34c0643f43a65..0000000000000000000000000000000000000000 --- a/tools/distribute/devkit_distribute/bin/report/template/perf_report.html +++ /dev/null @@ -1,25366 +0,0 @@ - - - - - - - Kunpeng DevKit - - - - - - - - - - - - - - - - - -
- -
-
-
-
- - - - \ No newline at end of file diff --git a/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml b/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml index d706c6f08438cef2a3c07e8dce0a4011b2dc402c..102034e6c72193e2c8cc04b4b0237156f203e9f4 100644 --- a/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml +++ b/tools/distribute/devkit_distribute/config/devkit_distribute_template.yaml @@ -13,4 +13,5 @@ devkit_user: ${devkit_user} devkit_password: ${devkit_password} applications: ${applications} duration: ${duration} +git_path: ${git_path} diff --git a/tools/distribute/devkit_distribute/config/perf_report.html b/tools/distribute/devkit_distribute/config/perf_report.html new file mode 100644 index 0000000000000000000000000000000000000000..ba5c1bc23f0cc59fb1b3e49238ae9686a06270f7 --- /dev/null +++ b/tools/distribute/devkit_distribute/config/perf_report.html @@ -0,0 +1,322 @@ + + + + + + + Kunpeng DevKit + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+ + + \ No newline at end of file diff --git a/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh b/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh index 0f284ca612f8d004157ee879d107dc5bff8d8846..ecd950c675d96ba14dc44c51fc53d51b7d45abfa 100644 --- a/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh +++ b/tools/distribute/devkit_distribute/script/generate_lkptest_config.sh @@ -21,8 +21,9 @@ local devkit_user="devadmin" local devkit_password="admin100" local applications="" local duration=10 +local git_path="" -while getopts "i:u:f:a:d:D:" opts; do +while getopts "i:u:f:a:d:D:g:" opts; do case $opts in i) ips_list=$OPTARG ;; @@ -36,6 +37,8 @@ while getopts "i:u:f:a:d:D:" opts; do duration=$OPTARG ;; D) devkit_ip=$OPTARG ;; + g) + git_path=$OPTARG ;; ?) echo "not recogize paramters";; esac @@ -52,6 +55,7 @@ sed -i "s/\${devkit_port}/${devkit_port}/g" "${root_path}/config/devkit_distribu sed -i "s/\${devkit_user}/${devkit_user}/g" "${root_path}/config/devkit_distribute.yaml" sed -i "s/\${devkit_password}/${devkit_password}/g" "${root_path}/config/devkit_distribute.yaml" sed -i "s?\${applications}?${applications}?g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s?\${git_path}?${git_path}?g" "${root_path}/config/devkit_distribute.yaml" sed -i "s/\${duration}/${duration}/g" "${root_path}/config/devkit_distribute.yaml" } diff --git a/tools/distribute/devkit_distribute/script/devkit_distribute_start.sh b/tools/distribute/devkit_distribute/script/start.sh similarity index 57% rename from tools/distribute/devkit_distribute/script/devkit_distribute_start.sh rename to tools/distribute/devkit_distribute/script/start.sh index 9347bd3ff9e02c9a1ebbd4b4e983894b2684ac7d..fb50848c0c1a70a92ee9fe4cdfb4fbabe2f9282d 100644 --- a/tools/distribute/devkit_distribute/script/devkit_distribute_start.sh +++ b/tools/distribute/devkit_distribute/script/start.sh @@ -2,8 +2,7 @@ # SourceCode build script # Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. - # shellcheck disable=SC2154 -"${root_path}/bin/entrance" -i "${ips_list}" -u "${user}" -p "${port}" -f "${pkey_file}" --duration "${duration}" --app "${applications}" \ +"${root_path}/bin/entrance" -i "${ips_list}" -u "${user}" --P "${port}" -f "${pkey_file}" --duration "${duration}" --app "${applications}" \ --devkit-ip "${devkit_ip}" --devkit-port "${devkit_port}" --devkit-password "${devkit_password}" --devkit-user "${devkit_user}" \ - --pkey-password "${pkey_password}" \ No newline at end of file + --pkey-password "${pkey_password}" --git-path "${git_path}" \ No newline at end of file