diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..02db4d00b2c815b72582d251162a74111d7ad8f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build +.vscode +__pycache__ +tools/install_dependency/src/component \ No newline at end of file diff --git a/component/DevkitDistribute/install.sh b/component/DevkitDistribute/install.sh index 83d7ecd4b5c03eaa52f5a440c53921c5933298a6..0080c065b5529e4376ee242fd1c4290be102cb5a 100644 --- a/component/DevkitDistribute/install.sh +++ b/component/DevkitDistribute/install.sh @@ -5,10 +5,10 @@ set -e function main (){ devkit_distribute_tar=$1 - echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local" - tar -zvxf "${devkit_distribute_tar}" -C /"${HOME}"/.local/lkp-tests/programs/devkit_distribute - chmod 777 /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh - ln -s /"${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh /"${HOME}"/.local/lkp-tests/tests/devkit_distribute + echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local/lkp-tests/programs" + tar -zvxf "${devkit_distribute_tar}" -C /"${HOME}"/.local/lkp-tests/programs + chmod 755 "${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh + ln -s "${HOME}"/.local/lkp-tests/programs/devkit_distribute/bin/start.sh "${HOME}"/.local/lkp-tests/tests/devkit_distribute echo "Decompress devkit_distribute.tar.gz to ${HOME}/.local finished." } diff --git a/tools/common/devkit_utils/devkit_client.py b/tools/common/devkit_utils/devkit_client.py index 1941da2b0a99e7af0bc03506fe44bea6feab43eb..e508f3889dc75f0ca5584a885e5087d296a4cffe 100644 --- a/tools/common/devkit_utils/devkit_client.py +++ b/tools/common/devkit_utils/devkit_client.py @@ -91,7 +91,7 @@ class DevKitClient: if self.use_proxy: requests.delete(url=url, headers=self.header, verify=False) else: - requests.delete(url=url, headers=self.header, proxies=self.NO_PROXY) + requests.delete(url=url, headers=self.header, verify=False, proxies=self.NO_PROXY) def upload_report_by_force(self, file_path): ret = self.upload_report(file_path) diff --git a/tools/distribute/build_devkit_distribute.sh b/tools/distribute/build_devkit_distribute.sh index 697704547c0b987663a9e0f1781a92561f21233a..f99ce6c464b3a27f233c5997c7effa8ee50dec8d 100644 --- a/tools/distribute/build_devkit_distribute.sh +++ b/tools/distribute/build_devkit_distribute.sh @@ -36,6 +36,8 @@ mkdir -p devkit_distribute/log cp "${build_dir}"/dist/entrance devkit_distribute/bin cp -rf "${current_dir}"/devkit_distribute/config devkit_distribute +cp -rf "${current_dir}"/devkit_distribute/script/start.sh devkit_distribute/bin +cp -rf "${current_dir}"/devkit_distribute/script/generate_lkptest_config.sh devkit_distribute/bin cp devkit_pipeline_agent.tar.gz devkit_distribute/config tar -czf devkit_distribute.tar.gz devkit_distribute diff --git a/tools/distribute/devkit_distribute/bin/entrance.py b/tools/distribute/devkit_distribute/bin/entrance.py index 57b4d3ce34e05e980d1c8f6cc1c02090ca018901..4c3194c6c1a58e6236c840dbc41e7d621a002843 100644 --- a/tools/distribute/devkit_distribute/bin/entrance.py +++ b/tools/distribute/devkit_distribute/bin/entrance.py @@ -33,8 +33,9 @@ class Distributor: self.devkit_port = args.devkit_port self.devkit_user = args.devkit_user self.devkit_password = args.devkit_password - self.report = Report("./") file_utils.create_dir(self.data_path) + self.template_path=os.path.join(self.root_path, "config") + self.git_path = args.git_path def distribute(self): task_id = str(uuid.uuid4()) @@ -52,6 +53,11 @@ class Distributor: client.logout() # 清空本地jfr文件 file_utils.clear_dir(self.data_path) + report = Report(report_path=self.data_path,template_path=self.template_path, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + report.report() + self.__print_result(jfr_names) def __print_result(self, jfr_names): print("=============================================================") @@ -60,7 +66,7 @@ class Distributor: print(jfr_name) print(f"Please open the following address to view:\n" f"https://{self.devkit_ip}:{self.devkit_port}") - print(f"user :{self.devkit_user}, password: ${self.devkit_password}") + print(f"user :{self.devkit_user}, password: {self.devkit_password}") def obtain_jfrs(self, local_jfrs, task_id): # 顺序获取 @@ -186,6 +192,8 @@ def main(): help="the process names that can be multiple, each separated by a comma") parser.add_argument("-d", "--duration", required=True, dest="duration", type=int, help="the time of the sample") + parser.add_argument("--git-path", required=True, dest="git_path", type=str, + help="git path") parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) parser.set_defaults(password="") args = parser.parse_args() diff --git a/tools/distribute/devkit_distribute/bin/report/report.py b/tools/distribute/devkit_distribute/bin/report/report.py index aec913941d331e709180e8a126c278630b984bdd..39eea96abbf562340eeb8c1adaf85da9c5a849f7 100644 --- a/tools/distribute/devkit_distribute/bin/report/report.py +++ b/tools/distribute/devkit_distribute/bin/report/report.py @@ -17,23 +17,47 @@ git --git-dir={}/.git log --since=1.day \ """ JMETER_REPORT_NAME = "result.csv" HTML_TEMPLATE_NAME = "perf_report.html" -JMETER_SUMMARY_TEMPLATE_HOLDER = "/** jmeter-trend-data **/" -GIT_TEMPLATE_HOLDER = "/** git-history **/" +DEVKIT_REPORT_DATA_LINE_NUM = 32 +GIT_REPORT_DATA_LINE_NUM = 41 + class Report: - def __init__(self, report_path): + def __init__(self, report_path="./", template_path="./", git_path="./", jmeter_report_path="./", devkit_tool_ip="", devkit_tool_port="8086", devkit_user_name="devadmin"): if not os.path.isdir(report_path): raise Exception(f"Report path:{report_path} illegal.") - self.report_dir = os.path.join(report_path, str(time.time())) - os.mkdir(self.report_dir) - self.main_page = "" - self.summary_page = "" - self.git_log_page = "" - self.chart_page = "" + self.report_dir = report_path + self.git_path = git_path + self.template_path = template_path + self.jmeter_report_path = jmeter_report_path + self.devkit_tool_ip = devkit_tool_ip + self.devkit_tool_port = devkit_tool_port + self.devkit_user_name = devkit_user_name + + def report(self): + html_lines = self.read_template() + git_log = self.generate_git_log() + devkit_report_json = self.generate_devkit_html() + + html_lines[DEVKIT_REPORT_DATA_LINE_NUM] = "report_tb_data: {}".format(devkit_report_json) + html_lines[GIT_REPORT_DATA_LINE_NUM] = "git_tb_data: {},".format(git_log) + + final_report = os.path.join(self.report_dir, "devkit_performance_report.html") + with open(final_report, "w") as file: + file.writelines(html_lines) + return final_report - def generate_git_log(self, repo_path): - full_cmd = GIT_LOG_RECORD_COMMAND.format(repo_path) - parent_url_cmd = GIT_REMOTE_URL_COMMAND.format(repo_path) + def read_template(self): + html_lines = [] + with open(os.path.join(self.template_path, HTML_TEMPLATE_NAME), "r") as file: + html_lines = file.readlines() + return html_lines + + def generate_devkit_html(self): + return json.dumps(["Devkit URL", "user name", "https://{}:{}/#login".format(self.devkit_tool_ip, self.devkit_tool_port), self.devkit_user_name]) + + def generate_git_log(self): + full_cmd = GIT_LOG_RECORD_COMMAND.format(self.git_path) + parent_url_cmd = GIT_REMOTE_URL_COMMAND.format(self.git_path) parent_url = subprocess.Popen(parent_url_cmd, shell=True, stdout=subprocess.PIPE, encoding="utf-8").stdout.read() git_url = re.sub(PORT_SUB_PATTERN, '', parent_url.replace("ssh://git@", "https://").replace(".git", "")).strip("\n") + "/commit/" git_data = subprocess.Popen(full_cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8').stdout.readlines() @@ -41,13 +65,8 @@ class Report: for x in git_data: data.extend([value.strip("'").rstrip("'") if key != "commit" else git_url + value.strip("'").rstrip("'") for key, value in json.loads(x).items()]) git_log = json.dumps(data) + return git_log - with open(os.path.join(HTML_TEMPLATE_NAME), "r") as f: - html_lines = f.readlines() - res = [sub.replace(GIT_TEMPLATE_HOLDER, git_log) for sub in html_lines] - with open(os.path.join(HTML_TEMPLATE_NAME), "w") as file: - file.writelines(res) - def jmeter_report_to_html(self, jmeter_report_path): all_data= [] jmeter_report = os.path.join(jmeter_report_path, JMETER_REPORT_NAME) @@ -57,8 +76,4 @@ class Report: all_data.extend(row) all_data_json = json.dumps(all_data) - with open(HTML_TEMPLATE_NAME, "r") as file: - html_lines = file.readlines() - res = [sub.replace(JMETER_SUMMARY_TEMPLATE_HOLDER, all_data_json) for sub in html_lines] - with open(HTML_TEMPLATE_NAME, "w") as new_file: - new_file.writelines(res) + return all_data_json diff --git a/tools/distribute/devkit_distribute/bin/report/template/perf_report.html b/tools/distribute/devkit_distribute/bin/report/template/perf_report.html deleted file mode 100644 index 7a79e45e12c8625f05b2c4e549b34c0643f43a65..0000000000000000000000000000000000000000 --- a/tools/distribute/devkit_distribute/bin/report/template/perf_report.html +++ /dev/null @@ -1,25366 +0,0 @@ - - - -
- - -