From 88192f8b7487c8b3829b1635ab4b5d1899d9d4fe Mon Sep 17 00:00:00 2001 From: zoupanpan Date: Wed, 20 Mar 2024 10:18:04 +0800 Subject: [PATCH] =?UTF-8?q?=E9=9B=86=E6=88=90jmeter=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 9 +- common/devkit_utils/devkit_client.py | 6 +- .../devkit_distribute/bin/entrance.py | 109 +++++++++++++++++- .../devkit_distribute/bin/report/report.py | 29 +++-- .../config/devkit_distribute_template.yaml | 1 + .../devkit_distribute/config/log.ini.template | 7 +- .../devkit_distribute/config/perf_report.html | 2 +- .../script/generate_lkptest_config.sh | 48 +++++--- .../devkit_distribute/script/start.sh | 2 +- tools/.DS_Store | Bin 6148 -> 0 bytes tools/distribute/.DS_Store | Bin 6148 -> 0 bytes tools/distribute/devkit_distribute/.DS_Store | Bin 6148 -> 0 bytes .../devkit_distribute/bin/.DS_Store | Bin 6148 -> 0 bytes .../devkit_distribute/bin/report/.DS_Store | Bin 6148 -> 0 bytes 15 files changed, 171 insertions(+), 42 deletions(-) delete mode 100644 .DS_Store delete mode 100644 tools/.DS_Store delete mode 100644 tools/distribute/.DS_Store delete mode 100644 tools/distribute/devkit_distribute/.DS_Store delete mode 100644 tools/distribute/devkit_distribute/bin/.DS_Store delete mode 100644 tools/distribute/devkit_distribute/bin/report/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 89fb2dbad82975cec03ac002f50f7196b2b6daa0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK&u`N(6n^eH<}i?Y0BIMbNL;JZwG$HBC3M|^1D6HC0Z?g{ibZ5`)g<&#Rmyq% zF_%!uas)H1$B(=^G8Zuh%r>@;`p?8!a3FW-k3Y91EjqRa>5*;~GP zrBoVS*T>Q8WHKMM9-OGG7$@0erU^+rLCCu|NfxX5K$TgXYi?*dq$j;m>tL}s?DgAz z@2J0Q`-^A2ZreZVKV2?8`S9_RbJz~doA~GNxm*kr+`!7?-k(n!G|*j7Au2#=|G{60Kg`i zwV}?x49xK@1{N!W=z$4C1sbZbM+{-;X!mVgV6if2=p^jnL)beDdqNTV?ik 0: + raise Exception(f"the directory {self.result_dir} is exist or not empty") + if not os.path.exists(self.jmx_file): + raise Exception(f"the jmx file {self.jmx_file} is not exist") + + class Distributor: def __init__(self, args): self.ips_list = args.ips_list.split(",") @@ -34,13 +92,25 @@ class Distributor: self.devkit_user = args.devkit_user self.devkit_password = args.devkit_password file_utils.create_dir(self.data_path) - self.template_path=os.path.join(self.root_path, "config") + self.template_path = os.path.join(self.root_path, "config") self.git_path = args.git_path + self.jmeter_command: JmeterCommand = JmeterCommand(args.jmeter_command) + self.jmeter_thread: typing.Optional[threading.Thread] = None + self.enable_jmeter_command = True if args.jmeter_command else False def distribute(self): + # jmeter 命令校验 + self.jmeter_command.check_and_init_jmeter_cmd() + # 校验联通性 + self.__check_ips_connected() + # 启动jmeter + if self.enable_jmeter_command: + self.__start_jmeter_thread() task_id = str(uuid.uuid4()) # 分发采集任务 self.distribute_to_sample_task(task_id) + if self.enable_jmeter_command: + self.jmeter_thread.join() # 获取jfr文件，删除任务文件 local_jfrs = list() self.obtain_jfrs(local_jfrs, task_id) @@ -53,12 +123,31 @@ class Distributor: client.logout() # 清空本地jfr文件 file_utils.clear_dir(self.data_path) - report = Report(report_path=self.data_path,template_path=self.template_path, - git_path=self.git_path, devkit_tool_ip=self.devkit_ip, - devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + # 等待jmeter完成 + if self.enable_jmeter_command: + report = Report(report_path=self.data_path, template_path=self.template_path, + jmeter_report_path=self.jmeter_command.csv_file, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + else: + report = Report(report_path=self.data_path, template_path=self.template_path, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) report.report() self.__print_result(jfr_names) + def __start_jmeter_thread(self): + self.jmeter_command.check_and_init_jmeter_cmd() + self.jmeter_thread = threading.Thread(target=self.__jmeter_start, args=(self.jmeter_command.origin_command,)) + self.jmeter_thread.start() + + @staticmethod + def __jmeter_start(command): + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=None) + logging.info("return_code: %s", outcome.return_code) + logging.info("error: %s", outcome.err) + logging.info("out: %s", outcome.out) + def __print_result(self, jfr_names): print("=============================================================") print("The following files have been uploaded to the DevKit server:") @@ -68,6 +157,16 @@ class Distributor: f"https://{self.devkit_ip}:{self.devkit_port}") print(f"user :{self.devkit_user}, password: {self.devkit_password}") + def __check_ips_connected(self): + for ip in self.ips_list: + factory = SSHClientFactory(ip=ip, user=self.user, port=self.port, password=self.password, + pkey_file=self.pkey_file, pkey_content=self.pkey_content, + pkey_password=self.pkey_password) + ssh_client = factory.create_ssh_client() + ssh_client.close() + client = DevKitClient(self.devkit_ip, self.devkit_port, self.devkit_user, self.devkit_password) + client.logout() + def obtain_jfrs(self, local_jfrs, task_id): # 顺序获取 for ip in self.ips_list: @@ -194,6 +293,8 @@ def main(): help="the time of the sample") parser.add_argument("--git-path", required=True, dest="git_path", type=str, help="git path") + parser.add_argument("--jmeter-command", dest="jmeter_command", type=str, + help="the command that start jmeter tests") parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) parser.set_defaults(password="") args = parser.parse_args() diff --git a/component/DevkitDistribute/devkit_distribute/bin/report/report.py b/component/DevkitDistribute/devkit_distribute/bin/report/report.py index 39eea96..31a8818 100644 --- a/component/DevkitDistribute/devkit_distribute/bin/report/report.py +++ b/component/DevkitDistribute/devkit_distribute/bin/report/report.py @@ -1,10 +1,8 @@ import csv import json import os -import subprocess -import time import re - +import subprocess PORT_SUB_PATTERN = re.compile(r':[0-9]+') GIT_REMOTE_URL_COMMAND = """ @@ -19,10 +17,15 @@ JMETER_REPORT_NAME = "result.csv" HTML_TEMPLATE_NAME = "perf_report.html" DEVKIT_REPORT_DATA_LINE_NUM = 32 GIT_REPORT_DATA_LINE_NUM = 41 +REPORT_VALID_LINE = 28 +JMETER_REPORT_DATA_HEADER_LEN = 35 +JMETER_REPORT_DATA_LINE_NUM = 36 class Report: - def __init__(self, report_path="./", template_path="./", git_path="./", jmeter_report_path="./", devkit_tool_ip="", devkit_tool_port="8086", devkit_user_name="devadmin"): + def __init__(self, report_path="./", template_path="./", git_path="./", + jmeter_report_path=None, devkit_tool_ip="", + devkit_tool_port="8086", devkit_user_name="devadmin"): if not os.path.isdir(report_path): raise Exception(f"Report path:{report_path} illegal.") self.report_dir = report_path @@ -32,6 +35,7 @@ class Report: self.devkit_tool_ip = devkit_tool_ip self.devkit_tool_port = devkit_tool_port self.devkit_user_name = devkit_user_name + self.jmeter_report_data_cols = 17 def report(self): html_lines = self.read_template() @@ -40,6 +44,11 @@ class Report: html_lines[DEVKIT_REPORT_DATA_LINE_NUM] = "report_tb_data: {}".format(devkit_report_json) html_lines[GIT_REPORT_DATA_LINE_NUM] = "git_tb_data: {},".format(git_log) + if self.jmeter_report_path: + html_lines[REPORT_VALID_LINE] = "const valid_pages = ['report', 'trend', 'git'];\n" + jmeter_report_data = self.jmeter_report_to_html() + html_lines[JMETER_REPORT_DATA_HEADER_LEN] = "trend_tb_cols: {},\n".format(self.jmeter_report_data_cols) + html_lines[JMETER_REPORT_DATA_LINE_NUM] = "trend_tb_data: {},\n".format(jmeter_report_data) final_report = os.path.join(self.report_dir, "devkit_performance_report.html") with open(final_report, "w") as file: @@ -67,10 +76,14 @@ class Report: git_log = json.dumps(data) return git_log - def jmeter_report_to_html(self, jmeter_report_path): - all_data= [] - jmeter_report = os.path.join(jmeter_report_path, JMETER_REPORT_NAME) - with open(jmeter_report, newline='') as csvfile: + def jmeter_report_to_html(self): + all_data = [] + with open(self.jmeter_report_path) as csvfile: + reader = csv.reader(csvfile) + for row in reader: + self.jmeter_report_data_cols = len(row) + break + with open(self.jmeter_report_path, newline='') as csvfile: reader = csv.reader(csvfile) for row in reader: all_data.extend(row) diff --git a/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml b/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml index 2bcc15b..09a23fc 100644 --- a/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml +++ b/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml @@ -15,4 +15,5 @@ devkit_password: ${devkit_password} applications: ${applications} duration: ${duration} git_path: ${git_path} +jmeter_command: "${jmeter_command}" diff --git a/component/DevkitDistribute/devkit_distribute/config/log.ini.template b/component/DevkitDistribute/devkit_distribute/config/log.ini.template index ac3abb5..2490d70 100644 --- a/component/DevkitDistribute/devkit_distribute/config/log.ini.template +++ b/component/DevkitDistribute/devkit_distribute/config/log.ini.template @@ -3,7 +3,7 @@ keys=root [handlers] # handlers 对象列表 -keys=consoleHandler,fileHandler +keys=consoleHandler [formatters] # formatters 对象列表 keys=fmt @@ -19,11 +19,6 @@ level = INFO formatter = fmt args = (sys.stdout,) -[handler_fileHandler]# fileHandler 控制器输出方向、级别、输出格式、参数 -class = FileHandler -level = INFO -formatter = fmt -args = ("LOG_PATH/LOG_NAME.log", "a") [formatter_fmt] format=[%(asctime)s] [%(levelname)s] [processID:%(process)d][%(threadName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s diff --git a/component/DevkitDistribute/devkit_distribute/config/perf_report.html b/component/DevkitDistribute/devkit_distribute/config/perf_report.html index ba5c1bc..e530a71 100644 --- a/component/DevkitDistribute/devkit_distribute/config/perf_report.html +++ b/component/DevkitDistribute/devkit_distribute/config/perf_report.html @@ -35,7 +35,7 @@ trend: { trend_tb_cols: 4, trend_tb_data: [], // must be countable, idx_0 represents x-axis - trend_chart_labels: [], + trend_chart_labels: ["TimeStamp", "Elapsed"], }, git: { git_tb_cols: 5, diff --git a/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh b/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh index ecd950c..b35fd2d 100644 --- a/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh +++ b/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh @@ -22,26 +22,35 @@ local devkit_password="admin100" local applications="" local duration=10 local git_path="" +local jmeter_command="" -while getopts "i:u:f:a:d:D:g:" opts; do - case $opts in - i) - ips_list=$OPTARG ;; - u) - user=$OPTARG ;; - f) - pkey_file=$OPTARG ;; - a) - applications=$OPTARG ;; - d) - duration=$OPTARG ;; - D) - devkit_ip=$OPTARG ;; - g) - git_path=$OPTARG ;; - ?) - echo "not recogize paramters";; - esac +while getopts "i:u:f:a:d:D:P:U:W:g:j:" opts; do + case $opts in + i) + ips_list=$OPTARG ;; + u) + user=$OPTARG ;; + f) + pkey_file=$OPTARG ;; + a) + applications=$OPTARG ;; + d) + duration=$OPTARG ;; + D) + devkit_ip=$OPTARG ;; + P) + devkit_port=$OPTARG ;; + U) + devkit_user=$OPTARG ;; + W) + devkit_password=$OPTARG ;; + g) + git_path=$OPTARG ;; + j) + jmeter_command=$OPTARG ;; + ?) + echo "The Parameters are not recognized";; + esac done sed -i "s?\${root_path}?${root_path}?g" "${root_path}/config/devkit_distribute.yaml" @@ -57,6 +66,7 @@ sed -i "s/\${devkit_password}/${devkit_password}/g" "${root_path}/config/devkit_ sed -i "s?\${applications}?${applications}?g" "${root_path}/config/devkit_distribute.yaml" sed -i "s?\${git_path}?${git_path}?g" "${root_path}/config/devkit_distribute.yaml" sed -i "s/\${duration}/${duration}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s?\${jmeter_command}?${jmeter_command}?g" "${root_path}/config/devkit_distribute.yaml" } diff --git a/component/DevkitDistribute/devkit_distribute/script/start.sh b/component/DevkitDistribute/devkit_distribute/script/start.sh index a2967ee..6ea230c 100644 --- a/component/DevkitDistribute/devkit_distribute/script/start.sh +++ b/component/DevkitDistribute/devkit_distribute/script/start.sh @@ -5,4 +5,4 @@ # shellcheck disable=SC2154 "${root_path}/bin/entrance" -i "${ips_list}" -u "${user}" -P "${port}" -f "${pkey_file}" --duration "${duration}" --app "${applications}" \ --devkit-ip "${devkit_ip}" --devkit-port "${devkit_port}" --devkit-password "${devkit_password}" --devkit-user "${devkit_user}" \ - --pkey-password "${pkey_password}" --git-path "${git_path}" \ No newline at end of file + --pkey-password "${pkey_password}" --git-path "${git_path}" --jmeter-command "${jmeter_command}" \ No newline at end of file diff --git a/tools/.DS_Store b/tools/.DS_Store deleted file mode 100644 index 5e2bd9676140361212eb36c42bc41ed5c88e6fac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKU5nE|6ur}}-PA=We!#v60bh%D+eHvxVqG76HKGqHHQ59WW@k#18l)8RtUtzA ze~Eub&z+f-)~>KmB64rIbMAbcN#@KjlOZBCU1mo_10o9GjGaD`3yka8cdX+}dO&6S zFr=TB<}@u$y3<+;{zV0N?s{}ehR*P8&mS*}NASBptl7`MN*NVYQB4bAN=rof#+)t#4;Q_cN4`IgShqN+!u@2a!cz0%tc_Jf1qP5j<8ah=r5YMLxwaqhV>WqKYb>5C$7 zW`moLjjoeI=L<_HvK&obzbtfSnyFdptg_tLRRsN@KN}pbR!8H>FdQFG*28f1U_2Uz z$CLZ(bw9Xu=ib9-vk&E&HlOW?NnveUyXNs6z96_zkB{*}mqx!sJ#%kBGeZq&DuD@d zGk8vOxVDkT41C&FQQZb}Q~B%ATU+H5^iy7{5B|cR%GDso{{8a^h E0#|2@)Bpeg diff --git a/tools/distribute/.DS_Store b/tools/distribute/.DS_Store deleted file mode 100644 index ca4fa5e6bd8895a471e5b31eefd0d0e28db7cd7a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKL2DC16n@jDcGDt6uuueHz-wsJqzK|=P3ys{Tk1h&yX%H_={gg#o8}M#d)6P* ztG~p*qu-mE+NLpJFCumxy!qa|dGBrJ+sw=k5sCgd+aqcbQ2=FZY@qp#@Hp#&tT@ja zP?#DS#gx&2F6ewA+B-Z0o`HXj0baXLD5WvwRAB91uiu<1)61!ZHNrSYRao6S@SMFV z^ry7YDe`&qcm>`0VFVV@qhI^L(mJ;)T;9DxIw^@6FnB`l8!u%l+Q-*{m6CJ==ZxHa;&tEB(c=K?3`uk%tbi z;46$j;OZh8sY0t$M02IBTkH<;c7}K}1yBKFO6U}*sG4V4JVo}+_k^yOd&+?OPX?R! zbpC$Yo&nFmat3&RkWj|ZVQtZD9cauI0N8-r2yFfw;2h~NbXZ%&2t?RapiLF-iXm(| z{L;mR4r_}xorJr52=`>+ZYaV$9rdLNClOlot!Kb9FwelM?Y8**f4%(vKcD1#o&nFm zf5m`k97czIOv#jzaDTTQCiJNvu}4k*50?)u7`+JZ#vv1Y7$WZWvn#N{6*N$`a~+;vIbN< zhdEC&lVHTtiTn!;@ZMD^p_KGVwzPi-@VxzH#V*80C9{0&cll>B&eFVSx9?P?T3f5H z2kXH`@HPHsW^pknrg?8L`OKq_#$?HTe~^4iN3(wO)uGYFAl0LZO-P3$gnT(o_0Y_E zW~zs|&5fNSXatRZb89}|?Q~mVXRo_xh4Xivb}QWL?k^UN;PsoG_aFM-vny?G9PtX@ zq?JvN_i&4jU(Izq(V5YguyAyp&dHFb3-EFIh>sue|H((4azsxMIi_McR&76u#ZW{T z5C(*S6=J}jkHN+Yy_39(Fdz)91_OLPgiyxFW9iTy9cXL`0Ib1m1%3PT4-DA=j69YO zVSy-@3UsN;ml(>|4$7s&ANlywp-U%apNx6@la;SflrKG0Xv0ZG4y6?agn?xSs&3fi z`~Ud)`hPh|M#6wF@Lw^YYDe)=4@>fQ>yyRtT^mBrpe!6$I($e$$6m#V<*RrLY6bR? WJHW_e=@1r({0LYYq!9*Ql!4!thHF;< diff --git a/tools/distribute/devkit_distribute/bin/.DS_Store b/tools/distribute/devkit_distribute/bin/.DS_Store deleted file mode 100644 index 4a74d9fe6b65a63810df1058ff8371ca55e24ad9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKO>Yx15FMwX-6%rJ0i+%+ag9ovR7i-+63T&Vw}Jzpu-mL)#bQUvrV)xF?V0|V zUil^cJM@k1AVPp6L}*4Dzwvm+yMDI(K}2deA0HESh$w7WoC5_-|0+s6{ zB|{gK(-BR!qQ$X`4Dj5wF=K`*n$nN+_xVL>FS62Rh~VY%6ngZ{2`r+0#OIi23#@*M zz5Y{dwwO&qzhARSSP*@ zt4{`xR;yNY|G~p2XQPkhCu6_35t72Hc5*l18m?e$jQ=v3n$nv0@UL^$v4X;YFdz)< z9s^;girTyPYw}0JfH3fn8Q}dvLmA_MwMVygpm0wBAYUV$z~;*$=jecOz}h2PAj+o# zeX0si4CT||*C8$rSbOy8q{8Dvg)gh{LQ(GP=wG{VQn5!Vg#lq;lYu>7?eO{k{P*ww z%_OM_1H!<6#eiy_CZ|ITDV(iagX6PSL*GGJIIi~iJq3n4iV@34@gdX+{F)759I*BX Q4@52kjs_`&fxpVYEzAgI8UO$Q diff --git a/tools/distribute/devkit_distribute/bin/report/.DS_Store b/tools/distribute/devkit_distribute/bin/report/.DS_Store deleted file mode 100644 index 514b652a5aa6777ddd1fe06b6f72cac9ff987832..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!H&}~5FK~BtfLC42axu%k|nOy(rs5rh)XETfh$3908|nhK}0sL+LQ`am2zG_ zhAUsf@4y?|MRW_|v;skS zqoA6`_#M-9CtE81Mg@58I&RJis_4i0`|zl=ud~u-$l%B0KK%HrGekrY%_zq_TVU;G z{4(RyC5A>`PUqm1lyba1f;#$Lezu%X!$10*P0M0l_xoQ|Yp=b3?LZ&sj((ZEwacVV z>&1MO&Yttt6Kl(C6Q|kJB43WXHy+rkPKzp^IYKeXk@DhMQBCY}WEa(B?zo9>(2d??LKOI;3VZ_>L3Dlq -- Gitee