diff --git a/test/scripts/email_config.yaml b/test/scripts/email_config.yaml index 77a779d374bd56457f52577dcd3a8fd7b33e4dfd..10e9b8c9ed1d9ee26260c17e0b31216ce8a41bb0 100644 --- a/test/scripts/email_config.yaml +++ b/test/scripts/email_config.yaml @@ -11,17 +11,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -sender_email_address : -auth_code : -receiver_list : -smtp_server : -smtp_port : +user_name : "" +sender_email_address : "" +auth_code : "" +receiver_list : + - "" +smtp_server: "" +smtp_port: "25" -xts_report_file : ".\\auto_xts_test\\result\\summary_report.html" -sdk_report_file : ".\\sdk_test\\sdk_test_report.html" -perf_report_file : "" +xts_report_file : "./auto_xts_test/result/summary_report.html" +sdk_report_file : "./sdk_test/sdk_test_report.html" +perf_report_file : "./performance_test/mail_data/email_msg.html" attatchment_files : - - ".\\auto_xts_test\\result\\details_report.html" - - ".\\auto_xts_test\\result\\failures_report.html" - - ".\\sdk_test\\sdk_test_report.html" - - ".\\sdk_test\\sdk_test_log.txt" + - "./auto_xts_test/result/details_report.html" + - "./auto_xts_test/result/failures_report.html" + - "./performance_test/mail_data/performance_logs.zip" +image_files: + "./performance_test/mail_data/debug_full_time.jpg": performance00 + "./performance_test/mail_data/debug_incremental_time.jpg": performance01 + "./performance_test/mail_data/release_full_time.jpg": performance10 + "./performance_test/mail_data/release_incremental_time.jpg": performance11 diff --git a/test/scripts/performance_test/performance_build.py b/test/scripts/performance_test/performance_build.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d7572c569e41ebc8b4b7c48932a404d1a95593 --- /dev/null +++ b/test/scripts/performance_test/performance_build.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import subprocess +import stat +import time +import zipfile + +import performance_config + + +class PerformanceBuild(): + def __init__(self, config_input, mail_obj): + self.config = None + self.first_line_in_avg_excel = "" + self.time_avg_dic = {} + self.all_time_dic = {} + self.size_avg_dic = {} + self.all_size_dic = {} + self.mail_helper = None + self.built_times = 0 + self.mail_msg = '' + # If developing_test_mode is True, the project will not be built and use following test data + self.developing_test_mode = False + self.developing_test_data = 'entry:clean|18 ms :clean|1 ms total build cost|1 s 204 ms ' + \ + 'entry:default@OneTime|1 s 204 ms ' + \ + 'entry:default@MergeProfile|4 ms entry:default@BuildNativeWithCmake|1 ms ' + \ + 'entry:default@GenerateLoaderJson|2 ms entry:default@MakePackInfo|8 ms ' + \ + 'entry:default@ProcessProfile|85 ms entry:default@BuildNativeWithNinja|1 ms ' + \ + 'entry:default@ProcessResource|2 ms entry:default@ProcessLibs|3 ms entry:default@CompileResource|78 ms ' + \ + 'entry:default@CompileJS|2 ms entry:default@CompileArkTS|2 s 591 ms entry:default@PackageHap|924 ms ' + \ + 'entry:default@SignHap|2 ms entry:assembleHap|1 ms total build cost|4 s 886 ms ' + \ + 'entry:default@BuildNativeWithCmake|1 ms entry:default@BuildNativeWithNinja|1 ms ' + \ + 'entry:default@CompileJS|4 ms entry:default@CompileArkTS|2 s 472 ms entry:default@PackageHap|919 ms ' + \ + 'entry:assembleHap|1 ms entry:default@SignHap|2 ms total build cost|4 s 632 ms ' + \ + 'entry:clean|18 ms :clean|1 ms total build cost|1 s 204 ms entry:default@MergeProfile|4 ms ' + \ + 'entry:default@BuildNativeWithCmake|1 ms entry:default@GenerateLoaderJson|2 ms ' + \ + 'entry:default@MakePackInfo|8 ms entry:default@ProcessProfile|85 ms ' + \ + 'entry:default@BuildNativeWithNinja|1 ms entry:default@ProcessResource|2 ms ' + \ + 'entry:default@ProcessLibs|3 ms entry:default@CompileResource|78 ms entry:default@CompileJS|2 ms ' + \ + 'entry:default@CompileArkTS|2 s 591 ms entry:default@PackageHap|924 ms entry:default@SignHap|2 ms ' + \ + 'entry:assembleHap|1 ms total build cost|4 s 886 ms entry:default@BuildNativeWithCmake|1 ms ' + \ + 'entry:default@BuildNativeWithNinja|1 ms entry:default@CompileJS|4 ms ' + \ + 'entry:default@CompileArkTS|2 s 472 ms entry:default@PackageHap|919 ms entry:default@SignHap|2 ms ' + \ + 'entry:assembleHap|1 ms total build cost|4 s 632 ms entry:clean|18 ms :clean|1 ms ' + \ + 'total build cost|1 s 204 ms entry:default@MergeProfile|4 ms entry:default@BuildNativeWithCmake|1 ms' + \ + ' entry:default@GenerateLoaderJson|2 ms entry:default@MakePackInfo|8 ms ' + \ + 'entry:default@ProcessProfile|85 ms entry:default@BuildNativeWithNinja|1 ms' + \ + ' entry:default@ProcessResource|2 ms entry:default@ProcessLibs|3 ms entry:default@CompileResource|78 ms' + \ + ' entry:default@CompileJS|2 ms entry:default@CompileArkTS|2 s 591 ms ' + \ + 'entry:default@PackageHap|924 ms entry:default@SignHap|2 ms entry:assembleHap|1 ms ' + \ + 'total build cost|4 s 886 ms entry:default@BuildNativeWithCmake|1 ms ' + \ + 'entry:default@BuildNativeWithNinja|1 ms entry:default@CompileJS|4 ms ' + \ + 'entry:default@CompileArkTS|2 s 472 ms entry:default@PackageHap|919 ms entry:default@SignHap|2 ms ' + \ + 'entry:assembleHap|1 ms total build cost|4 s 632 ms ' + self.mail_helper = mail_obj + self.config = config_input + + def start(self): + self.init() + self.start_test() + self.write_mail_msg() + os.chdir(self.config.project_path) + + @staticmethod + def append_into_dic(key, value, dic): + if key not in dic: + dic[key] = [] + dic[key].append(value) + + def init(self): + self.developing_test_mode = self.config.developing_test_mode + if self.config.ide == performance_config.IdeType.DevEco: + os.environ['path'] = self.config.node_js_path + ";" + os.environ['path'] + os.chdir(self.config.project_path) + os.environ['path'] = os.path.join(self.config.jbr_path, "bin") + ";" + os.environ['path'] + os.environ['JAVA_HOME'] = self.config.jbr_path + self.config.cmd_prefix = os.path.join(self.config.project_path, self.config.cmd_prefix) + self.config.log_direct = os.path.join(self.config.project_path, self.config.log_direct) + self.config.temp_filename = os.path.join(self.config.log_direct, self.config.temp_filename) + self.config.debug_package_path = os.path.join(self.config.project_path, self.config.debug_package_path) + self.config.release_package_path = os.path.join(self.config.project_path, self.config.release_package_path) + self.config.incremental_code_path = os.path.join(self.config.project_path, self.config.incremental_code_path) + if not os.path.exists(self.config.log_direct): + os.makedirs(self.config.log_direct) + self.config.log_direct = os.path.join(self.config.log_direct, + time.strftime(self.config.log_direct_data_format, + time.localtime())) + if not os.path.exists(self.config.log_direct): + os.makedirs(self.config.log_direct) + self.config.log_direct = os.path.join(self.config.project_path, self.config.log_direct) + if self.developing_test_mode: + self.config.build_times = 3 + + def clean_temp_log(self): + if os.path.exists(self.config.temp_filename): + os.remove(self.config.temp_filename) + + @staticmethod + def add_code(code_path, start_pos, end_pos, code_str, lines): + with open(code_path, 'r+', encoding='UTF-8') as modified_file: + content = modified_file.read() + add_str_end_pos = content.find(end_pos) + if add_str_end_pos == -1: + print('Can not find code : {end_pos} in {code_path}, please check config') + return + add_str_start_pos = content.find(start_pos) + if add_str_start_pos == -1: + if lines == 0: + return + add_str_start_pos = add_str_end_pos + content_add = "" + for i in range(lines, 0, -1): + if "%d" in code_str: + content_add = content_add + code_str % i + else: + content_add = content_add + code_str + content = content[:add_str_start_pos] + content_add + content[add_str_end_pos:] + modified_file.seek(0) + modified_file.write(content) + modified_file.truncate() + + def add_incremental_code(self, lines): + PerformanceBuild.add_code(self.config.incremental_code_path, + self.config.incremental_code_start_pos, + self.config.incremental_code_end_pos, + self.config.incremental_code_str, + lines) + + def revert_incremental_code(self): + self.add_incremental_code(0) + + def reset(self): + self.first_line_in_avg_excel = "" + self.time_avg_dic = {} + self.all_time_dic = {} + self.size_avg_dic = {} + self.all_size_dic = {} + self.built_times = 0 + self.clean_temp_log() + self.revert_incremental_code() + + def clean_project(self): + if not self.developing_test_mode: + print(self.config.cmd_prefix + " clean --no-daemon") + subprocess.Popen(self.config.cmd_prefix + " clean --no-daemon").wait() + + def get_bytecode_size(self, is_debug): + if self.developing_test_mode: + # test data for size + PerformanceBuild.append_into_dic("ets/mudules.abc rawSize", 44444, self.all_size_dic) + PerformanceBuild.append_into_dic("ets/mudules.abc Compress_size", 33333, self.all_size_dic) + PerformanceBuild.append_into_dic("ets/mudules2.abc rawSize", 44444, self.all_size_dic) + PerformanceBuild.append_into_dic("ets/mudules2.abc Compress_size", 33333, self.all_size_dic) + return + package_path = self.config.debug_package_path if is_debug else self.config.release_package_path + package = zipfile.ZipFile(package_path) + extension_name = ".abc" if self.config.ide == performance_config.IdeType.DevEco else ".dex" + for info in package.infolist(): + if info.filename.endswith(extension_name): + name_str1 = info.filename + " rawSize" + name_str2 = info.filename + " compress_size" + PerformanceBuild.append_into_dic(name_str1, info.file_size, self.all_size_dic) + PerformanceBuild.append_into_dic(name_str2, info.compress_size, self.all_size_dic) + + def start_build(self, is_debug): + if self.developing_test_mode: + return + cmd_suffix = self.config.cmd_debug_suffix if is_debug else self.config.cmd_release_suffix + print(self.config.cmd_prefix + cmd_suffix) + subprocess.Popen(self.config.cmd_prefix + cmd_suffix).wait() + + def get_millisecond(self, time_string): + if self.config.ide != performance_config.IdeType.DevEco and not self.developing_test_mode: + return int(time_string) + else: + cost_time = 0 + res = time_string.split(" min ") + target_str = "" + if len(res) > 1: + cost_time = int(res[0]) * 60000 + target_str = res[1] + else: + target_str = res[0] + res = target_str.split(" s ") + if len(res) > 1: + cost_time = cost_time + int(res[0]) * 1000 + target_str = res[1] + else: + target_str = res[0] + + res = target_str.split(" ms") + if len(res) > 1: + cost_time = cost_time + int(res[0]) + return cost_time + + def cal_incremental_avg_time(self): + self.first_line_in_avg_excel = self.first_line_in_avg_excel + "\n" + content = "" + if self.developing_test_mode: + content = self.developing_test_data.split("\t") + else: + src = open(self.config.temp_filename, "r", encoding='UTF-8') + content = src.read().split("\t") + src.close() + clean_filter = False + for task_build_time_str in content: + if len(task_build_time_str) == 0: + continue + pair = task_build_time_str.split("|") + key_str = pair[0] + if clean_filter: + if "total" in key_str: + clean_filter = False + continue + if "clean" in key_str: + clean_filter = True + continue + cost_time = self.get_millisecond(pair[1]) + PerformanceBuild.append_into_dic(key_str, cost_time, self.all_time_dic) + for key in self.all_time_dic: + task_count = len(self.all_time_dic[key]) + has_task = True + if task_count != 2 * self.config.build_times: + if task_count == self.config.build_times: + has_task = False + else: + continue + # average of first build + sum_build_time = 0 + for i in range(0, self.config.build_times): + index = i * 2 + if not has_task: + self.all_time_dic[key].insert(index + 1, 0) + sum_build_time = sum_build_time + self.all_time_dic[key][index] + cost = "%.2f s" % (sum_build_time / self.config.build_times / 1000) + PerformanceBuild.append_into_dic(key, cost, self.time_avg_dic) + # average of incremental build + sum_build_time = 0 + for i in range(1, len(self.all_time_dic[key]), 2): + sum_build_time = sum_build_time + self.all_time_dic[key][i] + cost = "%.2f s" % (sum_build_time / self.config.build_times / 1000) + PerformanceBuild.append_into_dic(key, cost, self.time_avg_dic) + + def cal_incremental_avg_size(self): + total_raw_size = [] + total_compressed_size = [] + for i in range(0, self.config.build_times * 2): + total_raw_size.append(0) + total_compressed_size.append(0) + for key in self.all_size_dic: + if "raw" in key: + total_raw_size[i] += self.all_size_dic[key][i] + else: + total_compressed_size[i] += self.all_size_dic[key][i] + self.all_size_dic["total_raw_size"] = total_raw_size + self.all_size_dic["total_compressed_size"] = total_compressed_size + for key in self.all_size_dic: + # sizes should be the same, just check + is_size_the_same = True + full_first_size = self.all_size_dic[key][0] + for i in range(0, len(self.all_size_dic[key]), 2): + if full_first_size != self.all_size_dic[key][i]: + is_size_the_same = False + break + is_size_the_same = is_size_the_same and full_first_size != -1 + full_avg_size = f"{full_first_size} Byte" if is_size_the_same else "size is not the same" + PerformanceBuild.append_into_dic(key, full_avg_size, self.size_avg_dic) + + is_size_the_same = True + incremental_first_size = self.all_size_dic[key][1] + for i in range(1, len(self.all_size_dic[key]), 2): + if incremental_first_size != self.all_size_dic[key][i]: + is_size_the_same = False + break + is_size_the_same = is_size_the_same and incremental_first_size != -1 + incremental_avg_size = f"{incremental_first_size} Byte" if is_size_the_same else "size is not the same" + PerformanceBuild.append_into_dic(key, incremental_avg_size, self.size_avg_dic) + + def cal_incremental_avg(self): + self.cal_incremental_avg_time() + self.cal_incremental_avg_size() + self.clean_temp_log() + + @staticmethod + def add_row(context): + return rf'{context}' + + @staticmethod + def add_td(context): + return rf'{context}' + + @staticmethod + def add_th(context): + return rf'{context}' + + @staticmethod + def test_type_title(context): + return rf'{context}' + + @staticmethod + def app_title(context): + return rf'{context}' + + def write_mail_files(self, file_path, first_line, dic, mail_table_title="", is_debug=""): + msg = PerformanceBuild.test_type_title(mail_table_title) + if first_line: + first_row = "" + first_line_res = first_line.replace("\n", "").split("\t") + for i in first_line_res: + first_row += PerformanceBuild.add_th(i) + rows = PerformanceBuild.add_row(first_row) + + for key in dic: + content_row = PerformanceBuild.add_th(key) + if "total" in key: + for v in dic[key]: + content_row += PerformanceBuild.add_td(v) + rows += PerformanceBuild.add_row(content_row) + if is_debug != '': + full_time = dic[key][0] + full_time = float(full_time[:len(full_time) - 2]) + incremental_time = dic[key][1] + incremental_time = float(incremental_time[:len(incremental_time) - 2]) + self.mail_helper.add_pic_data(is_debug, [full_time, incremental_time]) + msg += rows + return msg + + def write_from_dic(self, file_path, first_line, dic, mail_table_title="", is_debug=""): + content_list = [] + if first_line: + content_list.append(first_line) + for key in dic: + content_list.append(key) + for v in dic[key]: + content_list.append("\t") + content_list.append(str(v)) + content_list.append("\n") + excel_path = os.path.join(self.config.log_direct, os.path.basename(file_path)) + content = "".join(content_list) + with os.fdopen(os.open(excel_path, + os.O_WRONLY | os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG), 'w') as excel: + excel.write(content) + self.mail_helper.add_logs_file(file_path, content.encode()) + + if mail_table_title: + return self.write_mail_files(file_path, first_line, dic, mail_table_title, is_debug) + else: + return "" + + def generate_full_and_incremental_results(self, is_debug): + path_prefix = self.config.output_split.join( + (self.config.ide_filename[self.config.ide - 1], + self.config.debug_or_release[0 if is_debug else 1], + self.config.build_type_of_log[0]) + ) + prj_name = os.path.basename(self.config.project_path) + temp_mail_msg = "" + # sizeAll + file_path = self.config.output_split.join((path_prefix, self.config.log_filename[0])) + file_path = os.path.join(prj_name, file_path) + self.write_from_dic(file_path, None, self.all_size_dic) + # sizeAvg and mailmsg + file_path = self.config.output_split.join((path_prefix, self.config.log_filename[1])) + file_path = os.path.join(prj_name, file_path) + temp_mail_msg += self.write_from_dic(file_path, self.first_line_in_avg_excel, self.size_avg_dic, "Size") + # timeAll + file_path = self.config.output_split.join((path_prefix, self.config.log_filename[2])) + file_path = os.path.join(prj_name, file_path) + self.write_from_dic(file_path, None, self.all_time_dic) + # timeAvg and mailmsg + file_path = self.config.output_split.join((path_prefix, self.config.log_filename[3])) + file_path = os.path.join(prj_name, file_path) + temp_mail_msg += self.write_from_dic(file_path, self.first_line_in_avg_excel, + self.time_avg_dic, "Time", is_debug) + # mail files + if self.config.send_mail: + temp_mail_msg = '' + \ + PerformanceBuild.app_title(prj_name + (' Debug' if is_debug else ' Release')) + \ + temp_mail_msg + '
' + self.mail_msg += temp_mail_msg + + def full_and_incremental_build(self, is_debug): + self.reset() + self.first_line_in_avg_excel = self.first_line_in_avg_excel + "\tfirst build\tincremental build" + for i in range(self.config.build_times): + self.clean_project() + self.start_build(is_debug) + self.get_bytecode_size(is_debug) + self.add_incremental_code(1) + self.start_build(is_debug) + self.get_bytecode_size(is_debug) + self.revert_incremental_code() + self.cal_incremental_avg() + self.generate_full_and_incremental_results(is_debug) + + def start_test(self): + self.full_and_incremental_build(True) + self.full_and_incremental_build(False) + + def write_mail_msg(self): + if self.config.send_mail: + self.mail_helper.add_msg(self.mail_msg) + + +def run(config_input, mail_obj): + start_time = time.time() + PerformanceBuild(config_input, mail_obj).start() + print("Test [%s] finished at: %s\n"\ + "total cost: %ds" + % (os.path.basename(config_input.project_path), + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), + time.time() - start_time)) \ No newline at end of file diff --git a/test/scripts/performance_test/performance_config.py b/test/scripts/performance_test/performance_config.py new file mode 100644 index 0000000000000000000000000000000000000000..3135e38839f4316b41342328bab15d7c8aa37d51 --- /dev/null +++ b/test/scripts/performance_test/performance_config.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +class IdeType(): + AS = 1 + DevEco = 2 + + +class Config(): + log_direct = "buildTestData" + log_direct_data_format = "%Y-%m-%d-%H-%M-%S" + temp_filename = "temp.txt" + send_mail = True + + def __init__(self): + # Default config settings for all projects, if it's not what you need, config them in application_configs + self.cmd_prefix = r"hvigorw.bat" + + self.output_split = "_" + self.ide_filename = ["AS", "DevEco"] + self.debug_or_release = ["Debug", "Release"] + self.build_type_of_log = ["full_then_incremental", "add_code_and_ui"] + self.log_filename = ["sizeAll.csv", "sizeAvg.csv", + "timeAll.csv", "timeAvg.csv"] + self.ide = IdeType.DevEco + self.incremental_code_str = "let index = 5 + 6\n" + self.incremental_code_start_pos = "let index = 5 + 6\n" + self.incremental_code_end_pos = 'this.num = num' + self.cmd_debug_suffix = r' --mode module -p product=default assembleHap --no-daemon' + self.cmd_release_suffix = r' --mode project -p product=default assembleApp --no-daemon' + self.debug_package_path = r'entry/build/default/outputs/default/entry-default-signed.hap' + self.release_package_path = r'entry/build/default/outputs/default/app/entry-default.hap' + self.incremental_code_path = r'entry/src/main/ets/pages/Index.ets' + + # build serveral times then calculate the average value + self.build_times = 3 + # Do not build the project，use the test data if you need to debug the scripts + self.developing_test_mode = False + # set your node_js path, it should be the same to the setting in your IDE + self.node_js_path = r"%s/nodejs" % os.environ['USERPROFILE'] + # Must set according environment + self.jbr_path = r'xxx/DevEco Studio/jbr' + + # If Default config is not what you need, you can set here! + application_configs = dict( + [ + ( + "FTB", dict + ( + project_path=r"D:/FTB", + ) + ), + ( + "FDY", dict + ( + project_path=r"D:/FDY", + ) + ), + ( + "FWX", dict + ( + project_path=r"D:/FWX", + ) + ), + ( + "HelloWorld", dict + ( + # The following params must be set according you environment + project_path=r"D:/HelloWorld", + + # The following params is not neccessary to be modified + debug_package_path=r'entry/build/default/outputs/default/entry-default-unsigned.hap', + release_package_path=r'entry/build/default/outputs/default/app/entry-default.hap', + incremental_code_path=r'entry/src/main/ets/pages/Index.ets', + incremental_code_end_pos='build() {', + incremental_code_str="a: number=5 + 6\n", + incremental_code_start_pos="a: number=5 + 6\n", + ) + ) + ] + ) + + +def get_config(index): + config = Config() + res = config.application_configs.get(index) + if not res: + print("No key in config, please check: " + index) + return res + for k in res: + setattr(config, k, res[k]) + return config + + +def get_html_prefix(): + return '' + \ + '
Daily Performance Test
' + + +def get_html_suffix(): + return '



' + \ + '



' + + +class BuildMode(): + DEBUG = 0 + RELEASE = 1 + + +class BuildType(): + FULL = 0 + INCREMENTAL = 1 + + +class MailPicConfig(): + mail_data_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'mail_data', + ) + + html_file_path = os.path.join( + mail_data_path, + "email_msg.html" + ) + + attach_path = os.path.join( + mail_data_path, + "performance_logs.zip" + ) + + # Count of days which will be add into the email picture + mail_pic_table_name = { + BuildMode.DEBUG: { + BuildType.FULL: os.path.join(mail_data_path, 'debug_full_time.csv'), + BuildType.INCREMENTAL: os.path.join(mail_data_path, 'debug_incremental_time.csv') + }, + BuildMode.RELEASE:{ + BuildType.FULL: os.path.join(mail_data_path, 'release_full_time.csv'), + BuildType.INCREMENTAL: os.path.join(mail_data_path, 'release_incremental_time.csv') + } + } + + mail_pic_name = { + BuildMode.DEBUG: { + BuildType.FULL:os.path.join(mail_data_path, 'debug_full_time.jpg'), + BuildType.INCREMENTAL: os.path.join(mail_data_path, 'debug_incremental_time.jpg') + }, + BuildMode.RELEASE:{ + BuildType.FULL: os.path.join(mail_data_path, 'release_full_time.jpg'), + BuildType.INCREMENTAL: os.path.join(mail_data_path, 'release_incremental_time.jpg') + } + } + + mail_pic_table_lable = { + BuildMode.DEBUG: { + BuildType.FULL: 'Debug Full Build', + BuildType.INCREMENTAL: 'Debug Incremental Build' + }, + BuildMode.RELEASE:{ + BuildType.FULL: 'Release Full Build', + BuildType.INCREMENTAL: 'Release Incremental Time' + } + } \ No newline at end of file diff --git a/test/scripts/performance_test/performance_entry.py b/test/scripts/performance_test/performance_entry.py new file mode 100644 index 0000000000000000000000000000000000000000..be8c89f3c79c995e53ccc087fbd9af337877dab4 --- /dev/null +++ b/test/scripts/performance_test/performance_entry.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import stat +import time +import zipfile + +import pandas as pd +import numpy as np + + +import performance_build +import performance_config + + +class MailHelper(): + def __init__(self): + self.pic_table = {} + self.head_line = [] + self.current_add_prj = '' + self.mail_msg = performance_config.get_html_prefix() + self.logs_file = {} + if os.path.exists(performance_config.MailPicConfig.html_file_path): + os.remove(performance_config.MailPicConfig.html_file_path) + if os.path.exists(performance_config.MailPicConfig.attach_path): + os.remove(performance_config.MailPicConfig.attach_path) + if not os.path.exists(performance_config.MailPicConfig.mail_data_path): + os.mkdir(performance_config.MailPicConfig.mail_data_path) + for build_mode in range(2): + for build_type in range(2): + pic_path = MailHelper.find_in_double_map(build_mode, + build_type, + performance_config.MailPicConfig.mail_pic_name, + "mail_pic_name") + if not pic_path: + return + if os.path.exists(pic_path): + os.remove(pic_path) + + def create(self, prj_name): + self.pic_table[prj_name] = { + performance_config.BuildMode.DEBUG:[np.nan, np.nan], + performance_config.BuildMode.RELEASE:[np.nan, np.nan] + } + self.head_line.append(prj_name) + self.current_add_prj = prj_name + self.time_index = [time.strftime("%Y/%m/%d", time.localtime())] + + @staticmethod + def find_in_double_map(key_1, key_2, double_dic, error_table_name): + it_1 = double_dic.get(key_1) + if it_1: + it_2 = it_1.get(key_2) + if not it_2: + print(f"Can not find key_2: {key_2} in {error_table_name}, please check") + return it_2 + else: + print(f"Can not find key_1: {key_1} in {error_table_name}, please check") + return it_1 + + def add_msg(self, msg): + self.mail_msg += msg + + def add_logs_file(self, filename, buffer): + self.logs_file[filename] = buffer + + def add_pic_data(self, is_debug, time_list): + build_type = performance_config.BuildMode.DEBUG if is_debug else performance_config.BuildMode.RELEASE + prj_table = self.pic_table.get(self.current_add_prj) + if prj_table: + prj_table.update({build_type: time_list}) + else: + print(f"Can not find {self.current_add_prj} in pic_table, please check") + + def create_msg_file(self): + with os.fdopen( + os.open(performance_config.MailPicConfig.html_file_path, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG), + 'w' + ) as msg_file: + msg_file.write(self.mail_msg + performance_config.get_html_suffix()) + + def create_logs_file(self): + mail_zip = zipfile.ZipFile(performance_config.MailPicConfig.attach_path, 'w') + for file in self.logs_file: + mail_zip.writestr(file, self.logs_file.get(file)) + mail_zip.close() + + def create_pic(self): + for build_mode in range(2): + for build_type in range(2): + title_name = MailHelper.find_in_double_map(build_mode, + build_type, + performance_config.MailPicConfig.mail_pic_table_lable, + "mail_pic_table_lable") + if not title_name: + continue + pic_name = MailHelper.find_in_double_map(build_mode, + build_type, + performance_config.MailPicConfig.mail_pic_name, + "mail_pic_name") + if not pic_name: + continue + df = None + csv_filename = MailHelper.find_in_double_map(build_mode, + build_type, + performance_config.MailPicConfig.mail_pic_table_name, + "pic_table_name") + if not csv_filename: + continue + if os.path.exists(csv_filename): + df = pd.read_csv(csv_filename, index_col=0) + else: + df = pd.DataFrame(columns=self.head_line) + dic = {} + for prj_name in self.pic_table: + dic[prj_name] = [self.pic_table[prj_name][build_mode][build_type]] + df_inserted = pd.DataFrame(dic, index=self.time_index) + df = df._append(df_inserted) + if len(df) > 5: + df = df[1:len(df)] + df.to_csv(csv_filename) + df.plot( + linestyle='-', + linewidth=2, + marker='o', + markersize=6, + markeredgecolor='black', + title=title_name, + ylabel='build time (s)', + grid=True + ).get_figure().savefig(pic_name) + + def create_mail_files(self): + self.create_msg_file() + self.create_pic() + self.create_logs_file() + + +class PerformanceEntry(): + def __init__(self) -> None: + self.mail_helper = MailHelper() + + def run(self): + self.init() + # You can control which project you need to test here, the first param is the key in performance_config.py + self.start_test("FDY") + self.start_test("FTB") + self.start_test("FWX") + os.chdir(os.path.dirname(__file__)) + self.create_mail_files() + + def start_test(self, index): + config = performance_config.get_config(index) + if not config: + return + self.mail_helper.create(os.path.basename(config.project_path)) + performance_build.run(config, self.mail_helper) + + def create_mail_files(self): + if performance_config.Config.send_mail: + self.mail_helper.create_mail_files() + + def init(self): + self.mail_helper = MailHelper() + + +if __name__ == '__main__': + strat_time = time.time() + PerformanceEntry().run() + print("All test finished at %s\ntotal cost: %ds" + % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), time.time() - strat_time)) \ No newline at end of file diff --git a/test/scripts/send_email.py b/test/scripts/send_email.py index 44cfdd000f8e43d98d45dd3dce49efee63d74b86..27243c19e4d63eda829a0cf6363191890fe03c65 100755 --- a/test/scripts/send_email.py +++ b/test/scripts/send_email.py @@ -17,9 +17,10 @@ import os import smtplib -import socket -from email.message import EmailMessage +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText import yaml @@ -38,13 +39,25 @@ def add_content(content, file_name, test_part): def add_attachment(msg, file_list): for file in file_list: if os.path.exists(file): - with open(file, 'r', encoding='utf-8') as f: - msg.add_attachment(f.read(), 'html', filename=os.path.basename(file)) + with open(file, 'rb') as f: + attachment = MIMEText(f.read(), 'base64', 'utf-8') + attachment['Content-Disposition'] = f'attachment; filename="{os.path.basename(file)}"' + msg.attach(attachment) + + +def add_image(msg, img_dic): + for path in img_dic: + if os.path.exists(path): + with open(path, 'rb') as f: + img = MIMEImage(f.read()) + img.add_header('Content-ID', img_dic[path]) + msg.attach(img) def send_email(): yl = open(r".\email_config.yaml", 'r') data = yaml.safe_load(yl.read()) + user_name = data["user_name"] sender = data["sender_email_address"] auth_code = data["auth_code"] receiver = data["receiver_list"] @@ -54,9 +67,10 @@ def send_email(): sdk_test = data["sdk_report_file"] perf_test = data["perf_report_file"] attachment_files = data["attatchment_files"] + image_files = data["image_files"] yl.close() - msg = EmailMessage() + msg = MIMEMultipart() msg['From'] = sender msg['To'] = ", ".join(receiver) msg['Subject'] = "Arkcompiler Test" @@ -68,12 +82,11 @@ def send_email(): html = add_content(html, sdk_test, "sdk_test") html += dividing_line html = add_content(html, perf_test, "perf_test") - msg.add_related(html, "html") - + msg.attach(MIMEText(html, 'html', 'utf-8')) add_attachment(msg, attachment_files) - + add_image(msg, image_files) smtp = smtplib.SMTP(smtp_server, smtp_port) - smtp.login(sender, auth_code) + smtp.login(user_name, auth_code) smtp.sendmail(sender, receiver, msg.as_string()) smtp.quit() diff --git a/test/scripts/timer.py b/test/scripts/timer.py index f27fca80a3ca79c0324a821bc85453ffaa1f3532..82a593883f6f21784667bd33c997d58c90f3a5f3 100755 --- a/test/scripts/timer.py +++ b/test/scripts/timer.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# Copyright (c) 2023 Huawei Device Co., Ltd. +# Copyright (c) 0123 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -28,11 +28,14 @@ def job(cmd): if __name__ == "__main__": os.chdir(os.path.dirname(os.path.realpath(__file__))) - #do preparations - schedule.every().day.at("20:00").do(job, cmd=r'.\auto_xts_test\run.bat').tag('daily_xts_task') - schedule.every().day.at("20:00").do(job, cmd=r'python .\sdk_test\entry.py').tag('daily_sdk_task') - #do perf_test - schedule.every().day.at("20:00").do(job, cmd=r'python .\send_email.py').tag("send_email") + # do preparations + schedule.every().day.at("01:00").do(job, cmd=r'./auto_xts_test/run.bat').tag('daily_xts_task') + # do sdk_test + schedule.every().day.at("01:00").do(job, cmd=r'python ./sdk_test/entry.py').tag('daily_sdk_task') + # do perf_test + schedule.every().day.at("01:00").do(job, cmd=r'python ./performance_test/performance_entry.py') + # send mail + schedule.every().day.at("01:00").do(job, cmd=r'python ./send_email.py').tag("send_email") schedule.run_all() while True: schedule.run_pending()