From c788be8b11c9eea5585e413d96108dd154526e72 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 15:04:25 +0800 Subject: [PATCH 01/10] change table format --- .../api_accuracy_checker/compare/compare.py | 69 ++++++++++++------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 957de66ac..c31bccc2b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -40,10 +40,11 @@ class Comparator: def print_pretest_result(self): self.get_statistics_from_result_csv() - if self.test_result_cnt.get("total_num") != 0: - passing_rate = str(self.test_result_cnt.get("success_num") / self.test_result_cnt.get("total_num")) + total_tests = self.test_result_cnt.get("total_num", 0) + if total_tests != 0: + passing_rate = "{:.2%}".format(self.test_result_cnt.get("success_num", 0) / total_tests) else: - passing_rate = "0" + passing_rate = "0%" console = Console() table_total = Table( @@ -51,9 +52,9 @@ class Comparator: ) table_total.add_column("Result") table_total.add_column("Statistics") - table_total.add_row("[green]Pass[/green]", str(self.test_result_cnt.get("success_num"))) - table_total.add_row("[red]Fail[/red]", str(self.test_result_cnt.get("forward_and_backward_fail_num") + - self.test_result_cnt.get("forward_or_backward_fail_num"))) + table_total.add_row("Pass", str(self.test_result_cnt.get("success_num", 0))) + table_total.add_row("Warning", str(self.test_result_cnt.get("warning_num", 0))) + table_total.add_row("Error", str(self.test_result_cnt.get("error_num", 0))) table_total.add_row("Passing Rate", passing_rate) table_detail = Table( @@ -61,16 +62,20 @@ class Comparator: ) table_detail.add_column("Result") table_detail.add_column("Statistics") - table_detail.add_row("Only Forward Fail", str(self.test_result_cnt.get("forward_fail_num"))) - table_detail.add_row("Only Backward Fail", str(self.test_result_cnt.get("backward_fail_num"))) - table_detail.add_row( - "Both Forward & Backward Fail", str(self.test_result_cnt.get("forward_and_backward_fail_num"))) + table_detail.add_row("Forward Error", str(self.test_result_cnt.get("forward_fail_num", 0))) + table_detail.add_row("Backward Error", str(self.test_result_cnt.get("backward_fail_num", 0))) + table_detail.add_row("Both Forward & Backward Error", str(self.test_result_cnt.get("forward_and_backward_fail_num", 0))) console.print(table_total) console.print(table_detail) def get_statistics_from_result_csv(self): - checklist = [CompareConst.TRUE, CompareConst.FALSE, CompareConst.NA, CompareConst.SKIP] + checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.NA, CompareConst.SKIP] + self.test_result_cnt = { + "success_num": 0, "warning_num": 0, "error_num": 0, + "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, + "total_num": 0 + } with FileOpen(self.save_path, 'r') as file: reader = csv.reader(file) result_csv_rows = [row for row in reader] @@ -80,23 +85,25 @@ class Comparator: raise ValueError("The number of columns in %s is incorrect" % result_csv_name) if not all(item[i] and item[i].upper() in checklist for i in (1, 2)): raise ValueError( - "The value in the 2nd or 3rd column of %s is wrong, it must be TRUE, FALSE, SKIP or N/A" + "The value in the 2nd or 3rd column of %s is wrong, it must be TRUE, FALSE, SKIP, WARNING, or N/A" % result_csv_name) column1 = item[1].upper() column2 = item[2].upper() if column1 == CompareConst.SKIP: continue self.test_result_cnt["total_num"] += 1 - if column1 == CompareConst.TRUE and column2 in [CompareConst.TRUE, 'N/A']: + if column1 == CompareConst.TRUE and column2 in [CompareConst.TRUE, CompareConst.NA]: self.test_result_cnt['success_num'] += 1 elif column1 == CompareConst.FALSE and column2 == CompareConst.FALSE: self.test_result_cnt['forward_and_backward_fail_num'] += 1 elif column1 == CompareConst.FALSE: self.test_result_cnt['forward_fail_num'] += 1 - self.test_result_cnt['forward_or_backward_fail_num'] += 1 - else: + elif column2 == CompareConst.FALSE: self.test_result_cnt['backward_fail_num'] += 1 - self.test_result_cnt['forward_or_backward_fail_num'] += 1 + if column1 == CompareConst.WARNING or column2 == CompareConst.WARNING: + self.test_result_cnt['warning_num'] += 1 + if column1 == CompareConst.ERROR or column2 == CompareConst.ERROR: + self.test_result_cnt['error_num'] += 1 def write_csv_title(self): summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, @@ -156,18 +163,26 @@ class Comparator: else: is_bwd_success, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) else: - is_bwd_success, bwd_compare_alg_results = True, None + is_bwd_success, bwd_compare_alg_results = CompareConst.PASS, None if is_bwd_success and bwd_compare_alg_results is None: self.record_results(api_name, is_fwd_success, CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) else: self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results) + if is_fwd_success == CompareConst.PASS: + is_fwd_success = True + else: + is_fwd_success = False + if is_bwd_success == CompareConst.PASS: + is_bwd_success = True + else: + is_bwd_success = False return is_fwd_success, is_bwd_success def _compare_core_wrapper(self, bench_output, device_output): detailed_result_total = [] - test_final_success = True + test_final_success = CompareConst.PASS if isinstance(bench_output, (list, tuple)): status, compare_result, message = [], [], [] if len(bench_output) != len(device_output): @@ -183,13 +198,17 @@ class Comparator: status, compare_result, message = self._compare_core(bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) - if status in [CompareConst.ERROR, CompareConst.WARNING]: - test_final_success = False + if status == CompareConst.ERROR: + test_final_success = CompareConst.ERROR + elif status == CompareConst.WARNING: + test_final_success = CompareConst.WARNING else: for item, item_status in enumerate(status): detailed_result_total.append(compare_result[item].to_column_value(item_status, message[item])) - if item_status in [CompareConst.ERROR, CompareConst.WARNING]: - test_final_success = False + if item_status == CompareConst.ERROR: + test_final_success = CompareConst.ERROR + elif item_status == CompareConst.WARNING: + test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total def _compare_core(self, bench_output, device_output): @@ -257,11 +276,11 @@ class Comparator: tensor_num = bench_output.numel() if tensor_num >= 100: if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: - return True, 1 + return CompareConst.PASS, 1 else: - return False, 0 + return CompareConst.ERROR, 0 else: - return True, 1 + return CompareConst.PASS, 1 @staticmethod def _compare_builtin_type(bench_output, device_output, compare_column): -- Gitee From 463603b54788a5da9ff83a589d7aec6796aa9002 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 15:18:38 +0800 Subject: [PATCH 02/10] update --- .../api_accuracy_checker/compare/compare.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index c31bccc2b..53b45206b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -52,9 +52,9 @@ class Comparator: ) table_total.add_column("Result") table_total.add_column("Statistics") - table_total.add_row("Pass", str(self.test_result_cnt.get("success_num", 0))) - table_total.add_row("Warning", str(self.test_result_cnt.get("warning_num", 0))) - table_total.add_row("Error", str(self.test_result_cnt.get("error_num", 0))) + table_total.add_row("[green]Pass[/green]", str(self.test_result_cnt.get("success_num", 0))) + table_total.add_row("[yellow]Warning[/yellow]", str(self.test_result_cnt.get("warning_num", 0))) + table_total.add_row("[red]Error[/red]", str(self.test_result_cnt.get("error_num", 0))) table_total.add_row("Passing Rate", passing_rate) table_detail = Table( @@ -83,27 +83,28 @@ class Comparator: for item in result_csv_rows[1:]: if not isinstance(item, list) or len(item) < 3: raise ValueError("The number of columns in %s is incorrect" % result_csv_name) - if not all(item[i] and item[i].upper() in checklist for i in (1, 2)): + if not all(item[i] and item[i] in checklist for i in (1, 2)): raise ValueError( "The value in the 2nd or 3rd column of %s is wrong, it must be TRUE, FALSE, SKIP, WARNING, or N/A" % result_csv_name) - column1 = item[1].upper() - column2 = item[2].upper() + column1 = item[1] + column2 = item[2] if column1 == CompareConst.SKIP: continue self.test_result_cnt["total_num"] += 1 - if column1 == CompareConst.TRUE and column2 in [CompareConst.TRUE, CompareConst.NA]: + if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.NA]: self.test_result_cnt['success_num'] += 1 - elif column1 == CompareConst.FALSE and column2 == CompareConst.FALSE: + elif column1 == CompareConst.ERROR and column2 == CompareConst.ERROR: self.test_result_cnt['forward_and_backward_fail_num'] += 1 - elif column1 == CompareConst.FALSE: + self.test_result_cnt['error_num'] += 1 + elif column1 == CompareConst.ERROR: self.test_result_cnt['forward_fail_num'] += 1 - elif column2 == CompareConst.FALSE: + self.test_result_cnt['error_num'] += 1 + elif column2 == CompareConst.ERROR: self.test_result_cnt['backward_fail_num'] += 1 - if column1 == CompareConst.WARNING or column2 == CompareConst.WARNING: - self.test_result_cnt['warning_num'] += 1 - if column1 == CompareConst.ERROR or column2 == CompareConst.ERROR: self.test_result_cnt['error_num'] += 1 + elif column1 == CompareConst.WARNING or column2 == CompareConst.WARNING: + self.test_result_cnt['warning_num'] += 1 def write_csv_title(self): summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, -- Gitee From 118f848b027cea40187a2a3fcf49cbb86b63520f Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 15:32:10 +0800 Subject: [PATCH 03/10] bug fix --- .../accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py index 085f2f931..038ed1e9f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py @@ -157,7 +157,7 @@ def prepare_config(args): print_info_log(f"UT task result will be saved in {result_csv_path}") print_info_log(f"UT task details will be saved in {details_csv_path}") else: - result_csv_path = get_validated_result_csv_path(args.result_csv_path) + result_csv_path = get_validated_result_csv_path(args.result_csv_path, 'result') details_csv_path = get_validated_details_csv_path(result_csv_path) print_info_log(f"UT task result will be saved in {result_csv_path}") print_info_log(f"UT task details will be saved in {details_csv_path}") -- Gitee From 64207a879348f4c115a595908960402168abbe23 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 16:49:50 +0800 Subject: [PATCH 04/10] clean code --- .../api_accuracy_checker/compare/compare.py | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 53b45206b..c11eb3c05 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -155,30 +155,24 @@ class Comparator: def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): if "dropout" in api_name: - is_fwd_success, fwd_compare_alg_results = self._compare_dropout(bench_output, device_output) + fwd_success_status, fwd_compare_alg_results = self._compare_dropout(bench_output, device_output) else: - is_fwd_success, fwd_compare_alg_results = self._compare_core_wrapper(bench_output, device_output) + fwd_success_status, fwd_compare_alg_results = self._compare_core_wrapper(bench_output, device_output) if bench_grad and npu_grad: if "dropout" in api_name: - is_bwd_success, bwd_compare_alg_results = self._compare_dropout(bench_grad[0], npu_grad[0]) + bwd_success_status, bwd_compare_alg_results = self._compare_dropout(bench_grad[0], npu_grad[0]) else: - is_bwd_success, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) + bwd_success_status, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) else: - is_bwd_success, bwd_compare_alg_results = CompareConst.PASS, None - if is_bwd_success and bwd_compare_alg_results is None: - self.record_results(api_name, is_fwd_success, CompareConst.NA, fwd_compare_alg_results, + bwd_success_status, bwd_compare_alg_results = CompareConst.PASS, None + if bwd_success_status and bwd_compare_alg_results is None: + self.record_results(api_name, fwd_success_status, CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) else: - self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, + self.record_results(api_name, fwd_success_status, bwd_success_status, fwd_compare_alg_results, bwd_compare_alg_results) - if is_fwd_success == CompareConst.PASS: - is_fwd_success = True - else: - is_fwd_success = False - if is_bwd_success == CompareConst.PASS: - is_bwd_success = True - else: - is_bwd_success = False + is_fwd_success = True if fwd_success_status == CompareConst.PASS else False + is_bwd_success = True if bwd_success_status == CompareConst.PASS else False return is_fwd_success, is_bwd_success def _compare_core_wrapper(self, bench_output, device_output): -- Gitee From cebff4d37fa4719fa11f1d977131ce64f4090334 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 16:52:25 +0800 Subject: [PATCH 05/10] update --- .../api_accuracy_checker/test/ut/run_ut/test_multi_run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_multi_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_multi_run_ut.py index 95683cda5..18293a4bc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_multi_run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_multi_run_ut.py @@ -20,7 +20,7 @@ class TestMultiRunUT(unittest.TestCase): def test_split_json_file(self, mock_FileOpen): mock_FileOpen.return_value.__enter__.return_value = mock_open(read_data=self.test_json_content).return_value num_splits = 2 - split_files, total_items = split_json_file(self.test_json_file, num_splits) + split_files, total_items = split_json_file(self.test_json_file, num_splits, False) self.assertEqual(len(split_files), num_splits) self.assertEqual(total_items, len(self.test_data)) -- Gitee From ee89d6ace7b52d5c804feb1c8d0a5b7f02d2ceab Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 17:09:07 +0800 Subject: [PATCH 06/10] cleancode --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index c11eb3c05..a08026bde 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -164,7 +164,7 @@ class Comparator: else: bwd_success_status, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) else: - bwd_success_status, bwd_compare_alg_results = CompareConst.PASS, None + bwd_success_status, bwd_compare_alg_results = CompareConst.PASS, [] if bwd_success_status and bwd_compare_alg_results is None: self.record_results(api_name, fwd_success_status, CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) -- Gitee From 1e9ae442c8af2995f3ddb98118fac2c4aab16d1f Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 17:30:28 +0800 Subject: [PATCH 07/10] cleancode --- .../api_accuracy_checker/compare/compare.py | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index a08026bde..01344b8b3 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -154,26 +154,11 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): - if "dropout" in api_name: - fwd_success_status, fwd_compare_alg_results = self._compare_dropout(bench_output, device_output) - else: - fwd_success_status, fwd_compare_alg_results = self._compare_core_wrapper(bench_output, device_output) - if bench_grad and npu_grad: - if "dropout" in api_name: - bwd_success_status, bwd_compare_alg_results = self._compare_dropout(bench_grad[0], npu_grad[0]) - else: - bwd_success_status, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) - else: - bwd_success_status, bwd_compare_alg_results = CompareConst.PASS, [] - if bwd_success_status and bwd_compare_alg_results is None: - self.record_results(api_name, fwd_success_status, CompareConst.NA, fwd_compare_alg_results, - bwd_compare_alg_results) - else: - self.record_results(api_name, fwd_success_status, bwd_success_status, fwd_compare_alg_results, - bwd_compare_alg_results) - is_fwd_success = True if fwd_success_status == CompareConst.PASS else False - is_bwd_success = True if bwd_success_status == CompareConst.PASS else False - return is_fwd_success, is_bwd_success + compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper + fwd_success_status, fwd_compare_alg_results = compare_func(bench_output, device_output) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(bench_grad, npu_grad) + self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) + return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS def _compare_core_wrapper(self, bench_output, device_output): detailed_result_total = [] -- Gitee From a7465535eb7d45b1a831c517894ab419b6b1c121 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 18:48:20 +0800 Subject: [PATCH 08/10] update --- .../api_accuracy_checker/compare/compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 01344b8b3..85239b3be 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -70,7 +70,7 @@ class Comparator: console.print(table_detail) def get_statistics_from_result_csv(self): - checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.NA, CompareConst.SKIP] + checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.NA, CompareConst.SKIP, "skip"] self.test_result_cnt = { "success_num": 0, "warning_num": 0, "error_num": 0, "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, @@ -85,11 +85,11 @@ class Comparator: raise ValueError("The number of columns in %s is incorrect" % result_csv_name) if not all(item[i] and item[i] in checklist for i in (1, 2)): raise ValueError( - "The value in the 2nd or 3rd column of %s is wrong, it must be TRUE, FALSE, SKIP, WARNING, or N/A" + "The value in the 2nd or 3rd column of %s is wrong, it must be pass, error, warning, skip, or N/A" % result_csv_name) column1 = item[1] column2 = item[2] - if column1 == CompareConst.SKIP: + if column1.upper() == CompareConst.SKIP: continue self.test_result_cnt["total_num"] += 1 if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.NA]: @@ -120,7 +120,7 @@ class Comparator: name = test_result[0] df_row = list(test_result[:3]) - if test_result[1] == "SKIP" or test_result[2] == "SKIP": + if test_result[1].upper() == "SKIP" or test_result[2].upper() == "SKIP": df_row.append(test_result[3]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) -- Gitee From 9bafda85148b72daff4b1e1540fbe57209346358 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 18:57:48 +0800 Subject: [PATCH 09/10] add skip --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 85239b3be..83af0d228 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -56,6 +56,7 @@ class Comparator: table_total.add_row("[yellow]Warning[/yellow]", str(self.test_result_cnt.get("warning_num", 0))) table_total.add_row("[red]Error[/red]", str(self.test_result_cnt.get("error_num", 0))) table_total.add_row("Passing Rate", passing_rate) + table_total.add_row("Skip Tests", str(self.test_result_cnt.get("total_skip_num", 0))) table_detail = Table( show_header=True, title="Detail Statistics", show_lines=True, width=75 @@ -74,7 +75,7 @@ class Comparator: self.test_result_cnt = { "success_num": 0, "warning_num": 0, "error_num": 0, "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, - "total_num": 0 + "total_num": 0, "total_skip_num": 0 } with FileOpen(self.save_path, 'r') as file: reader = csv.reader(file) @@ -90,6 +91,7 @@ class Comparator: column1 = item[1] column2 = item[2] if column1.upper() == CompareConst.SKIP: + self.test_result_cnt["total_skip_num"] += 1 continue self.test_result_cnt["total_num"] += 1 if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.NA]: -- Gitee From e035c0f5d966812d6fd21ee449c28ccd0abae63d Mon Sep 17 00:00:00 2001 From: s30048155 Date: Sat, 2 Mar 2024 19:10:52 +0800 Subject: [PATCH 10/10] update --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 83af0d228..d9c94b784 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -122,7 +122,7 @@ class Comparator: name = test_result[0] df_row = list(test_result[:3]) - if test_result[1].upper() == "SKIP" or test_result[2].upper() == "SKIP": + if test_result[1] == "SKIP" or test_result[2] == "SKIP": df_row.append(test_result[3]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) -- Gitee