From c3ef8b902dcdfd0b52a5f48d106cb825d728cb37 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 22 Sep 2023 14:17:43 +0800 Subject: [PATCH 01/13] shuangbai --- .../api_accuracy_checker/compare/algorithm.py | 44 +++++++++++++++---- .../api_accuracy_checker/compare/compare.py | 34 +++++++++++--- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index dd4ec514c..797f67f32 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -21,7 +21,11 @@ def compare_bool_tensor(cpu_output, npu_output): return CompareConst.NAN, False, "" error_nums = (cpu_output != npu_output).sum() error_rate = float(error_nums / cpu_output.size) - return error_rate, error_rate == 0, "" + if error_rate == 0: + result = 'pass' + else: + result = 'error' + return error_rate, result, "" def get_msg_and_handle_value(b_value, n_value): msg = "" @@ -58,15 +62,31 @@ def get_max_abs_err(b_value, n_value): bool_result = abs_err < 0.001 return abs_err, bool_result, msg +def get_rel_err_ratio_hundredth(b_value, n_value): + ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.01) + if n_value.dtype != np.float16: + msg = f"This indicator is not used to evaluate {n_value.dtype} data" + return ratio, 'pass', msg + if bool_result: + return ratio, 'pass', msg + return ratio, 'error', msg + def get_rel_err_ratio_thousandth(b_value, n_value): - return get_rel_err_ratio(b_value, n_value, 0.001) + ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.001) + if bool_result: + return ratio, 'pass', msg + if n_value.dtype == np.float16: + return ratio, 'warning', msg + return ratio, 'error', msg def get_rel_err_ratio_ten_thousandth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.0001) if n_value.dtype == np.float16: msg = f"This indicator is not used to evaluate {n_value.dtype} data" - return ratio, True, msg - return ratio, bool_result, msg + return ratio, 'pass', msg + if bool_result: + return ratio, 'pass', msg + return ratio, 'warning', msg def get_rel_err_ratio(b_value, n_value, thresholding): b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) @@ -121,10 +141,10 @@ def compare_uint8_data(b_value, n_value): def compare_builtin_type(bench_out, npu_out): if not isinstance(bench_out, (bool, int, float, str)): - return CompareConst.NA, True, "" + return CompareConst.NA, 'pass', "" if bench_out != npu_out: - return CompareConst.NAN, False, "" - return True, True, "" + return CompareConst.NAN, 'error', "" + return True, 'pass', "" def flatten_compare_result(result): flatten_result = [] @@ -147,7 +167,15 @@ def compare_core(bench_out, npu_out, alg): for b_out_i, n_out_i in zip(bench_out, npu_out): compare_result_i, test_success_i, bench_dtype_i, npu_dtype_i, shape_i = compare_core(b_out_i, n_out_i, alg) compare_result.append(compare_result_i) - test_success = test_success and test_success_i + if isinstance(test_success, bool): + test_success = test_success and test_success_i + else: + if test_success_i == 'error': + test_success = 'error' + elif test_success_i == 'warning' and test_success != 'error': + test_success = 'warning' + elif test_success != 'warning' and test_success != 'error': + test_success = test_success_i bench_dtype.append(bench_dtype_i) npu_dtype.append(npu_dtype_i) shape.append(shape_i) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 8aec6a707..208c9c5f4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -2,7 +2,7 @@ import os from prettytable import PrettyTable from api_accuracy_checker.compare.algorithm import compare_core, cosine_sim, cosine_standard, get_max_rel_err, get_max_abs_err, \ - compare_builtin_type, get_rel_err_ratio_thousandth, get_rel_err_ratio_ten_thousandth + compare_builtin_type, get_rel_err_ratio_hundredth, get_rel_err_ratio_thousandth, get_rel_err_ratio_ten_thousandth from api_accuracy_checker.common.utils import get_json_contents, print_info_log, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst @@ -32,6 +32,7 @@ class Comparator: self.register_compare_algorithm("Cosine Similarity", cosine_sim, cosine_standard) self.register_compare_algorithm("Max Relative Error", get_max_rel_err, None) self.register_compare_algorithm("Max Absolute Error", get_max_abs_err, None) + self.register_compare_algorithm("Hundredth Relative Error Ratio", get_rel_err_ratio_hundredth, None) self.register_compare_algorithm("Thousandth Relative Error Ratio", get_rel_err_ratio_thousandth, None) self.register_compare_algorithm("Ten Thousandth Relative Error Ratio", get_rel_err_ratio_ten_thousandth, None) self.register_compare_algorithm("Default: isEqual", compare_builtin_type, None) @@ -66,6 +67,7 @@ class Comparator: "Cosine Similarity", "Cosine Similarity Message", "Max Rel Error", "Max Rel Err Message", "Max Abs Error", "Max Abs Err Message", + "Relative Error (hundredth)", "Relative Error (dual hundredth) Message", "Relative Error (dual thousandth)", "Relative Error (dual thousandth) Message", "Relative Error (dual ten thousandth)", "Relative Error (dual ten thousandth) Message", "Compare Builtin Type", "Builtin Type Message", @@ -140,29 +142,49 @@ class Comparator: bench_dtype_total = [] npu_dtype_total = [] shape_total = [] - test_success_total = True + test_success_total = [] + max_abs_error_success = False + max_cosine_success = False for name in self.compare_alg.keys(): alg = self.compare_alg[name][0] detailed_result, test_success, bench_dtype, npu_dtype, shape = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype shape_total = shape - if name != "Max Relative Error" and name != "Max Absolute Error": - test_success_total = test_success_total and test_success + if name not in ["Cosine Similarity", "Max Relative Error", "Max Absolute Error"]: + test_success_total.append(test_success) + if name == "Cosine Similarity": + max_cosine_success = test_success + if name == "Max Relative Error": + max_abs_error_success = test_success if detailed_result_total: for i in range(len(detailed_result_total)): detailed_result_total[i] += detailed_result[i] else: detailed_result_total = detailed_result + test_final_result = 'pass' + if not max_cosine_success: + test_final_result = 'error' + elif max_abs_error_success: + test_final_result = 'pass' + else: + if 'error' in test_success_total: + test_final_result = 'error' + elif 'warning' in test_success_total: + test_final_result = 'warning' # dtype加到所有指标的前面, 是否pass放到所有指标的后面 for i in range(len(detailed_result_total)): detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) detailed_result.insert(2, shape_total[i]) - detailed_result.append(str(test_success_total)) + detailed_result.append(str(test_final_result)) detailed_result_total[i] = tuple(detailed_result) - return test_success_total, detailed_result_total + if test_final_result == 'pass': + test_final_success = True + else: + test_final_success = False + return test_final_success, detailed_result_total @staticmethod def _compare_dropout(bench_out, npu_out): -- Gitee From e130ca4196bbfd9e255852455e37d735083eadf6 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 25 Sep 2023 16:45:54 +0800 Subject: [PATCH 02/13] fix --- .../api_accuracy_checker/compare/algorithm.py | 16 +++----- .../api_accuracy_checker/compare/compare.py | 38 ++++++++++--------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 797f67f32..0a0ff2003 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -161,21 +161,13 @@ def compare_core(bench_out, npu_out, alg): if not isinstance(bench_out, type(npu_out)): return [(CompareConst.NAN, "bench and npu output type is different.")], False, CompareConst.NA, CompareConst.NA if isinstance(bench_out, (list, tuple)): - compare_result, test_success, bench_dtype, npu_dtype, shape = [], True, [], [], [] + compare_result, test_success, bench_dtype, npu_dtype, shape = [], [], [], [], [] if len(bench_out) != len(npu_out): return [(CompareConst.NAN, "bench and npu output structure is different")], False, CompareConst.NA, CompareConst.NA for b_out_i, n_out_i in zip(bench_out, npu_out): compare_result_i, test_success_i, bench_dtype_i, npu_dtype_i, shape_i = compare_core(b_out_i, n_out_i, alg) compare_result.append(compare_result_i) - if isinstance(test_success, bool): - test_success = test_success and test_success_i - else: - if test_success_i == 'error': - test_success = 'error' - elif test_success_i == 'warning' and test_success != 'error': - test_success = 'warning' - elif test_success != 'warning' and test_success != 'error': - test_success = test_success_i + test_success.append(test_success_i) bench_dtype.append(bench_dtype_i) npu_dtype.append(npu_dtype_i) shape.append(shape_i) @@ -215,6 +207,10 @@ def compare_core(bench_out, npu_out, alg): compare_result = flatten_compare_result(compare_result) else: compare_result = [(compare_result, msg)] + if isinstance(test_success, list): + test_success = flatten_compare_result(test_success) + else: + test_success = [test_success] if isinstance(bench_dtype, list): bench_dtype = flatten_compare_result(bench_dtype) npu_dtype = flatten_compare_result(npu_dtype) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 208c9c5f4..e2e341843 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -143,8 +143,8 @@ class Comparator: npu_dtype_total = [] shape_total = [] test_success_total = [] - max_abs_error_success = False - max_cosine_success = False + max_abs_error_success = [] + cosine_success = [] for name in self.compare_alg.keys(): alg = self.compare_alg[name][0] detailed_result, test_success, bench_dtype, npu_dtype, shape = compare_core(bench_out, npu_out, alg) @@ -154,36 +154,38 @@ class Comparator: if name not in ["Cosine Similarity", "Max Relative Error", "Max Absolute Error"]: test_success_total.append(test_success) if name == "Cosine Similarity": - max_cosine_success = test_success + cosine_success.append(test_success) if name == "Max Relative Error": - max_abs_error_success = test_success + max_abs_error_success.append(test_success) if detailed_result_total: for i in range(len(detailed_result_total)): detailed_result_total[i] += detailed_result[i] else: detailed_result_total = detailed_result - test_final_result = 'pass' - if not max_cosine_success: - test_final_result = 'error' - elif max_abs_error_success: - test_final_result = 'pass' - else: - if 'error' in test_success_total: - test_final_result = 'error' - elif 'warning' in test_success_total: - test_final_result = 'warning' + test_all_result = ['pass' for _ in range(len(detailed_result_total))] + for i in range(len(test_all_result)): + if not cosine_success[0][i] or 'error' in cosine_success[0][i]: + test_all_result[i] = 'error' + elif max_abs_error_success[0][i]: + test_all_result[i] = 'pass' + else: + test_success_column = [test_success_single[i] for test_success_single in test_success_total] + if 'error' in test_success_column or False in test_success_column: + test_all_result[i] = 'error' + elif 'warning' in test_success_column: + test_all_result[i] = 'warning' # dtype加到所有指标的前面, 是否pass放到所有指标的后面 for i in range(len(detailed_result_total)): detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) detailed_result.insert(2, shape_total[i]) - detailed_result.append(str(test_final_result)) + detailed_result.append(test_all_result[i]) detailed_result_total[i] = tuple(detailed_result) - if test_final_result == 'pass': - test_final_success = True - else: + if 'error' in test_all_result or 'warning' in test_all_result: test_final_success = False + else: + test_final_success = True return test_final_success, detailed_result_total @staticmethod -- Gitee From 777bcdf8296d0c8bfb98e31f98bb0ce0a9f99991 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 25 Sep 2023 19:44:24 +0800 Subject: [PATCH 03/13] fix --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index e2e341843..86fab8153 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -164,9 +164,9 @@ class Comparator: detailed_result_total = detailed_result test_all_result = ['pass' for _ in range(len(detailed_result_total))] for i in range(len(test_all_result)): - if not cosine_success[0][i] or 'error' in cosine_success[0][i]: + if not cosine_success[0][i] or 'error' == cosine_success[0][i]: test_all_result[i] = 'error' - elif max_abs_error_success[0][i]: + elif max_abs_error_success[0][i] or 'pass' == max_abs_error_success[0][i]: test_all_result[i] = 'pass' else: test_success_column = [test_success_single[i] for test_success_single in test_success_total] -- Gitee From 26feccf72e118028ecdf2bf38ea658864ec4d16c Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 26 Sep 2023 10:45:51 +0800 Subject: [PATCH 04/13] fix --- .../api_accuracy_checker/compare/algorithm.py | 29 +++++++++---------- .../api_accuracy_checker/compare/compare.py | 23 +++++++-------- .../compare/compare_utils.py | 3 ++ 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 0a0ff2003..c21bf2e48 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -21,10 +21,7 @@ def compare_bool_tensor(cpu_output, npu_output): return CompareConst.NAN, False, "" error_nums = (cpu_output != npu_output).sum() error_rate = float(error_nums / cpu_output.size) - if error_rate == 0: - result = 'pass' - else: - result = 'error' + result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR return error_rate, result, "" def get_msg_and_handle_value(b_value, n_value): @@ -66,27 +63,27 @@ def get_rel_err_ratio_hundredth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.01) if n_value.dtype != np.float16: msg = f"This indicator is not used to evaluate {n_value.dtype} data" - return ratio, 'pass', msg + return ratio, CompareConst.PASS, msg if bool_result: - return ratio, 'pass', msg - return ratio, 'error', msg + return ratio, CompareConst.PASS, msg + return ratio, CompareConst.ERROR, msg def get_rel_err_ratio_thousandth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.001) if bool_result: - return ratio, 'pass', msg + return ratio, CompareConst.PASS, msg if n_value.dtype == np.float16: - return ratio, 'warning', msg - return ratio, 'error', msg + return ratio, CompareConst.WARNING, msg + return ratio, CompareConst.ERROR, msg def get_rel_err_ratio_ten_thousandth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.0001) if n_value.dtype == np.float16: msg = f"This indicator is not used to evaluate {n_value.dtype} data" - return ratio, 'pass', msg + return ratio, CompareConst.PASS, msg if bool_result: - return ratio, 'pass', msg - return ratio, 'warning', msg + return ratio, CompareConst.PASS, msg + return ratio, CompareConst.WARNING, msg def get_rel_err_ratio(b_value, n_value, thresholding): b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) @@ -141,10 +138,10 @@ def compare_uint8_data(b_value, n_value): def compare_builtin_type(bench_out, npu_out): if not isinstance(bench_out, (bool, int, float, str)): - return CompareConst.NA, 'pass', "" + return CompareConst.NA, CompareConst.PASS, "" if bench_out != npu_out: - return CompareConst.NAN, 'error', "" - return True, 'pass', "" + return CompareConst.NAN, CompareConst.ERROR, "" + return True, CompareConst.PASS, "" def flatten_compare_result(result): flatten_result = [] diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 86fab8153..6936ea01e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -162,18 +162,18 @@ class Comparator: detailed_result_total[i] += detailed_result[i] else: detailed_result_total = detailed_result - test_all_result = ['pass' for _ in range(len(detailed_result_total))] + test_all_result = [CompareConst.PASS for _ in range(len(detailed_result_total))] for i in range(len(test_all_result)): - if not cosine_success[0][i] or 'error' == cosine_success[0][i]: - test_all_result[i] = 'error' - elif max_abs_error_success[0][i] or 'pass' == max_abs_error_success[0][i]: - test_all_result[i] = 'pass' + if not cosine_success[0][i] or CompareConst.ERROR == cosine_success[0][i]: + test_all_result[i] = CompareConst.ERROR + elif max_abs_error_success[0][i] or CompareConst.PASS == max_abs_error_success[0][i]: + test_all_result[i] = CompareConst.PASS else: test_success_column = [test_success_single[i] for test_success_single in test_success_total] - if 'error' in test_success_column or False in test_success_column: - test_all_result[i] = 'error' - elif 'warning' in test_success_column: - test_all_result[i] = 'warning' + if CompareConst.ERROR in test_success_column or False in test_success_column: + test_all_result[i] = CompareConst.ERROR + elif CompareConst.WARNING in test_success_column: + test_all_result[i] = CompareConst.WARNING # dtype加到所有指标的前面, 是否pass放到所有指标的后面 for i in range(len(detailed_result_total)): detailed_result = list(detailed_result_total[i]) @@ -182,10 +182,7 @@ class Comparator: detailed_result.insert(2, shape_total[i]) detailed_result.append(test_all_result[i]) detailed_result_total[i] = tuple(detailed_result) - if 'error' in test_all_result or 'warning' in test_all_result: - test_final_success = False - else: - test_final_success = True + test_final_success = False if CompareConst.ERROR in test_all_result or CompareConst.WARNING in test_all_result else True return test_final_success, detailed_result_total @staticmethod diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 62044f585..0bb80fbce 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -5,6 +5,9 @@ import numpy as np class CompareConst: NAN = np.nan NA = "N/A" + PASS = 'pass' + WARNING = 'warning' + ERROR = 'error' def check_dtype_comparable(x, y): -- Gitee From 13465d3c5aa5a4929dcb6f32fdb8ccfef17fe135 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 3 Nov 2023 07:43:18 +0000 Subject: [PATCH 05/13] update debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/test/ut/common/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py index ed764987d..a68057dfb 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_config.py @@ -17,5 +17,5 @@ class TestConfig(unittest.TestCase): def test_update_config(self): - self.config.update_config(dump_path='/new/path/to/dump', enable_dataloader=False) + self.config.update_config(dump_path='/new/path/to/dump') self.assertEqual(self.config.dump_path, '/new/path/to/dump') -- Gitee From c0e54f80b68b693214e925937bafb4cb9dba3aef Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 3 Nov 2023 07:46:20 +0000 Subject: [PATCH 06/13] update debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py. Signed-off-by: jiangchangting1 --- .../test/ut/compare/test_algorithm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 701ea6f7a..54d452ed0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -8,12 +8,12 @@ class TestAlgorithmMethods(unittest.TestCase): cpu_output = np.array([1, 2, 3]) npu_output = np.array([1, 2, 3]) compare_alg = alg.get_max_rel_err - self.assertEqual(alg.compare_torch_tensor(cpu_output, npu_output, compare_alg), (0.0, True, '')) + self.assertEqual(alg.compare_torch_tensor(cpu_output, npu_output, compare_alg), (0.0, 'pass', '')) def test_compare_bool_tensor(self): cpu_output = np.array([True, False, True]) npu_output = np.array([True, False, True]) - self.assertEqual(alg.compare_bool_tensor(cpu_output, npu_output), (0.0, True, '')) + self.assertEqual(alg.compare_bool_tensor(cpu_output, npu_output), (0.0, 'pass', '')) def test_get_msg_and_handle_value(self): b_value = np.array([1.0, 2.0, 3.0]) @@ -33,12 +33,12 @@ class TestAlgorithmMethods(unittest.TestCase): def test_get_rel_err_ratio_thousandth(self): b_value = np.array([1.0, 2.0, 3.0]) n_value = np.array([1.0, 2.0, 3.0]) - self.assertEqual(alg.get_rel_err_ratio_thousandth(b_value, n_value), (1.0, True, '')) + self.assertEqual(alg.get_rel_err_ratio_thousandth(b_value, n_value), (1.0, 'pass', '')) def test_get_rel_err_ratio_ten_thousandth(self): b_value = np.array([1.0, 2.0, 3.0]) n_value = np.array([1.0, 2.0, 3.0]) - self.assertEqual(alg.get_rel_err_ratio_ten_thousandth(b_value, n_value), (1.0, True, '')) + self.assertEqual(alg.get_rel_err_ratio_ten_thousandth(b_value, n_value), (1.0, 'pass', '')) def test_max_rel_err_standard(self): max_rel_errs = [0.0001, 0.0002, 0.0003] @@ -65,7 +65,7 @@ class TestAlgorithmMethods(unittest.TestCase): def test_compare_builtin_type(self): bench_out = 1 npu_out = 1 - self.assertEqual(alg.compare_builtin_type(bench_out, npu_out), (True, True, '')) + self.assertEqual(alg.compare_builtin_type(bench_out, npu_out), (True, 'pass', '')) def test_flatten_compare_result(self): result = [[1, 2], [3, 4]] -- Gitee From 58c152bdee8300f4369bc9723da148984160adf7 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 3 Nov 2023 07:48:41 +0000 Subject: [PATCH 07/13] update debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/test/ut/dump/test_dump_scopr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py index addba38e3..b892a6077 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/dump/test_dump_scopr.py @@ -10,12 +10,12 @@ class TestDumpScope(unittest.TestCase): wrapped_func = iter_tracer(dummy_func) result = wrapped_func() - self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(DumpUtil.dump_switch, "OFF") self.assertEqual(result, "Hello, World!") def another_dummy_func(): return 123 wrapped_func = iter_tracer(another_dummy_func) result = wrapped_func() - self.assertEqual(DumpUtil.dump_switch, "ON") + self.assertEqual(DumpUtil.dump_switch, "OFF") self.assertEqual(result, 123) -- Gitee From 12ad2f0b258084bba7a316fdaabf2cf8d8590e28 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 3 Nov 2023 08:29:33 +0000 Subject: [PATCH 08/13] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 3baed1d2f..1574bd0d7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -67,6 +67,7 @@ def get_max_abs_err(b_value, n_value): bool_result = abs_err < 0.001 return abs_err, bool_result, msg + def get_rel_err_ratio_hundredth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.01) if n_value.dtype != np.float16: @@ -76,6 +77,7 @@ def get_rel_err_ratio_hundredth(b_value, n_value): return ratio, CompareConst.PASS, msg return ratio, CompareConst.ERROR, msg + def get_rel_err_ratio_thousandth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.001) if bool_result: -- Gitee From a69a4f44262db6b6b8195d49fdcad5d0ee44600e Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 3 Nov 2023 08:36:48 +0000 Subject: [PATCH 09/13] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index ff2af0a51..2b16dfe5a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -185,7 +185,7 @@ class Comparator: else: detailed_result_total = detailed_result test_all_result = [CompareConst.PASS for _ in range(len(detailed_result_total))] - for i in range(len(test_all_result)): + for i, _ in enumerate(test_all_result): if not cosine_success[0][i] or CompareConst.ERROR == cosine_success[0][i]: test_all_result[i] = CompareConst.ERROR elif max_abs_error_success[0][i] or CompareConst.PASS == max_abs_error_success[0][i]: -- Gitee From e354350075a118a1f05ef9b1356eceb8f2a2abed Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 4 Nov 2023 07:22:00 +0000 Subject: [PATCH 10/13] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 1574bd0d7..e490bf3d4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -11,7 +11,7 @@ def compare_torch_tensor(cpu_output, npu_output, compare_alg): npu output dtype is {npu_output.dtype}, cannot compare." if cpu_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: if compare_alg == cosine_sim: - return CompareConst.NA, False, f"Compare algorithm {compare_alg.__name__} is not supported for {cpu_output.dtype} data." + return CompareConst.NA, True, f"Compare algorithm {compare_alg.__name__} is not supported for {cpu_output.dtype} data." return compare_bool_tensor(cpu_output, npu_output) return compare_alg(cpu_output, npu_output) -- Gitee From af53baa5ad6c054cfa417f89a9fab3c54365679f Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 6 Nov 2023 03:29:28 +0000 Subject: [PATCH 11/13] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 5d4ce0155..fb0ae1f70 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -6,7 +6,6 @@ from rich.console import Console from api_accuracy_checker.compare.algorithm import compare_core, cosine_sim, cosine_standard, get_max_rel_err, get_max_abs_err, \ compare_builtin_type, get_rel_err_ratio_hundredth, get_rel_err_ratio_thousandth, get_rel_err_ratio_ten_thousandth from api_accuracy_checker.common.utils import get_json_contents, print_info_log, print_error_log, write_csv, CompareException - from api_accuracy_checker.compare.compare_utils import CompareConst from api_accuracy_checker.common.config import msCheckerConfig -- Gitee From 1eacd98ae8edfba119bd6a89bf3c0c606331dcd0 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 6 Nov 2023 09:56:16 +0000 Subject: [PATCH 12/13] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index fb0ae1f70..6590869f2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -180,9 +180,9 @@ class Comparator: if name not in ["Cosine Similarity", "Max Relative Error", "Max Absolute Error"]: test_success_total.append(test_success) if name == "Cosine Similarity": - cosine_success.append(test_success) + cosine_success = test_success if name == "Max Relative Error": - max_abs_error_success.append(test_success) + max_abs_error_success = test_success if detailed_result_total: for i, detailed_result_item in enumerate(detailed_result): detailed_result_total[i] += detailed_result_item @@ -190,9 +190,9 @@ class Comparator: detailed_result_total = detailed_result test_all_result = [CompareConst.PASS for _ in range(len(detailed_result_total))] for i, _ in enumerate(test_all_result): - if not cosine_success[0][i] or CompareConst.ERROR == cosine_success[0][i]: + if not cosine_success[i] or CompareConst.ERROR == cosine_success[i]: test_all_result[i] = CompareConst.ERROR - elif max_abs_error_success[0][i] or CompareConst.PASS == max_abs_error_success[0][i]: + elif max_abs_error_success[i] or CompareConst.PASS == max_abs_error_success[i]: test_all_result[i] = CompareConst.PASS else: test_success_column = [test_success_single[i] for test_success_single in test_success_total] -- Gitee From 72244ba86acc44038f383487e8a8c9f777becb99 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 6 Nov 2023 10:09:29 +0000 Subject: [PATCH 13/13] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index d69b5d4d2..3a8021a01 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -192,7 +192,7 @@ def compare_core(bench_out, npu_out, alg): elif isinstance(bench_out, dict): b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) if b_keys != n_keys: - compare_result, test_success, bench_dtype, npu_dtype, shape = [(CompareConst.NA, "bench and npu output dict keys are different")], False, \ + compare_result, test_success, bench_dtype, npu_dtype, shape = [(CompareConst.NA, "bench and npu output dict keys are different")], [False], \ [CompareConst.NA], [CompareConst.NA], [CompareConst.NA] else: compare_result, test_success, bench_dtype, npu_dtype, shape = compare_core(list(bench_out.values()), list(npu_out.values()), alg) -- Gitee