From 1feebe07c7a68b2fe3df5c1f158ded308a38e3dd Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 10:05:52 +0800 Subject: [PATCH 1/9] update --- .../api_accuracy_checker/compare/algorithm.py | 6 +++--- .../compare/compare_utils.py | 16 ++++------------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 7314f51736..694f33c3b9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -7,10 +7,10 @@ from api_accuracy_checker.common.utils import Const def compare_torch_tensor(cpu_output, npu_output, compare_alg): - if not check_dtype_comparable(cpu_output, npu_output): + if check_dtype_comparable(cpu_output, npu_output): return CompareConst.NAN, False, f"Bench out dtype is {cpu_output.dtype} but\ npu output dtype is {npu_output.dtype}, cannot compare." - if cpu_output.dtype == np.bool or cpu_output.dtype == np.uint8: + if cpu_output.dtype == np.bool or cpu_output.dtype == np.uint8 or cpu_output.dtype in Const.INT_TYPE: return compare_bool_tensor(cpu_output, npu_output) return compare_alg(cpu_output, npu_output) @@ -26,7 +26,7 @@ def compare_bool_tensor(cpu_output, npu_output): data_size = bench_data.size error_nums = (bench_data != npu_data).sum() error_rate = float(error_nums / data_size) - return error_rate, error_rate < 0.001, "" + return error_rate, error_rate == 0, "" def get_msg_and_handle_value(n_value, b_value): diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 62044f5852..b102730658 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -8,17 +8,9 @@ class CompareConst: def check_dtype_comparable(x, y): - if x.dtype in Const.FLOAT_TYPE: - if y.dtype in Const.FLOAT_TYPE: - return True - return False - if x.dtype in Const.BOOL_TYPE: - if y.dtype in Const.BOOL_TYPE: - return True - return False - if x.dtype in Const.INT_TYPE: - if y.dtype in Const.INT_TYPE: - return True + if x.dtype in Const.FLOAT_TYPE and y.dtype in Const.FLOAT_TYPE: + return False + if x.dtype in Const.BOOL_TYPE and y.dtype in Const.BOOL_TYPE: return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") - return False \ No newline at end of file + return True \ No newline at end of file -- Gitee From 92abcf2d313351c3fabf91c0a0326046006b30ae Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 10:25:35 +0800 Subject: [PATCH 2/9] update --- .../api_accuracy_checker/compare/compare_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index b102730658..90ec37e28f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -12,5 +12,7 @@ def check_dtype_comparable(x, y): return False if x.dtype in Const.BOOL_TYPE and y.dtype in Const.BOOL_TYPE: return False + if x.dtype in Const.INT_TYPE and y.dtype in Const.INT_TYPE: + return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") return True \ No newline at end of file -- Gitee From 5733f861304a54dc72c03f0dddc84f2e270e8d84 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 11:23:36 +0800 Subject: [PATCH 3/9] update --- .../api_accuracy_checker/compare/compare_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 90ec37e28f..6a60290415 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -8,11 +8,8 @@ class CompareConst: def check_dtype_comparable(x, y): - if x.dtype in Const.FLOAT_TYPE and y.dtype in Const.FLOAT_TYPE: - return False - if x.dtype in Const.BOOL_TYPE and y.dtype in Const.BOOL_TYPE: - return False - if x.dtype in Const.INT_TYPE and y.dtype in Const.INT_TYPE: + dtype_set = {x.dtype, y.dtype} + if dtype_set.issubset(Const.FLOAT_TYPE) or dtype_set.issubset(Const.BOOL_TYPE) or dtype_set.issubset(Const.INT_TYPE): return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") return True \ No newline at end of file -- Gitee From e41a627783dc814fc7b776173e7f5da3e7f01a5a Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 11:39:29 +0800 Subject: [PATCH 4/9] update --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 694f33c3b9..314cbbcc5d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -10,7 +10,7 @@ def compare_torch_tensor(cpu_output, npu_output, compare_alg): if check_dtype_comparable(cpu_output, npu_output): return CompareConst.NAN, False, f"Bench out dtype is {cpu_output.dtype} but\ npu output dtype is {npu_output.dtype}, cannot compare." - if cpu_output.dtype == np.bool or cpu_output.dtype == np.uint8 or cpu_output.dtype in Const.INT_TYPE: + if cpu_output.dtype in [bool, np.uint8, np.int8, np.uint16, np.int32, np.int64]: return compare_bool_tensor(cpu_output, npu_output) return compare_alg(cpu_output, npu_output) -- Gitee From 861d9fa068ef6b29485ff9929a53fe44470b6ee5 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 11:51:34 +0800 Subject: [PATCH 5/9] update --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 314cbbcc5d..02326c224d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -10,7 +10,7 @@ def compare_torch_tensor(cpu_output, npu_output, compare_alg): if check_dtype_comparable(cpu_output, npu_output): return CompareConst.NAN, False, f"Bench out dtype is {cpu_output.dtype} but\ npu output dtype is {npu_output.dtype}, cannot compare." - if cpu_output.dtype in [bool, np.uint8, np.int8, np.uint16, np.int32, np.int64]: + if cpu_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: return compare_bool_tensor(cpu_output, npu_output) return compare_alg(cpu_output, npu_output) -- Gitee From abc4b73811478ca0e8adce1fea11a52acd15e8aa Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 12:00:37 +0800 Subject: [PATCH 6/9] revert --- .../accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- .../api_accuracy_checker/compare/compare_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 02326c224d..1745461b65 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -7,7 +7,7 @@ from api_accuracy_checker.common.utils import Const def compare_torch_tensor(cpu_output, npu_output, compare_alg): - if check_dtype_comparable(cpu_output, npu_output): + if not check_dtype_comparable(cpu_output, npu_output): return CompareConst.NAN, False, f"Bench out dtype is {cpu_output.dtype} but\ npu output dtype is {npu_output.dtype}, cannot compare." if cpu_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 6a60290415..4770c5ef40 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -10,6 +10,6 @@ class CompareConst: def check_dtype_comparable(x, y): dtype_set = {x.dtype, y.dtype} if dtype_set.issubset(Const.FLOAT_TYPE) or dtype_set.issubset(Const.BOOL_TYPE) or dtype_set.issubset(Const.INT_TYPE): - return False + return True print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") - return True \ No newline at end of file + return False \ No newline at end of file -- Gitee From be267809092c2d357de89b5efb7d7771e80ebef7 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 11 Sep 2023 12:01:14 +0800 Subject: [PATCH 7/9] update --- .../api_accuracy_checker/compare/compare_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 4770c5ef40..928ba0099f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -12,4 +12,4 @@ def check_dtype_comparable(x, y): if dtype_set.issubset(Const.FLOAT_TYPE) or dtype_set.issubset(Const.BOOL_TYPE) or dtype_set.issubset(Const.INT_TYPE): return True print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") - return False \ No newline at end of file + return False \ No newline at end of file -- Gitee From 023377dbf55ed24b422b25782318bccc9e0ef8f9 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Tue, 12 Sep 2023 16:51:21 +0800 Subject: [PATCH 8/9] revert --- .../api_accuracy_checker/compare/compare_utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 928ba0099f..62044f5852 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -8,8 +8,17 @@ class CompareConst: def check_dtype_comparable(x, y): - dtype_set = {x.dtype, y.dtype} - if dtype_set.issubset(Const.FLOAT_TYPE) or dtype_set.issubset(Const.BOOL_TYPE) or dtype_set.issubset(Const.INT_TYPE): - return True + if x.dtype in Const.FLOAT_TYPE: + if y.dtype in Const.FLOAT_TYPE: + return True + return False + if x.dtype in Const.BOOL_TYPE: + if y.dtype in Const.BOOL_TYPE: + return True + return False + if x.dtype in Const.INT_TYPE: + if y.dtype in Const.INT_TYPE: + return True + return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") return False \ No newline at end of file -- Gitee From c0b0da97faab2b1941b5dd50c4b1deba2d2581ee Mon Sep 17 00:00:00 2001 From: s30048155 Date: Thu, 14 Sep 2023 11:52:06 +0800 Subject: [PATCH 9/9] update --- .../api_accuracy_checker/compare/algorithm.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 1745461b65..f3fd31675e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -16,16 +16,12 @@ def compare_torch_tensor(cpu_output, npu_output, compare_alg): def compare_bool_tensor(cpu_output, npu_output): - error_rate = CompareConst.NAN cpu_shape = cpu_output.shape npu_shape = npu_output.shape if cpu_shape != npu_shape: - return error_rate, False, "" - npu_data = npu_output - bench_data = cpu_output - data_size = bench_data.size - error_nums = (bench_data != npu_data).sum() - error_rate = float(error_nums / data_size) + return CompareConst.NAN, False, "" + error_nums = (cpu_output != npu_output).sum() + error_rate = float(error_nums / cpu_output.size) return error_rate, error_rate == 0, "" -- Gitee