From 5aa169a8babbea1eb28e66a7709bccabb43d583a Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 15 Sep 2023 16:27:35 +0800 Subject: [PATCH 1/5] 16 --- .../api_accuracy_checker/compare/algorithm.py | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 9b6b853073..c7f2957e1e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -25,58 +25,58 @@ def compare_bool_tensor(cpu_output, npu_output): return error_rate, error_rate == 0, "" -def get_msg_and_handle_value(n_value, b_value): +def get_msg_and_handle_value(b_value, n_value): msg = "" - if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray): - msg = f"Max rel err only support numpy array! The actual type is {type(n_value)}, {type(b_value)}." + if not isinstance(b_value, np.ndarray) or not isinstance(n_value, np.ndarray): + msg = f"Max rel err only support numpy array! The actual type is {type(b_value)}, {type(n_value)}." return CompareConst.NAN, False, msg - if n_value.shape != b_value.shape: - msg = f"Shape of npu and bench outputs don't match. NPU: {n_value.shape}, bench: {b_value.shape}." + if b_value.shape != n_value.shape: + msg = f"Shape of bench and npu outputs don't match. bench: {b_value.shape}, npu: {n_value.shape}." return CompareConst.NAN, False, msg if n_value.dtype != b_value.dtype: - msg = f"Dtype of npu and bench outputs don't match. NPU: {n_value.dtype}, bench: {b_value.dtype}." + msg = f"Dtype of bench and npu outputs don't match. bench: {b_value.dtype}, npu: {n_value.dtype}." - if b_value.dtype in Const.FLOAT_TYPE: - zero_mask = (b_value == 0) + if n_value.dtype in Const.FLOAT_TYPE: + zero_mask = (n_value == 0) # 给0的地方加上eps防止除0 - b_value[zero_mask] += np.finfo(b_value.dtype).eps - # 根据b_value为0的位置给n_value也加上eps，否则两者都是0的情况下相对误差会是1 - n_value[zero_mask] += np.finfo(b_value.dtype).eps + n_value[zero_mask] += np.finfo(n_value.dtype).eps + # 根据n_value为0的位置给n_value也加上eps，否则两者都是0的情况下相对误差会是1 + b_value[zero_mask] += np.finfo(n_value.dtype).eps else: # int type + float eps 会报错，所以这里要强转 - n_value, b_value = n_value.astype(float), b_value.astype(float) - zero_mask = (b_value == 0) - b_value[zero_mask] += np.finfo(float).eps + b_value, n_value = b_value.astype(float), n_value.astype(float) + zero_mask = (n_value == 0) n_value[zero_mask] += np.finfo(float).eps - return n_value, b_value, msg + b_value[zero_mask] += np.finfo(float).eps + return b_value, n_value, msg -def get_max_rel_err(n_value, b_value): - n_value, b_value, msg = get_msg_and_handle_value(n_value, b_value) - rel_err = np.abs((n_value - b_value) / b_value).max() +def get_max_rel_err(b_value, n_value): + b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) + rel_err = np.abs((b_value - n_value) / n_value).max() bool_result = rel_err < 0.001 return rel_err, bool_result, msg -def get_max_abs_err(n_value, b_value): - n_value, b_value, msg = get_msg_and_handle_value(n_value, b_value) - abs_err = np.abs(n_value - b_value).max() +def get_max_abs_err(b_value, n_value): + b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) + abs_err = np.abs(b_value - n_value).max() bool_result = abs_err < 0.001 return abs_err, bool_result, msg -def get_rel_err_ratio_thousandth(n_value, b_value): - return get_rel_err_ratio(n_value, b_value, 0.001) +def get_rel_err_ratio_thousandth(b_value, n_value): + return get_rel_err_ratio(b_value, n_value, 0.001) -def get_rel_err_ratio_ten_thousandth(n_value, b_value): - ratio, bool_result, msg = get_rel_err_ratio(n_value, b_value, 0.0001) - if b_value.dtype == np.float16: - msg = f"This indicator is not used to evaluate {b_value.dtype} data" +def get_rel_err_ratio_ten_thousandth(b_value, n_value): + ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.0001) + if n_value.dtype == np.float16: + msg = f"This indicator is not used to evaluate {n_value.dtype} data" return ratio, True, msg return ratio, bool_result, msg -def get_rel_err_ratio(n_value, b_value, thresholding): - n_value, b_value, msg = get_msg_and_handle_value(n_value, b_value) - rel_errs = np.abs((n_value - b_value) / b_value) +def get_rel_err_ratio(b_value, n_value, thresholding): + b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) + rel_errs = np.abs((b_value - n_value) / n_value) ratio = np.divide(np.sum(rel_errs < thresholding), np.size(rel_errs)) bool_result = ratio > (1 - thresholding) return ratio, bool_result, msg @@ -123,8 +123,8 @@ def cosine_sim(cpu_output, npu_output): return cos, cos > 0.99, msg -def compare_uint8_data(n_value, b_value): - if (n_value == b_value).all(): +def compare_uint8_data(b_value, n_value): + if (b_value == n_value).all(): return 1, True else: return 0, False @@ -175,8 +175,6 @@ def compare_core(bench_out, npu_out, alg): bench_dtype = str(copy_bench_out.dtype) npu_dtype = str(copy_npu_out.dtype) shape = list(npu_out.shape) - if copy_bench_out.dtype in [torch.float32, torch.float64] and copy_bench_out.dtype != copy_npu_out.dtype: - copy_npu_out = copy_npu_out.type(copy_bench_out.dtype) compare_result, test_success, msg = compare_torch_tensor(copy_bench_out.numpy(), copy_npu_out.cpu().numpy(), alg) elif isinstance(bench_out, (bool, int, float, str)): compare_result, test_success, msg = compare_builtin_type(bench_out, npu_out) -- Gitee From e7f269cdc36df3e519fe6d9b20058ce15e4f2558 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 18 Sep 2023 10:49:49 +0800 Subject: [PATCH 2/5] fix --- debug/accuracy_tools/api_accuracy_checker/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index 9304a9ca4f..065c0fe46d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -69,7 +69,7 @@ class Const: OFF = 'OFF' BACKWARD = 'backward' FORWARD = 'forward' - FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble] + FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16] BOOL_TYPE = [bool, np.uint8] INT_TYPE = [np.int32, np.int64] -- Gitee From aaec42ad80b2975200d7fe5238d0368053c444ce Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 18 Sep 2023 11:00:23 +0800 Subject: [PATCH 3/5] fix --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index c7f2957e1e..3ae9635bdc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -33,8 +33,6 @@ def get_msg_and_handle_value(b_value, n_value): if b_value.shape != n_value.shape: msg = f"Shape of bench and npu outputs don't match. bench: {b_value.shape}, npu: {n_value.shape}." return CompareConst.NAN, False, msg - if n_value.dtype != b_value.dtype: - msg = f"Dtype of bench and npu outputs don't match. bench: {b_value.dtype}, npu: {n_value.dtype}." if n_value.dtype in Const.FLOAT_TYPE: zero_mask = (n_value == 0) -- Gitee From 01dab172929641de0a78da50fa5674034498dcc3 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 18 Sep 2023 14:37:11 +0800 Subject: [PATCH 4/5] fix --- .../api_accuracy_checker/compare/algorithm.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 3ae9635bdc..7cc4cb388d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -14,7 +14,6 @@ def compare_torch_tensor(cpu_output, npu_output, compare_alg): return compare_bool_tensor(cpu_output, npu_output) return compare_alg(cpu_output, npu_output) - def compare_bool_tensor(cpu_output, npu_output): cpu_shape = cpu_output.shape npu_shape = npu_output.shape @@ -24,7 +23,6 @@ def compare_bool_tensor(cpu_output, npu_output): error_rate = float(error_nums / cpu_output.size) return error_rate, error_rate == 0, "" - def get_msg_and_handle_value(b_value, n_value): msg = "" if not isinstance(b_value, np.ndarray) or not isinstance(n_value, np.ndarray): @@ -48,10 +46,9 @@ def get_msg_and_handle_value(b_value, n_value): b_value[zero_mask] += np.finfo(float).eps return b_value, n_value, msg - def get_max_rel_err(b_value, n_value): b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) - rel_err = np.abs((b_value - n_value) / n_value).max() + rel_err = np.abs((n_value - b_value) / b_value).max() bool_result = rel_err < 0.001 return rel_err, bool_result, msg @@ -64,7 +61,6 @@ def get_max_abs_err(b_value, n_value): def get_rel_err_ratio_thousandth(b_value, n_value): return get_rel_err_ratio(b_value, n_value, 0.001) - def get_rel_err_ratio_ten_thousandth(b_value, n_value): ratio, bool_result, msg = get_rel_err_ratio(b_value, n_value, 0.0001) if n_value.dtype == np.float16: @@ -74,22 +70,19 @@ def get_rel_err_ratio_ten_thousandth(b_value, n_value): def get_rel_err_ratio(b_value, n_value, thresholding): b_value, n_value, msg = get_msg_and_handle_value(b_value, n_value) - rel_errs = np.abs((b_value - n_value) / n_value) + rel_errs = np.abs((n_value - b_value) / b_value) ratio = np.divide(np.sum(rel_errs < thresholding), np.size(rel_errs)) bool_result = ratio > (1 - thresholding) return ratio, bool_result, msg - def max_rel_err_standard(max_rel_errs): bool_result = np.array(max_rel_errs) < 0.001 return np.all(bool_result), bool_result - def cosine_standard(compare_result): bool_result = np.array(compare_result) > 0.99 return np.all(bool_result), bool_result - def cosine_sim(cpu_output, npu_output): msg = "" n_value = npu_output.reshape(-1) @@ -120,14 +113,12 @@ def cosine_sim(cpu_output, npu_output): msg = "Dump data has NaN when comparing with Cosine Similarity." return cos, cos > 0.99, msg - def compare_uint8_data(b_value, n_value): if (b_value == n_value).all(): return 1, True else: return 0, False - def compare_builtin_type(bench_out, npu_out): if not isinstance(bench_out, (bool, int, float, str)): return CompareConst.NA, True, "" @@ -135,7 +126,6 @@ def compare_builtin_type(bench_out, npu_out): return CompareConst.NAN, False, "" return True, True, "" - def flatten_compare_result(result): flatten_result = [] for result_i in result: @@ -202,6 +192,4 @@ def compare_core(bench_out, npu_out, alg): bench_dtype = [bench_dtype] npu_dtype = [npu_dtype] shape = [shape] - return compare_result, test_success, bench_dtype, npu_dtype, shape - - + return compare_result, test_success, bench_dtype, npu_dtype, shape \ No newline at end of file -- Gitee From 2767b15882de7b74f85a4a1a61556f2c7f628de5 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 18 Sep 2023 18:33:46 +0800 Subject: [PATCH 5/5] fix --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 7cc4cb388d..dd4ec514c2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -163,6 +163,9 @@ def compare_core(bench_out, npu_out, alg): bench_dtype = str(copy_bench_out.dtype) npu_dtype = str(copy_npu_out.dtype) shape = list(npu_out.shape) + if copy_npu_out.dtype == torch.bfloat16: + copy_bench_out = copy_bench_out.to(torch.float32) + copy_npu_out = copy_npu_out.to(torch.float32) compare_result, test_success, msg = compare_torch_tensor(copy_bench_out.numpy(), copy_npu_out.cpu().numpy(), alg) elif isinstance(bench_out, (bool, int, float, str)): compare_result, test_success, msg = compare_builtin_type(bench_out, npu_out) -- Gitee