From 018f0370564f09adf834be048e49955be3b54a6d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 27 Jan 2024 10:38:22 +0000 Subject: [PATCH 01/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../compare/compare_utils.py | 105 +++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 627dbc0449..fc4b9a8018 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -1,7 +1,78 @@ -from api_accuracy_checker.common.utils import Const, print_warn_log import numpy as np +import torch +from api_accuracy_checker.common.utils import Const, print_warn_log + + + +detail_test_rows = [[ + "API Name", "Bench Dtype", "NPU Dtype", "Shape", + "Dropout", + "Builtin", + "Inf/Nan", + "jueduiyuzhi", + "baiwanfenzhiyi", + "shiwanfenzhiyi", + "wanfenzhiyi", + "wanfenzhiwu", + "qianfenzhiyi", + "qianfenzhier", + "qianfenzhisi", + "qianfenzhiwu", + "baifenzhiyi", + "wuqiong", + "xiaozhiyu", + "RMSE", + "EB", + "Max_rel_error", + "Mean_rel_error" + ]] +precision_configs = { + torch.float16 : { + 'error_distribution' : [ + ('qianfenzhiyi', 0, 1e-3), + ('qianfenzhier', 1e-3, 2e-3), + ('qianfenzhiwu', 2e-3, 5e-3), + ('baifenzhiyi', 5e-3, 1e-2), + ('wuqiong', 1e-2, np.inf) + ], + 'small_value' : [ + 1e-3 + ], + 'small_value_atol' : [ + 1e-5 + ] + }, + torch.bfloat16: { + 'error_distribution' : [ + ('qianfenzhisi', 0, 4e-3), + ('wuqiong', 4e-3, np.inf) + ], + 'small_value' : [ + 1e-3 + ], + 'small_value_atol' : [ + 1e-5 + ] + }, + torch.float32:{ + 'error_distribution' : [ + ('baiwanfenzhiyi', 0, 1e-6), + ('shiwanfenzhiyi', 1e-6, 1e-5), + ('wanfenzhiyi', 1e-5, 1e-4), + ('wanfenzhiwu', 1e-4, 5e-4), + ('wuqiong', 5e-4, np.inf) + ], + 'small_value' : [ + 1e-6 + ], + 'small_value_atol' : [ + 1e-9 + ] + } +} + class CompareConst: NAN = np.nan NA = "N/A" @@ -13,6 +84,38 @@ class CompareConst: FALSE = 'FALSE' +class CompareColumn: + def __init__(self): + self.bench_type = CompareConst.NA + self.npu_type = CompareConst.NA + self.shape = CompareConst.NA + self.dropout = CompareConst.NA + self.builtin = CompareConst.NA + self.inf_or_nan = CompareConst.NA + self.jueduiyuzhi = CompareConst.NA + self.baiwanfenzhiyi = CompareConst.NA + self.shiwanfenzhiyi = CompareConst.NA + self.wanfenzhiyi = CompareConst.NA + self.wanfenzhiwu = CompareConst.NA + self.qianfenzhiyi = CompareConst.NA + self.qianfenzhier = CompareConst.NA + self.qianfenzhisi = CompareConst.NA + self.qianfenzhiwu = CompareConst.NA + self.baifenzhiyi = CompareConst.NA + self.wuqiong = CompareConst.NA + self.xiaozhiyu = CompareConst.NA + self.RMSE = CompareConst.NA + self.EB = CompareConst.NA + self.Max_rel_error = CompareConst.NA + self.Mean_rel_error = CompareConst.NA + + def to_column_value(self): + return [self.bench_type, self.npu_type, self.shape, self.dropout, self.builtin, self.inf_or_nan, + self.jueduiyuzhi, self.baiwanfenzhiyi, self.shiwanfenzhiyi, self.wanfenzhiyi, self.wanfenzhiwu, + self.qianfenzhiyi, self.qianfenzhier, self.qianfenzhisi, self.qianfenzhiwu, self.baifenzhiyi, + self.wuqiong, self.xiaozhiyu, self.RMSE, self.EB, self.Max_rel_error, self.Mean_rel_error] + + def check_dtype_comparable(x, y): if x.dtype in Const.FLOAT_TYPE: if y.dtype in Const.FLOAT_TYPE: -- Gitee From 81902dc1bc437101938dc88ae5980654e176bfde Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 27 Jan 2024 10:39:35 +0000 Subject: [PATCH 02/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 178 +++++++++++++++--- 1 file changed, 153 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index f4d4154d23..6ff398d252 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -1,11 +1,16 @@ # 进行比对及结果展示 import os import csv +import torch +import numpy as np +import xlsxwriter from rich.table import Table from rich.console import Console -from api_accuracy_checker.compare.algorithm import compare_core from api_accuracy_checker.common.utils import get_json_contents, write_csv -from api_accuracy_checker.compare.compare_utils import CompareConst +from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ + detail_test_rows, XlsxSheetWriter, precision_configs +from api_accuracy_checker.compare.algorithm import get_absolute_threshold, get_distribution_ratio, get_rmse, \ + get_small_value_error, get_error_balance, get_max_rel_err, get_mean_rel_err, get_rel_err, get_abs_err from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -96,17 +101,6 @@ class Comparator: summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]] write_csv(summary_test_rows, self.save_path) - detail_test_rows = [[ - "Npu Name", "Bench Dtype", "NPU Dtype", "Shape", - "Cosine Similarity", - "Max Abs Error", - "Relative Error (dual hundredth)", - "Relative Error (dual thousandth)", - "Relative Error (dual ten thousandth)", - "Error Rate", - "Status", - "Message" - ]] write_csv(detail_test_rows, self.detail_save_path) def write_summary_csv(self, test_result): @@ -147,11 +141,11 @@ class Comparator: self.write_summary_csv(args) self.write_detail_csv(args) - def compare_output(self, api_name, bench_out, npu_out, bench_grad=None, npu_grad=None): + def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): if "dropout" in api_name: - is_fwd_success, fwd_compare_alg_results = self._compare_dropout(bench_out, npu_out) + is_fwd_success, fwd_compare_alg_results = self._compare_dropout(bench_output, device_output) else: - is_fwd_success, fwd_compare_alg_results = self._compare_core_wrapper(bench_out, npu_out) + is_fwd_success, fwd_compare_alg_results = self._compare_core_wrapper(bench_output, device_output) if bench_grad and npu_grad: if "dropout" in api_name: is_bwd_success, bwd_compare_alg_results = self._compare_dropout(bench_grad[0], npu_grad[0]) @@ -167,29 +161,163 @@ class Comparator: bwd_compare_alg_results) return is_fwd_success, is_bwd_success - @staticmethod - def _compare_core_wrapper(bench_out, npu_out): + def _compare_core_wrapper(self, bench_output, device_output): detailed_result_total = [] test_final_success = True - status, compare_result, message = compare_core(bench_out, npu_out) + status, compare_result, message = self._compare_core(bench_output, device_output) if not isinstance(status, list): - detailed_result_total.append(compare_result.to_column_value(status, message)) + detailed_result_total.append(compare_result.to_column_value()) if status in [CompareConst.ERROR, CompareConst.WARNING]: test_final_success = False else: for item, item_status in enumerate(status): - detailed_result_total.append(compare_result[item].to_column_value(item_status, message[item])) + detailed_result_total.append(compare_result[item].to_column_value()) if item_status in [CompareConst.ERROR, CompareConst.WARNING]: test_final_success = False return test_final_success, detailed_result_total - @staticmethod - def _compare_dropout(bench_out, npu_out): - tensor_num = bench_out.numel() + + def _compare_dropout(self, bench_output, device_output): + tensor_num = bench_output.numel() if tensor_num >= 100: - if abs((bench_out == 0).sum() - (npu_out == 0).cpu().sum()) / tensor_num < 0.1: + if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: return True, 1 else: return False, 0 else: return True, 1 + + def _compare_core(self, bench_output, device_output): + compare_column = CompareColumn() + if not isinstance(bench_output, type(device_output)): + return CompareConst.ERROR, compare_column, "bench and npu output type is different." + if isinstance(bench_output, (list, tuple)): + status, compare_result, message = [], [], [] + if len(bench_output) != len(device_output): + return CompareConst.ERROR, compare_column, "bench and npu output structure is different." + for b_out_i, n_out_i in zip(bench_output, device_output): + status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) + status.append(status_i) + compare_result.append(compare_result_i) + message.append(message_i) + elif isinstance(bench_output, dict): + b_keys, n_keys = set(bench_output.keys()), set(device_output.keys()) + if b_keys != n_keys: + return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." + else: + status, compare_result, message = self._compare_core(list(bench_output.values()), list(device_output.values())) + elif isinstance(bench_output, torch.Tensor): + copy_bench_out = bench_output.detach().clone() + copy_device_output = device_output.detach().clone() + compare_column.bench_type = str(copy_bench_out.dtype) + compare_column.npu_type = str(copy_device_output.dtype) + compare_column.shape = tuple(device_output.shape) + status, compare_result, message = self._compare_torch_tensor(copy_bench_out, copy_device_output, + compare_column) + elif isinstance(bench_output, (bool, int, float, str)): + compare_column.bench_dtype = str(type(bench_output)) + compare_column.npu_dtype = str(type(device_output)) + compare_column.shape = str(type(device_output)) + compare_result = self._compare_builtin_type(bench_output, device_output, compare_column) + elif bench_output is None: + return CompareConst.PASS, compare_column, "Output is None." + else: + return CompareConst.PASS, compare_column, "Unexpected output type in compare_core: {}".format(type(bench_output)) + + return status, compare_result, message + + + def _compare_torch_tensor(self, bench_output, device_output, compare_column): + cpu_shape = bench_output.shape + npu_shape = device_output.shape + npu_dtype = device_output.dtype + if npu_dtype == torch.bfloat16: + bench_output = bench_output.to(torch.float32) + device_output = device_output.to(torch.float32) + bench_output = bench_output.numpy() + device_output = device_output.cpu().numpy() + if cpu_shape != npu_shape: + return CompareConst.ERROR, compare_column, f"The shape of bench{str(cpu_shape)} " \ + f"and npu{str(npu_shape)} not equal." + if not check_dtype_comparable(bench_output, device_output): + return CompareConst.ERROR, compare_column, f"Bench out dtype is {bench_output.dtype} but " \ + f"npu output dtype is {device_output.dtype}, cannot compare." + message = "" + if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: + message += f"Compare algorithm cosine_sim is not supported for {bench_output.dtype} data. " \ + f"Only judged by Error Rate." + err_rate, status, msg = self._compare_bool_tensor(bench_output, device_output) + message += msg + "\n" + compare_column.error_rate = err_rate + return status, compare_column, message + else: + compare_column = self._compare_float_tensor(bench_output, device_output, compare_column, npu_dtype) + + return CompareConst.PASS, compare_column, message + + @staticmethod + def _compare_builtin_type(bench_output, device_output, compare_column): + if bench_output != device_output: + compare_column.builtin = 0 + compare_column.builtin = 1 + return compare_column + + @staticmethod + def _flatten_compare_result(result): + flatten_result = [] + for result_i in result: + if isinstance(result_i, list): + flatten_result += flatten_compare_result(result_i) + else: + flatten_result.append(result_i) + return flatten_result + + @staticmethod + def _compare_bool_tensor(bench_output, device_output): + error_nums = (bench_output != device_output).sum() + if bench_output.size == 0: + return CompareConst.NAN, CompareConst.ERROR, "There is not cpu calculation result." + error_rate = float(error_nums / bench_output.size) + result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR + return error_rate, result, "" + + @staticmethod + def _compare_float_tensor(bench_output, device_output, compare_column, dtype): + dtype_config = precision_configs.get(dtype) + eps = np.finfo(bench_output.dtype).eps + abs_bench = np.abs(bench_output) + abs_bench_with_eps = abs_bench + eps + device_finite_mask = np.isfinite(device_output) + bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) + both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) + inf_nan_mask = np.logical_not(both_finite_mask) + #inf/nan处理 + if np.sum(inf_nan_mask) == 0: + compare_column.inf_or_nan = 0 + else: + compare_column.inf_or_nan = np.sum(inf_nan_mask) + compare_column.inf_or_nan = check_inf_nan_value(bench_output, device_output, inf_nan_mask, abs_bench_with_eps) + + #小值域 + abs_err = get_abs_err(bench_output, device_output) + small_value_mask = np.less_equal(np.abs(bench_output), dtype_config['small_value'][0]) + small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + + compare_column.xiaozhiyu = get_small_value_error(small_value_mask, abs_err_greater_mask) + + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + + #误差分布 + for attr, start, end in dtype_config['error_distribution']: + setattr(compare_column, attr, get_distribution_ratio(rel_err, start, end)) + + compare_column.jueduiyuzhi = get_absolute_threshold(rel_err, abs_err) + + compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) + + compare_column.EB = get_error_balance(bench_output, device_output) + compare_column.Max_rel_error = get_max_rel_err(rel_err) + compare_column.Mean_rel_error = get_mean_rel_err(rel_err) + + return compare_column -- Gitee From 3b177c8e3ce87c87c575c9c5b4ac84f76ed762db Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 27 Jan 2024 10:40:22 +0000 Subject: [PATCH 03/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/algorithm.py | 292 ++++++------------ 1 file changed, 90 insertions(+), 202 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index b3392546e0..eebdc147d7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -5,108 +5,7 @@ from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype from api_accuracy_checker.common.utils import Const -class CompareColumn: - def __init__(self): - self.bench_type = CompareConst.NA - self.npu_type = CompareConst.NA - self.shape = CompareConst.NA - self.cosine_sim = CompareConst.NA - self.max_abs_err = CompareConst.NA - self.rel_err_hundredth = CompareConst.NA - self.rel_err_thousandth = CompareConst.NA - self.rel_err_ten_thousandth = CompareConst.NA - self.error_rate = CompareConst.NA - - def to_column_value(self, is_pass, message): - return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, - self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, is_pass, message] - - -def compare_torch_tensor(cpu_output, npu_output, compare_column): - cpu_shape = cpu_output.shape - npu_shape = npu_output.shape - npu_dtype = npu_output.dtype - if npu_dtype == torch.bfloat16: - cpu_output = cpu_output.to(torch.float32) - npu_output = npu_output.to(torch.float32) - cpu_output = cpu_output.numpy() - npu_output = npu_output.cpu().numpy() - if cpu_shape != npu_shape: - return CompareConst.ERROR, compare_column, f"The shape of bench{str(cpu_shape)} " \ - f"and npu{str(npu_shape)} not equal." - if not check_dtype_comparable(cpu_output, npu_output): - return CompareConst.ERROR, compare_column, f"Bench out dtype is {cpu_output.dtype} but " \ - f"npu output dtype is {npu_output.dtype}, cannot compare." - message = "" - if cpu_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: - message += f"Compare algorithm cosine_sim is not supported for {cpu_output.dtype} data. " \ - f"Only judged by Error Rate." - err_rate, status, msg = compare_bool_tensor(cpu_output, npu_output) - message += msg + "\n" - compare_column.error_rate = err_rate - return status, compare_column, message - # cos - cos_res, cos_status, msg = cosine_sim(cpu_output, npu_output) - compare_column.cosine_sim = cos_res - message += msg + "\n" - if not cos_status: - return CompareConst.ERROR, compare_column, message - # abs err - b_value, n_value = get_msg_and_handle_value(cpu_output, npu_output) - abs_err = np.abs(b_value - n_value) - max_abs_res, max_abs_status = get_max_abs_err(abs_err) - compare_column.max_abs_err = max_abs_res - if max_abs_status: - return CompareConst.PASS, compare_column, message - # rel err - rel_err = get_rel_err(abs_err, b_value) - if npu_dtype in [torch.float16, torch.bfloat16]: - hundred_res, hundred_status = get_rel_err_ratio(rel_err, 0.01) - compare_column.rel_err_hundredth = hundred_res - if not hundred_status: - return CompareConst.ERROR, compare_column, message - thousand_res, thousand_status = get_rel_err_ratio(rel_err, 0.001) - compare_column.rel_err_thousandth = thousand_res - if npu_dtype in [torch.float16, torch.bfloat16]: - if thousand_status: - return CompareConst.PASS, compare_column, message - return CompareConst.WARNING, compare_column, message - ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err, 0.0001) - compare_column.rel_err_ten_thousandth = ten_thousand_res - if npu_dtype in [torch.float32, torch.float64]: - if not thousand_status: - return CompareConst.ERROR, compare_column, message - if not ten_thousand_status: - return CompareConst.WARNING, compare_column, message - return CompareConst.PASS, compare_column, message - - -def compare_bool_tensor(cpu_output, npu_output): - error_nums = (cpu_output != npu_output).sum() - if cpu_output.size == 0: - return CompareConst.NAN, CompareConst.ERROR, "There is not cpu calculation result." - error_rate = float(error_nums / cpu_output.size) - result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR - return error_rate, result, "" - - -def get_msg_and_handle_value(b_value, n_value): - if n_value.dtype in Const.FLOAT_TYPE: - zero_mask = (n_value == 0) - # 给0的地方加上eps防止除0 - n_value[zero_mask] += np.finfo(n_value.dtype).eps - # 根据n_value为0的位置给n_value也加上eps，否则两者都是0的情况下相对误差会是1 - b_value[zero_mask] += np.finfo(n_value.dtype).eps - else: - # int type + float eps 会报错，所以这里要强转 - b_value, n_value = b_value.astype(float), n_value.astype(float) - zero_mask = (n_value == 0) - n_value[zero_mask] += np.finfo(float).eps - b_value[zero_mask] += np.finfo(float).eps - return b_value, n_value - - -def get_rel_err(abs_err, b_value): +def get_rel_err1(abs_err, b_value): rel_err = np.abs(abs_err / b_value) return rel_err @@ -131,109 +30,98 @@ def max_rel_err_standard(max_rel_errs): return np.all(bool_result), bool_result -def cosine_standard(compare_result): - bool_result = np.array(compare_result) > 0.99 - return np.all(bool_result), bool_result -def cosine_sim(cpu_output, npu_output): - msg = "" - n_value = npu_output.reshape(-1) - b_value = cpu_output.reshape(-1) - cos = CompareConst.NA - np.seterr(divide="ignore", invalid="ignore") - if n_value.shape != b_value.shape: - msg = f"Shape of npu and bench outputs don't match. NPU: {n_value.shape}, bench: {b_value.shape}." - return -1, False, msg - if len(n_value) == 1: - msg = "All the data in npu dump data is scalar. Please refer to other compare algorithms." - return cos, True, msg - n_value_max = np.max(np.abs(n_value)) - b_value_max = np.max(np.abs(b_value)) - if n_value_max <= np.finfo(float).eps and b_value_max <= np.finfo(float).eps: - return cos, True, msg - elif n_value_max <= np.finfo(float).eps: - msg = "All the data is zero in npu dump data." - return CompareConst.NA, False, msg - elif b_value_max <= np.finfo(float).eps: - msg = "All the data is zero in bench dump data." - return CompareConst.NA, False, msg - else: - n_value = n_value_max.astype(float) / n_value_max - b_value = b_value_max.astype(float) / b_value_max - cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) - if np.isnan(cos): - msg = "Dump data has NaN when comparing with Cosine Similarity." - return cos, cos > 0.99, msg +#误差均衡性 +def get_error_balance(bench_data, device_data): + larger_count = np.sum(np.greater(device_data - bench_data.astype(device_data.dtype), 0)) + smaller_count = np.sum(np.less(device_data - bench_data.astype(device_data.dtype), 0)) + total_count = bench_data.size + error_balance = abs(larger_count - smaller_count) / total_count + return error_balance + +def get_big_value(bench_data, device_data): + small_value = 0.001 + bench_big_value = bench_data.copy() + bench_big_value[bench_data < small_value] = 1 + device_big_value = device_data.copy() + device_big_value[bench_data < small_value] = 1 + return bench_big_value, device_big_value + +def get_small_value(bench_data, device_data): + small_value = 0.001 + bench_small_value = bench_data.copy() + bench_small_value[bench_data > small_value] = 1 + device_small_value = device_data.copy() + device_small_value[bench_data > small_value] = 1 + return bench_small_value, device_small_value -def compare_uint8_data(b_value, n_value): - if (b_value == n_value).all(): - return 1, True +def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask): + rel_err_tmp = abs_err / abs_bench_with_eps + rel_err_mask = np.logical_or(small_value_mask, inf_nan_mask) + rel_err = np.where(rel_err_mask, -1, rel_err_tmp) + return rel_err + +def get_abs_err(bench_data, device_data): + abs_err = np.abs(device_data - bench_data) + return abs_err + +#相对误差最大值 +def get_max_rel_err(rel_err): + return np.max(rel_err) + +#相对误差均值 +def get_mean_rel_err(rel_err): + return np.mean(rel_err) + +#绝对阈值法 +def get_absolute_threshold(rel_err, abs_err): + rtol = 0 + etol = 0 + small_value = 0.001 + small_value_atol = 1e-9 + if np.size(rel_err) == 0: + rel_err_num = 0 else: - return 0, False - - -def compare_builtin_type(bench_out, npu_out, compare_column): - if not isinstance(bench_out, (bool, int, float, str)): - return CompareConst.PASS, compare_column, "" - if bench_out != npu_out: - return CompareConst.ERROR, compare_column, "" - compare_column.error_rate = 0 - return CompareConst.PASS, compare_column, "" - - -def flatten_compare_result(result): - flatten_result = [] - for result_i in result: - if isinstance(result_i, list): - flatten_result += flatten_compare_result(result_i) - else: - flatten_result.append(result_i) - return flatten_result - - -def compare_core(bench_out, npu_out): - compare_column = CompareColumn() - if not isinstance(bench_out, type(npu_out)): - return CompareConst.ERROR, compare_column, "bench and npu output type is different." - if isinstance(bench_out, (list, tuple)): - status, compare_result, message = [], [], [] - if len(bench_out) != len(npu_out): - return CompareConst.ERROR, compare_column, "bench and npu output structure is different." - for b_out_i, n_out_i in zip(bench_out, npu_out): - status_i, compare_result_i, message_i = compare_core(b_out_i, n_out_i) - status.append(status_i) - compare_result.append(compare_result_i) - message.append(message_i) - elif isinstance(bench_out, dict): - b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) - if b_keys != n_keys: - return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." - else: - status, compare_result, message = compare_core(list(bench_out.values()), list(npu_out.values())) - elif isinstance(bench_out, torch.Tensor): - if bench_out.numel() == 0 and npu_out.numel() == 0: - return CompareConst.PASS, compare_column, "bench and npu output is empty." - elif bench_out.numel() == 0 and npu_out.numel() != 0: - return CompareConst.ERROR, compare_column, "bench output is empty but npu output is not empty." - elif bench_out.numel() != 0 and npu_out.numel() == 0: - return CompareConst.ERROR, compare_column, "bench output is not empty but npu output is empty." - copy_bench_out = bench_out.detach().clone() - copy_npu_out = npu_out.detach().clone() - compare_column.bench_type = str(copy_bench_out.dtype) - compare_column.npu_type = str(copy_npu_out.dtype) - compare_column.shape = tuple(npu_out.shape) - status, compare_result, message = compare_torch_tensor(copy_bench_out, copy_npu_out, - compare_column) - elif isinstance(bench_out, (bool, int, float, str)): - compare_column.bench_dtype = str(type(bench_out)) - compare_column.npu_dtype = str(type(npu_out)) - compare_column.shape = str(type(npu_out)) - status, compare_result, message = compare_builtin_type(bench_out, npu_out, compare_column) - elif bench_out is None: - return CompareConst.PASS, compare_column, "Output is None." + rel_err_num = np.sum(rel_err > rtol) + if np.size(abs_err) == 0: + abs_err_num = 0 else: - return CompareConst.PASS, compare_column, "Unexpected output type in compare_core: {}".format(type(bench_out)) - - return status, compare_result, message + abs_err_num = np.sum(abs_err > small_value_atol) + return rel_err_num + abs_err_num + +#rmse +def get_rmse(abs_err, inf_nan_mask): + masked_ae = np.where(inf_nan_mask, 0, abs_err) + rmse = np.mean(np.square(masked_ae)) + inf_nan_cnt = np.sum(inf_nan_mask) + rmse = rmse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) + rmse = np.sqrt(rmse) + return rmse + +#小值域 +def get_small_value_error(small_value_mask, abs_err_greater_mask): + err_mask = np.logical_and(small_value_mask, abs_err_greater_mask) + small_value_err_num = np.sum(err_mask) + return small_value_err_num + +def get_distribution_ratio(rel_err, start, end): + + return np.sum((rel_err>start)&(rel_err Date: Mon, 29 Jan 2024 11:33:55 +0000 Subject: [PATCH 04/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../src/python/ptdbg_ascend/common/file_check_util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py index c876a019e6..ca34fc4164 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -139,14 +139,18 @@ class FileOpen: SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"] SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"] - def __init__(self, file_path, mode): + def __init__(self, file_path, mode, encoding='utf-8'): self.file_path = file_path self.mode = mode + self.encoding = encoding self._handle = None def __enter__(self): self.check_file_path() - self._handle = open(self.file_path, self.mode) + if "b" not in self.mode: + self._handle = open(self.file_path, self.mode, encoding=self.encoding) + else: + self._handle = open(self.file_path, self.mode) return self._handle def __exit__(self, exc_type, exc_val, exc_tb): -- Gitee From 66cfd9ebf60b121d10160f270598b5214b1e15ca Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 11:57:38 +0000 Subject: [PATCH 05/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../compare/compare_utils.py | 79 ++++++------------- 1 file changed, 23 insertions(+), 56 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index fc4b9a8018..7dedfc436d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -3,40 +3,25 @@ import torch from api_accuracy_checker.common.utils import Const, print_warn_log - detail_test_rows = [[ "API Name", "Bench Dtype", "NPU Dtype", "Shape", - "Dropout", - "Builtin", - "Inf/Nan", - "jueduiyuzhi", - "baiwanfenzhiyi", - "shiwanfenzhiyi", - "wanfenzhiyi", - "wanfenzhiwu", - "qianfenzhiyi", - "qianfenzhier", - "qianfenzhisi", - "qianfenzhiwu", - "baifenzhiyi", - "wuqiong", - "xiaozhiyu", - "RMSE", - "EB", - "Max_rel_error", - "Mean_rel_error" + "余弦相似度", + "最大绝对误差", + "双百指标", + "双千指标", + "双万指标", + "错误率", + "均方根误差", + "误差均衡性", + "相对误差最大值", + "相对误差平均值", + "Status", + "Message" ]] precision_configs = { torch.float16 : { - 'error_distribution' : [ - ('qianfenzhiyi', 0, 1e-3), - ('qianfenzhier', 1e-3, 2e-3), - ('qianfenzhiwu', 2e-3, 5e-3), - ('baifenzhiyi', 5e-3, 1e-2), - ('wuqiong', 1e-2, np.inf) - ], 'small_value' : [ 1e-3 ], @@ -45,10 +30,6 @@ precision_configs = { ] }, torch.bfloat16: { - 'error_distribution' : [ - ('qianfenzhisi', 0, 4e-3), - ('wuqiong', 4e-3, np.inf) - ], 'small_value' : [ 1e-3 ], @@ -57,13 +38,6 @@ precision_configs = { ] }, torch.float32:{ - 'error_distribution' : [ - ('baiwanfenzhiyi', 0, 1e-6), - ('shiwanfenzhiyi', 1e-6, 1e-5), - ('wanfenzhiyi', 1e-5, 1e-4), - ('wanfenzhiwu', 1e-4, 5e-4), - ('wuqiong', 5e-4, np.inf) - ], 'small_value' : [ 1e-6 ], @@ -73,6 +47,7 @@ precision_configs = { } } + class CompareConst: NAN = np.nan NA = "N/A" @@ -89,31 +64,23 @@ class CompareColumn: self.bench_type = CompareConst.NA self.npu_type = CompareConst.NA self.shape = CompareConst.NA - self.dropout = CompareConst.NA + self.cosine_sim = CompareConst.NA + self.max_abs_err = CompareConst.NA + self.rel_err_hundredth = CompareConst.NA + self.rel_err_thousandth = CompareConst.NA + self.rel_err_ten_thousandth = CompareConst.NA + self.error_rate = CompareConst.NA self.builtin = CompareConst.NA self.inf_or_nan = CompareConst.NA - self.jueduiyuzhi = CompareConst.NA - self.baiwanfenzhiyi = CompareConst.NA - self.shiwanfenzhiyi = CompareConst.NA - self.wanfenzhiyi = CompareConst.NA - self.wanfenzhiwu = CompareConst.NA - self.qianfenzhiyi = CompareConst.NA - self.qianfenzhier = CompareConst.NA - self.qianfenzhisi = CompareConst.NA - self.qianfenzhiwu = CompareConst.NA - self.baifenzhiyi = CompareConst.NA - self.wuqiong = CompareConst.NA - self.xiaozhiyu = CompareConst.NA self.RMSE = CompareConst.NA self.EB = CompareConst.NA self.Max_rel_error = CompareConst.NA self.Mean_rel_error = CompareConst.NA - def to_column_value(self): - return [self.bench_type, self.npu_type, self.shape, self.dropout, self.builtin, self.inf_or_nan, - self.jueduiyuzhi, self.baiwanfenzhiyi, self.shiwanfenzhiyi, self.wanfenzhiyi, self.wanfenzhiwu, - self.qianfenzhiyi, self.qianfenzhier, self.qianfenzhisi, self.qianfenzhiwu, self.baifenzhiyi, - self.wuqiong, self.xiaozhiyu, self.RMSE, self.EB, self.Max_rel_error, self.Mean_rel_error] + def to_column_value(self, is_pass, message): + return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, + self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.RMSE, self.EB, + self.Max_rel_error, self.Mean_rel_error, is_pass, message] def check_dtype_comparable(x, y): -- Gitee From e8aad333ed32630d832df6647a17f30faea90321 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 11:58:43 +0000 Subject: [PATCH 06/29] =?UTF-8?q?=E6=96=B0=E5=A2=9Eencoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index e0c893c06b..3de38addd6 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -203,7 +203,7 @@ def read_json(file): def write_csv(data, filepath): - with FileOpen(filepath, 'a') as f: + with FileOpen(filepath, 'a', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) -- Gitee From 7c55acd72af67593d34967fe829477f1356f1e6d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 12:07:17 +0000 Subject: [PATCH 07/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/algorithm.py | 143 +++++++----------- 1 file changed, 56 insertions(+), 87 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index eebdc147d7..f1be49a178 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -5,32 +5,45 @@ from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype from api_accuracy_checker.common.utils import Const -def get_rel_err1(abs_err, b_value): - rel_err = np.abs(abs_err / b_value) - return rel_err - - -def get_max_abs_err(abs_err): - max_abs_err = abs_err.max() - bool_result = max_abs_err < 0.001 - return max_abs_err, bool_result - - -def get_rel_err_ratio(rel_err, thresholding): - if np.size(rel_err) == 0: - ratio = 1 +#cos +def cosine_sim(cpu_output, npu_output): + msg = "" + n_value = npu_output.reshape(-1) + b_value = cpu_output.reshape(-1) + cos = CompareConst.NA + np.seterr(divide="ignore", invalid="ignore") + if n_value.shape != b_value.shape: + msg = f"Shape of npu and bench outputs don't match. NPU: {n_value.shape}, bench: {b_value.shape}." + return -1, False, msg + if len(n_value) == 1: + msg = "All the data in npu dump data is scalar. Please refer to other compare algorithms." + return cos, True, msg + n_value_max = np.max(np.abs(n_value)) + b_value_max = np.max(np.abs(b_value)) + if n_value_max <= np.finfo(float).eps and b_value_max <= np.finfo(float).eps: + return cos, True, msg + elif n_value_max <= np.finfo(float).eps: + msg = "All the data is zero in npu dump data." + return CompareConst.NA, False, msg + elif b_value_max <= np.finfo(float).eps: + msg = "All the data is zero in bench dump data." + return CompareConst.NA, False, msg else: - ratio = np.divide(np.sum(rel_err < thresholding), np.size(rel_err)) - bool_result = ratio > (1 - thresholding) - return ratio, bool_result - - -def max_rel_err_standard(max_rel_errs): - bool_result = np.array(max_rel_errs) < 0.001 - return np.all(bool_result), bool_result - - + n_value = n_value_max.astype(float) / n_value_max + b_value = b_value_max.astype(float) / b_value_max + cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) + if np.isnan(cos): + msg = "Dump data has NaN when comparing with Cosine Similarity." + return cos, cos > 0.99, msg +#rmse +def get_rmse(abs_err, inf_nan_mask): + masked_ae = np.where(inf_nan_mask, 0, abs_err) + rmse = np.mean(np.square(masked_ae)) + inf_nan_cnt = np.sum(inf_nan_mask) + rmse = rmse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) + rmse = np.sqrt(rmse) + return rmse #误差均衡性 @@ -41,21 +54,6 @@ def get_error_balance(bench_data, device_data): error_balance = abs(larger_count - smaller_count) / total_count return error_balance -def get_big_value(bench_data, device_data): - small_value = 0.001 - bench_big_value = bench_data.copy() - bench_big_value[bench_data < small_value] = 1 - device_big_value = device_data.copy() - device_big_value[bench_data < small_value] = 1 - return bench_big_value, device_big_value - -def get_small_value(bench_data, device_data): - small_value = 0.001 - bench_small_value = bench_data.copy() - bench_small_value[bench_data > small_value] = 1 - device_small_value = device_data.copy() - device_small_value[bench_data > small_value] = 1 - return bench_small_value, device_small_value def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask): rel_err_tmp = abs_err / abs_bench_with_eps @@ -63,10 +61,23 @@ def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask): rel_err = np.where(rel_err_mask, -1, rel_err_tmp) return rel_err + def get_abs_err(bench_data, device_data): abs_err = np.abs(device_data - bench_data) return abs_err + +def get_rel_err_origin(abs_err, b_value): + rel_err_origin = np.abs(abs_err / b_value) + return rel_err_origin + + +def get_max_abs_err(abs_err): + max_abs_err = abs_err.max() + bool_result = max_abs_err < 0.001 + return max_abs_err, bool_result + + #相对误差最大值 def get_max_rel_err(rel_err): return np.max(rel_err) @@ -75,53 +86,11 @@ def get_max_rel_err(rel_err): def get_mean_rel_err(rel_err): return np.mean(rel_err) -#绝对阈值法 -def get_absolute_threshold(rel_err, abs_err): - rtol = 0 - etol = 0 - small_value = 0.001 - small_value_atol = 1e-9 + +def get_rel_err_ratio(rel_err, thresholding): if np.size(rel_err) == 0: - rel_err_num = 0 - else: - rel_err_num = np.sum(rel_err > rtol) - if np.size(abs_err) == 0: - abs_err_num = 0 + ratio = 1 else: - abs_err_num = np.sum(abs_err > small_value_atol) - return rel_err_num + abs_err_num - -#rmse -def get_rmse(abs_err, inf_nan_mask): - masked_ae = np.where(inf_nan_mask, 0, abs_err) - rmse = np.mean(np.square(masked_ae)) - inf_nan_cnt = np.sum(inf_nan_mask) - rmse = rmse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) - rmse = np.sqrt(rmse) - return rmse - -#小值域 -def get_small_value_error(small_value_mask, abs_err_greater_mask): - err_mask = np.logical_and(small_value_mask, abs_err_greater_mask) - small_value_err_num = np.sum(err_mask) - return small_value_err_num - -def get_distribution_ratio(rel_err, start, end): - - return np.sum((rel_err>start)&(rel_err (1 - thresholding) + return ratio, bool_result -- Gitee From e751921f695a982c5cb7bff2da6e02c12d7ebb4d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 12:09:00 +0000 Subject: [PATCH 08/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 79 ++++++++++++------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 6ff398d252..051d8348ba 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -10,7 +10,8 @@ from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ detail_test_rows, XlsxSheetWriter, precision_configs from api_accuracy_checker.compare.algorithm import get_absolute_threshold, get_distribution_ratio, get_rmse, \ - get_small_value_error, get_error_balance, get_max_rel_err, get_mean_rel_err, get_rel_err, get_abs_err + get_small_value_error, get_error_balance, get_max_rel_err, get_mean_rel_err, get_rel_err, get_abs_err, \ + get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -166,12 +167,12 @@ class Comparator: test_final_success = True status, compare_result, message = self._compare_core(bench_output, device_output) if not isinstance(status, list): - detailed_result_total.append(compare_result.to_column_value()) + detailed_result_total.append(compare_result.to_column_value(status, message)) if status in [CompareConst.ERROR, CompareConst.WARNING]: test_final_success = False else: for item, item_status in enumerate(status): - detailed_result_total.append(compare_result[item].to_column_value()) + detailed_result_total.append(compare_result[item].to_column_value(item_status, message[item])) if item_status in [CompareConst.ERROR, CompareConst.WARNING]: test_final_success = False return test_final_success, detailed_result_total @@ -218,7 +219,7 @@ class Comparator: compare_column.bench_dtype = str(type(bench_output)) compare_column.npu_dtype = str(type(device_output)) compare_column.shape = str(type(device_output)) - compare_result = self._compare_builtin_type(bench_output, device_output, compare_column) + status, compare_result, message = self._compare_builtin_type(bench_output, device_output, compare_column) elif bench_output is None: return CompareConst.PASS, compare_column, "Output is None." else: @@ -251,16 +252,18 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - compare_column = self._compare_float_tensor(bench_output, device_output, compare_column, npu_dtype) + status, compare_column, message = self._compare_float_tensor(bench_output, device_output, compare_column, npu_dtype) + return status, compare_column, message - return CompareConst.PASS, compare_column, message @staticmethod def _compare_builtin_type(bench_output, device_output, compare_column): + if not isinstance(bench_output, (bool, int, float, str)): + return CompareConst.PASS, compare_column, "" if bench_output != device_output: - compare_column.builtin = 0 - compare_column.builtin = 1 - return compare_column + return CompareConst.ERROR, compare_column, "" + compare_column.error_rate = 0 + return CompareConst.PASS, compare_column, "" @staticmethod def _flatten_compare_result(result): @@ -276,13 +279,14 @@ class Comparator: def _compare_bool_tensor(bench_output, device_output): error_nums = (bench_output != device_output).sum() if bench_output.size == 0: - return CompareConst.NAN, CompareConst.ERROR, "There is not cpu calculation result." + return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result." error_rate = float(error_nums / bench_output.size) result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR return error_rate, result, "" @staticmethod def _compare_float_tensor(bench_output, device_output, compare_column, dtype): + message = "" dtype_config = precision_configs.get(dtype) eps = np.finfo(bench_output.dtype).eps abs_bench = np.abs(bench_output) @@ -291,28 +295,12 @@ class Comparator: bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) inf_nan_mask = np.logical_not(both_finite_mask) - #inf/nan处理 - if np.sum(inf_nan_mask) == 0: - compare_column.inf_or_nan = 0 - else: - compare_column.inf_or_nan = np.sum(inf_nan_mask) - compare_column.inf_or_nan = check_inf_nan_value(bench_output, device_output, inf_nan_mask, abs_bench_with_eps) - + abs_bench_with_eps) #小值域 abs_err = get_abs_err(bench_output, device_output) small_value_mask = np.less_equal(np.abs(bench_output), dtype_config['small_value'][0]) small_value_mask = np.logical_and(small_value_mask, both_finite_mask) - abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) - - compare_column.xiaozhiyu = get_small_value_error(small_value_mask, abs_err_greater_mask) - rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) - - #误差分布 - for attr, start, end in dtype_config['error_distribution']: - setattr(compare_column, attr, get_distribution_ratio(rel_err, start, end)) - - compare_column.jueduiyuzhi = get_absolute_threshold(rel_err, abs_err) compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) @@ -320,4 +308,39 @@ class Comparator: compare_column.Max_rel_error = get_max_rel_err(rel_err) compare_column.Mean_rel_error = get_mean_rel_err(rel_err) - return compare_column + #cos + cos_res, cos_status, msg = cosine_sim(bench_output, device_output) + compare_column.cosine_sim = cos_res + message += msg + "\n" + if not cos_status: + return CompareConst.ERROR, compare_column, message + + #max abs err + max_abs_res, max_abs_status = get_max_abs_err(abs_err) + compare_column.max_abs_err = max_abs_res + if max_abs_status: + return CompareConst.PASS, compare_column, message + + #rel error + rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) + if dtype in [torch.float16, torch.bfloat16]: + hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) + compare_column.rel_err_hundredth = hundred_res + if not hundred_status: + return CompareConst.ERROR, compare_column, message + thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, 0.001) + compare_column.rel_err_thousandth = thousand_res + if dtype in [torch.float16, torch.bfloat16]: + if thousand_status: + return CompareConst.PASS, compare_column, message + return CompareConst.WARNING, compare_column, message + ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, 0.0001) + compare_column.rel_err_ten_thousandth = ten_thousand_res + if dtype in [torch.float32, torch.float64]: + if not thousand_status: + return CompareConst.ERROR, compare_column, message + if not ten_thousand_status: + return CompareConst.WARNING, compare_column, message + + + return CompareConst.PASS, compare_column, message -- Gitee From 567c39db27e8f699b7e753832088752bfba4fbc9 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 12:17:14 +0000 Subject: [PATCH 09/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 051d8348ba..fd83fc0d37 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -295,7 +295,7 @@ class Comparator: bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) inf_nan_mask = np.logical_not(both_finite_mask) - abs_bench_with_eps) + #小值域 abs_err = get_abs_err(bench_output, device_output) small_value_mask = np.less_equal(np.abs(bench_output), dtype_config['small_value'][0]) -- Gitee From a222c316684111fd7e838058839945b3f482d484 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 12:18:04 +0000 Subject: [PATCH 10/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index fd83fc0d37..ee77d39d6b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -8,7 +8,7 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ - detail_test_rows, XlsxSheetWriter, precision_configs + detail_test_rows, precision_configs from api_accuracy_checker.compare.algorithm import get_absolute_threshold, get_distribution_ratio, get_rmse, \ get_small_value_error, get_error_balance, get_max_rel_err, get_mean_rel_err, get_rel_err, get_abs_err, \ get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin -- Gitee From 3914475b3b57bd85da0296b1dd6f8d2d99cad38d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 29 Jan 2024 12:21:47 +0000 Subject: [PATCH 11/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index ee77d39d6b..12f558f7c6 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -9,9 +9,8 @@ from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ detail_test_rows, precision_configs -from api_accuracy_checker.compare.algorithm import get_absolute_threshold, get_distribution_ratio, get_rmse, \ - get_small_value_error, get_error_balance, get_max_rel_err, get_mean_rel_err, get_rel_err, get_abs_err, \ - get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin +from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ + get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen -- Gitee From 033e218806a8bf43e59a6b05a90093a2ad55c29d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 30 Jan 2024 03:38:18 +0000 Subject: [PATCH 12/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 7dedfc436d..9dbf9294a2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -11,8 +11,9 @@ detail_test_rows = [[ "双千指标", "双万指标", "错误率", - "均方根误差", "误差均衡性", + "均方根误差", + "小值域错误占比", "相对误差最大值", "相对误差平均值", "Status", @@ -70,17 +71,16 @@ class CompareColumn: self.rel_err_thousandth = CompareConst.NA self.rel_err_ten_thousandth = CompareConst.NA self.error_rate = CompareConst.NA - self.builtin = CompareConst.NA - self.inf_or_nan = CompareConst.NA - self.RMSE = CompareConst.NA self.EB = CompareConst.NA + self.RMSE = CompareConst.NA + self.small_value_err_ratio = CompareConst.NA self.Max_rel_error = CompareConst.NA self.Mean_rel_error = CompareConst.NA def to_column_value(self, is_pass, message): return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, - self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.RMSE, self.EB, - self.Max_rel_error, self.Mean_rel_error, is_pass, message] + self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, + self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] def check_dtype_comparable(x, y): -- Gitee From b9ce4383845b3b5d2db4a3b35a2d761afe9df853 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 30 Jan 2024 03:39:03 +0000 Subject: [PATCH 13/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../accuracy_tools/api_accuracy_checker/compare/compare.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 12f558f7c6..0e2452e3a8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -10,7 +10,8 @@ from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ detail_test_rows, precision_configs from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ - get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin + get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ + get_small_value_err_ratio from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -299,6 +300,9 @@ class Comparator: abs_err = get_abs_err(bench_output, device_output) small_value_mask = np.less_equal(np.abs(bench_output), dtype_config['small_value'][0]) small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) -- Gitee From a481f411e02fb377bfafba023c034a9dc754c9f2 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 30 Jan 2024 03:39:46 +0000 Subject: [PATCH 14/29] =?UTF-8?q?=E6=96=B0=E7=B2=BE=E5=BA=A6=E6=A0=87?= =?UTF-8?q?=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/algorithm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index f1be49a178..56c08a3b50 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -55,6 +55,13 @@ def get_error_balance(bench_data, device_data): return error_balance +def get_small_value_err_ratio(small_value_mask, abs_err_greater_mask): + err_mask = np.logical_and(small_value_mask, abs_err_greater_mask) + small_value_err_num = np.sum(err_mask) + small_value_num = np.sum(small_value_mask) + return 0 if small_value_num == 0 else small_value_err_num / small_value_num + + def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask): rel_err_tmp = abs_err / abs_bench_with_eps rel_err_mask = np.logical_or(small_value_mask, inf_nan_mask) -- Gitee From 20d7a7033a9a95a92cdf5f6d13bed54ee4a4e28f Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 30 Jan 2024 17:03:17 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/compare/algorithm.py | 3 ++ .../api_accuracy_checker/compare/compare.py | 30 +++++++++++-------- .../compare/compare_utils.py | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 56c08a3b50..f169d11405 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -36,6 +36,7 @@ def cosine_sim(cpu_output, npu_output): msg = "Dump data has NaN when comparing with Cosine Similarity." return cos, cos > 0.99, msg + #rmse def get_rmse(abs_err, inf_nan_mask): masked_ae = np.where(inf_nan_mask, 0, abs_err) @@ -55,6 +56,7 @@ def get_error_balance(bench_data, device_data): return error_balance +#小值域错误占比 def get_small_value_err_ratio(small_value_mask, abs_err_greater_mask): err_mask = np.logical_and(small_value_mask, abs_err_greater_mask) small_value_err_num = np.sum(err_mask) @@ -89,6 +91,7 @@ def get_max_abs_err(abs_err): def get_max_rel_err(rel_err): return np.max(rel_err) + #相对误差均值 def get_mean_rel_err(rel_err): return np.mean(rel_err) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 0e2452e3a8..2ce5f9bd4b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -8,7 +8,7 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ - detail_test_rows, precision_configs + DETAIL_TEST_ROWS, precision_configs from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio @@ -99,10 +99,11 @@ class Comparator: self.test_result_cnt['forward_or_backward_fail_num'] += 1 def write_csv_title(self): - summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]] + summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, + self.COLUMN_BACKWARD_SUCCESS, "Message"]] write_csv(summary_test_rows, self.save_path) - write_csv(detail_test_rows, self.detail_save_path) + write_csv(DETAIL_TEST_ROWS, self.detail_save_path) def write_summary_csv(self, test_result): test_rows = [] @@ -128,12 +129,14 @@ class Comparator: if isinstance(fwd_result, list): for i, test_subject in enumerate(fwd_result): subject = subject_prefix + ".forward.output." + str(i) - test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) if isinstance(item, float) else item for item in test_subject] + test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) + if isinstance(item, float) else item for item in test_subject] test_rows.append([subject] + list(test_subject)) if isinstance(bwd_result, list): for i, test_subject in enumerate(bwd_result): subject = subject_prefix + ".backward.output." + str(i) - test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) if isinstance(item, float) else item for item in test_subject] + test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) + if isinstance(item, float) else item for item in test_subject] test_rows.append([subject] + list(test_subject)) write_csv(test_rows, self.detail_save_path) @@ -177,7 +180,6 @@ class Comparator: test_final_success = False return test_final_success, detailed_result_total - def _compare_dropout(self, bench_output, device_output): tensor_num = bench_output.numel() if tensor_num >= 100: @@ -206,7 +208,8 @@ class Comparator: if b_keys != n_keys: return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." else: - status, compare_result, message = self._compare_core(list(bench_output.values()), list(device_output.values())) + status, compare_result, message = self._compare_core(list(bench_output.values()), + list(device_output.values())) elif isinstance(bench_output, torch.Tensor): copy_bench_out = bench_output.detach().clone() copy_device_output = device_output.detach().clone() @@ -223,11 +226,11 @@ class Comparator: elif bench_output is None: return CompareConst.PASS, compare_column, "Output is None." else: - return CompareConst.PASS, compare_column, "Unexpected output type in compare_core: {}".format(type(bench_output)) + return CompareConst.PASS, compare_column, + "Unexpected output type in compare_core: {}".format(type(bench_output)) return status, compare_result, message - def _compare_torch_tensor(self, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape @@ -244,7 +247,8 @@ class Comparator: return CompareConst.ERROR, compare_column, f"Bench out dtype is {bench_output.dtype} but " \ f"npu output dtype is {device_output.dtype}, cannot compare." message = "" - if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, np.int64, np.uint64]: + if bench_output.dtype in [bool, np.uint8, np.int8, np.int16, np.uint16, np.uint32, np.int32, + np.int64, np.uint64]: message += f"Compare algorithm cosine_sim is not supported for {bench_output.dtype} data. " \ f"Only judged by Error Rate." err_rate, status, msg = self._compare_bool_tensor(bench_output, device_output) @@ -252,10 +256,10 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - status, compare_column, message = self._compare_float_tensor(bench_output, device_output, compare_column, npu_dtype) + status, compare_column, message = self._compare_float_tensor(bench_output, device_output, + compare_column, npu_dtype) return status, compare_column, message - @staticmethod def _compare_builtin_type(bench_output, device_output, compare_column): if not isinstance(bench_output, (bool, int, float, str)): @@ -305,8 +309,8 @@ class Comparator: rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + #rmse、eb、max_rel_err、mean_rel_err compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) - compare_column.EB = get_error_balance(bench_output, device_output) compare_column.Max_rel_error = get_max_rel_err(rel_err) compare_column.Mean_rel_error = get_mean_rel_err(rel_err) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 9dbf9294a2..bd87a283ed 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -3,7 +3,7 @@ import torch from api_accuracy_checker.common.utils import Const, print_warn_log -detail_test_rows = [[ +DETAIL_TEST_ROWS = [[ "API Name", "Bench Dtype", "NPU Dtype", "Shape", "余弦相似度", "最大绝对误差", -- Gitee From 0ddab8e6fe1276d9f6564b5d568719249d53ed89 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 30 Jan 2024 17:06:02 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 2ce5f9bd4b..f4f42be870 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -348,6 +348,5 @@ class Comparator: return CompareConst.ERROR, compare_column, message if not ten_thousand_status: return CompareConst.WARNING, compare_column, message - return CompareConst.PASS, compare_column, message -- Gitee From d8c27c6b6688e84f127ef4d13385e13bc9e9e140 Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 10:07:51 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/compare/algorithm.py | 6 +-- .../api_accuracy_checker/compare/compare.py | 48 +++++++++---------- .../compare/compare_utils.py | 5 +- .../run_ut/data_generate.py | 4 +- .../api_accuracy_checker/run_ut/run_ut.py | 8 ++-- .../ptdbg_ascend/common/file_check_util.py | 3 +- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index f169d11405..4a6ae1e413 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -29,8 +29,8 @@ def cosine_sim(cpu_output, npu_output): msg = "All the data is zero in bench dump data." return CompareConst.NA, False, msg else: - n_value = n_value_max.astype(float) / n_value_max - b_value = b_value_max.astype(float) / b_value_max + n_value = n_value.astype(float) / n_value + b_value = b_value.astype(float) / b_value cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) if np.isnan(cos): msg = "Dump data has NaN when comparing with Cosine Similarity." @@ -52,7 +52,7 @@ def get_error_balance(bench_data, device_data): larger_count = np.sum(np.greater(device_data - bench_data.astype(device_data.dtype), 0)) smaller_count = np.sum(np.less(device_data - bench_data.astype(device_data.dtype), 0)) total_count = bench_data.size - error_balance = abs(larger_count - smaller_count) / total_count + error_balance = abs(larger_count - smaller_count) / total_count if total_count > 0 else 0 return error_balance diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index f4f42be870..7254ea274b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -3,13 +3,12 @@ import os import csv import torch import numpy as np -import xlsxwriter from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ - DETAIL_TEST_ROWS, precision_configs -from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ + DETAIL_TEST_ROWS, precision_configs, Benchmark_Compare_Support_List +from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio from api_accuracy_checker.common.config import msCheckerConfig @@ -291,29 +290,30 @@ class Comparator: @staticmethod def _compare_float_tensor(bench_output, device_output, compare_column, dtype): message = "" - dtype_config = precision_configs.get(dtype) - eps = np.finfo(bench_output.dtype).eps - abs_bench = np.abs(bench_output) - abs_bench_with_eps = abs_bench + eps - device_finite_mask = np.isfinite(device_output) - bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) - both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) - inf_nan_mask = np.logical_not(both_finite_mask) + if str(dtype) in Benchmark_Compare_Support_List: + dtype_config = precision_configs.get(dtype) + eps = np.finfo(bench_output.dtype).eps + abs_bench = np.abs(bench_output) + abs_bench_with_eps = abs_bench + eps + device_finite_mask = np.isfinite(device_output) + bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) + both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) + inf_nan_mask = np.logical_not(both_finite_mask) - #小值域 - abs_err = get_abs_err(bench_output, device_output) - small_value_mask = np.less_equal(np.abs(bench_output), dtype_config['small_value'][0]) - small_value_mask = np.logical_and(small_value_mask, both_finite_mask) - abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) - compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) + #小值域 + abs_err = get_abs_err(bench_output, device_output) + small_value_mask = np.less_equal(abs_bench, dtype_config['small_value'][0]) + small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) - rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) - - #rmse、eb、max_rel_err、mean_rel_err - compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) - compare_column.EB = get_error_balance(bench_output, device_output) - compare_column.Max_rel_error = get_max_rel_err(rel_err) - compare_column.Mean_rel_error = get_mean_rel_err(rel_err) + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + + #rmse、eb、max_rel_err、mean_rel_err + compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) + compare_column.EB = get_error_balance(bench_output, device_output) + compare_column.Max_rel_error = get_max_rel_err(rel_err) + compare_column.Mean_rel_error = get_mean_rel_err(rel_err) #cos cos_res, cos_status, msg = cosine_sim(bench_output, device_output) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index bd87a283ed..24246dd9c5 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -4,7 +4,7 @@ from api_accuracy_checker.common.utils import Const, print_warn_log DETAIL_TEST_ROWS = [[ - "API Name", "Bench Dtype", "NPU Dtype", "Shape", + "API Name", "Bench Dtype", "DEVICE Dtype", "Shape", "余弦相似度", "最大绝对误差", "双百指标", @@ -49,6 +49,9 @@ precision_configs = { } +Benchmark_Compare_Support_List = ['torch.float16', 'torch.bfloat16', 'torch.float32'] + + class CompareConst: NAN = np.nan NA = "N/A" diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py index e9c25f7faf..65aa264ab2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py @@ -219,9 +219,7 @@ def gen_kwargs(api_info, convert_type=None, real_data_path=None): def gen_torch_kwargs(kwargs_params, key, value): - if value.get('type') == "torch.device": - kwargs_params[key] = eval(value.get('type'))(value.get('value')) - else: + if value.get('type') != "torch.device": kwargs_params[key] = eval(value.get('value')) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 72ca05d49e..243afc5e51 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -17,13 +17,12 @@ import torch from tqdm import tqdm from api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ - print_error_log, check_file_or_directory_path, initialize_save_path, Const + print_error_log, initialize_save_path, Const from api_accuracy_checker.compare.compare import Comparator from api_accuracy_checker.hook_module.wrap_tensor import TensorOPTemplate from api_accuracy_checker.hook_module.wrap_functional import FunctionalOPTemplate from api_accuracy_checker.hook_module.wrap_torch import TorchOPTemplate from api_accuracy_checker.common.config import msCheckerConfig -from api_accuracy_checker.compare.compare_utils import CompareConst from api_accuracy_checker.dump.api_info import APIInfo @@ -174,7 +173,10 @@ def run_ut(config): print_error_log(f"Run {api_full_name} UT Error: %s" % str(err)) compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err))) finally: - torch.npu.empty_cache() + if is_gpu: + torch.cuda.empty_cache() + else: + torch.npu.empty_cache() change_mode(compare.save_path, FileCheckConst.DATA_FILE_AUTHORITY) change_mode(compare.detail_save_path, FileCheckConst.DATA_FILE_AUTHORITY) compare.print_pretest_result() diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py index ca34fc4164..7f08303931 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/file_check_util.py @@ -147,7 +147,8 @@ class FileOpen: def __enter__(self): self.check_file_path() - if "b" not in self.mode: + binary_mode = "b" + if binary_mode not in self.mode: self._handle = open(self.file_path, self.mode, encoding=self.encoding) else: self._handle = open(self.file_path, self.mode) -- Gitee From de880363958d3fcbe5ea1f95d75b490b86c2a1e2 Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 10:12:05 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/compare/compare.py | 5 ++-- .../compare/compare_column.py | 24 +++++++++++++++++++ .../compare/compare_utils.py | 23 ------------------ 3 files changed, 27 insertions(+), 25 deletions(-) create mode 100644 debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 7254ea274b..05cfd5a3d2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -6,8 +6,9 @@ import numpy as np from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv -from api_accuracy_checker.compare.compare_utils import CompareConst, CompareColumn, check_dtype_comparable, \ - DETAIL_TEST_ROWS, precision_configs, Benchmark_Compare_Support_List +from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ + precision_configs, Benchmark_Compare_Support_List +from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py new file mode 100644 index 0000000000..9579f6a915 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -0,0 +1,24 @@ +from api_accuracy_checker.compare.compare_utils import CompareConst + + +class CompareColumn: + def __init__(self): + self.bench_type = CompareConst.NA + self.npu_type = CompareConst.NA + self.shape = CompareConst.NA + self.cosine_sim = CompareConst.NA + self.max_abs_err = CompareConst.NA + self.rel_err_hundredth = CompareConst.NA + self.rel_err_thousandth = CompareConst.NA + self.rel_err_ten_thousandth = CompareConst.NA + self.error_rate = CompareConst.NA + self.EB = CompareConst.NA + self.RMSE = CompareConst.NA + self.small_value_err_ratio = CompareConst.NA + self.Max_rel_error = CompareConst.NA + self.Mean_rel_error = CompareConst.NA + + def to_column_value(self, is_pass, message): + return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, + self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, + self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 24246dd9c5..6982569244 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -63,29 +63,6 @@ class CompareConst: FALSE = 'FALSE' -class CompareColumn: - def __init__(self): - self.bench_type = CompareConst.NA - self.npu_type = CompareConst.NA - self.shape = CompareConst.NA - self.cosine_sim = CompareConst.NA - self.max_abs_err = CompareConst.NA - self.rel_err_hundredth = CompareConst.NA - self.rel_err_thousandth = CompareConst.NA - self.rel_err_ten_thousandth = CompareConst.NA - self.error_rate = CompareConst.NA - self.EB = CompareConst.NA - self.RMSE = CompareConst.NA - self.small_value_err_ratio = CompareConst.NA - self.Max_rel_error = CompareConst.NA - self.Mean_rel_error = CompareConst.NA - - def to_column_value(self, is_pass, message): - return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, - self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, - self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] - - def check_dtype_comparable(x, y): if x.dtype in Const.FLOAT_TYPE: if y.dtype in Const.FLOAT_TYPE: -- Gitee From 621e0e403b76447f2f81016538b548d80929bc41 Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 11:38:48 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/compare/compare.py | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 05cfd5a3d2..f9a3db0bc3 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -168,7 +168,19 @@ class Comparator: def _compare_core_wrapper(self, bench_output, device_output): detailed_result_total = [] test_final_success = True - status, compare_result, message = self._compare_core(bench_output, device_output) + if isinstance(bench_output, (list, tuple)): + status, compare_result, message = [], [], [] + if len(bench_output) != len(device_output): + status = CompareConst.ERROR + message = "bench and npu output structure is different." + else: + for b_out_i, n_out_i in zip(bench_output, device_output): + status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) + status.append(status_i) + compare_result.append(compare_result_i) + message.append(message_i) + else: + status, compare_result, message = self._compare_core(bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) if status in [CompareConst.ERROR, CompareConst.WARNING]: @@ -194,15 +206,6 @@ class Comparator: compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): return CompareConst.ERROR, compare_column, "bench and npu output type is different." - if isinstance(bench_output, (list, tuple)): - status, compare_result, message = [], [], [] - if len(bench_output) != len(device_output): - return CompareConst.ERROR, compare_column, "bench and npu output structure is different." - for b_out_i, n_out_i in zip(bench_output, device_output): - status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) - status.append(status_i) - compare_result.append(compare_result_i) - message.append(message_i) elif isinstance(bench_output, dict): b_keys, n_keys = set(bench_output.keys()), set(device_output.keys()) if b_keys != n_keys: @@ -219,10 +222,9 @@ class Comparator: status, compare_result, message = self._compare_torch_tensor(copy_bench_out, copy_device_output, compare_column) elif isinstance(bench_output, (bool, int, float, str)): - compare_column.bench_dtype = str(type(bench_output)) - compare_column.npu_dtype = str(type(device_output)) - compare_column.shape = str(type(device_output)) - status, compare_result, message = self._compare_builtin_type(bench_output, device_output, compare_column) + compare_column.bench_type = str(type(bench_output)) + compare_column.npu_type = str(type(device_output)) + status, compare_result, message = self._compare_builtin_type(bench_output, device_output, compare_column) elif bench_output is None: return CompareConst.PASS, compare_column, "Output is None." else: @@ -269,15 +271,6 @@ class Comparator: compare_column.error_rate = 0 return CompareConst.PASS, compare_column, "" - @staticmethod - def _flatten_compare_result(result): - flatten_result = [] - for result_i in result: - if isinstance(result_i, list): - flatten_result += flatten_compare_result(result_i) - else: - flatten_result.append(result_i) - return flatten_result @staticmethod def _compare_bool_tensor(bench_output, device_output): @@ -291,18 +284,18 @@ class Comparator: @staticmethod def _compare_float_tensor(bench_output, device_output, compare_column, dtype): message = "" + eps = np.finfo(bench_output.dtype).eps + abs_bench = np.abs(bench_output) + abs_bench_with_eps = abs_bench + eps + abs_err = get_abs_err(bench_output, device_output) if str(dtype) in Benchmark_Compare_Support_List: dtype_config = precision_configs.get(dtype) - eps = np.finfo(bench_output.dtype).eps - abs_bench = np.abs(bench_output) - abs_bench_with_eps = abs_bench + eps device_finite_mask = np.isfinite(device_output) bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) inf_nan_mask = np.logical_not(both_finite_mask) #小值域 - abs_err = get_abs_err(bench_output, device_output) small_value_mask = np.less_equal(abs_bench, dtype_config['small_value'][0]) small_value_mask = np.logical_and(small_value_mask, both_finite_mask) abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) -- Gitee From 076d9b9ce614e8045e62a4172ed78f5fb1395309 Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 15:36:38 +0800 Subject: [PATCH 20/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index f9a3db0bc3..d820f53ccf 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -171,8 +171,8 @@ class Comparator: if isinstance(bench_output, (list, tuple)): status, compare_result, message = [], [], [] if len(bench_output) != len(device_output): - status = CompareConst.ERROR - message = "bench and npu output structure is different." + status = [CompareConst.ERROR] + message = ["bench and npu output structure is different."] else: for b_out_i, n_out_i in zip(bench_output, device_output): status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) -- Gitee From 9a5bbc758bb1e9e8d53959c4147d25977f8aaba2 Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 17:34:20 +0800 Subject: [PATCH 21/29] fix suggestions --- .../api_accuracy_checker/compare/algorithm.py | 20 +++++++++++++++--- .../api_accuracy_checker/compare/compare.py | 21 ++++--------------- .../test/ut/compare/test_algorithm.py | 2 +- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 4a6ae1e413..30471889f7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -40,10 +40,10 @@ def cosine_sim(cpu_output, npu_output): #rmse def get_rmse(abs_err, inf_nan_mask): masked_ae = np.where(inf_nan_mask, 0, abs_err) - rmse = np.mean(np.square(masked_ae)) + mse = np.mean(np.square(masked_ae)) inf_nan_cnt = np.sum(inf_nan_mask) - rmse = rmse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) - rmse = np.sqrt(rmse) + mse = mse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) + rmse = np.sqrt(mse) return rmse @@ -104,3 +104,17 @@ def get_rel_err_ratio(rel_err, thresholding): ratio = np.divide(np.sum(rel_err < thresholding), np.size(rel_err)) bool_result = ratio > (1 - thresholding) return ratio, bool_result + + +def get_finite_and_infinite_mask(bench_output, device_output): + device_finite_mask = np.isfinite(device_output) + bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) + both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) + inf_nan_mask = np.logical_not(both_finite_mask) + return both_finite_mask, inf_nan_mask + + +def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold): + small_value_mask = np.less_equal(abs_bench, small_value_threshold) + small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + return small_value_mask diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index d820f53ccf..cf1cf0e157 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -11,7 +11,7 @@ from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ - get_small_value_err_ratio + get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -290,39 +290,27 @@ class Comparator: abs_err = get_abs_err(bench_output, device_output) if str(dtype) in Benchmark_Compare_Support_List: dtype_config = precision_configs.get(dtype) - device_finite_mask = np.isfinite(device_output) - bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) - both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) - inf_nan_mask = np.logical_not(both_finite_mask) - - #小值域 - small_value_mask = np.less_equal(abs_bench, dtype_config['small_value'][0]) - small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) - rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) - - #rmse、eb、max_rel_err、mean_rel_err compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) compare_column.EB = get_error_balance(bench_output, device_output) compare_column.Max_rel_error = get_max_rel_err(rel_err) compare_column.Mean_rel_error = get_mean_rel_err(rel_err) - #cos cos_res, cos_status, msg = cosine_sim(bench_output, device_output) compare_column.cosine_sim = cos_res message += msg + "\n" if not cos_status: return CompareConst.ERROR, compare_column, message - #max abs err max_abs_res, max_abs_status = get_max_abs_err(abs_err) compare_column.max_abs_err = max_abs_res if max_abs_status: return CompareConst.PASS, compare_column, message - - #rel error + rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) if dtype in [torch.float16, torch.bfloat16]: hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) @@ -342,5 +330,4 @@ class Comparator: return CompareConst.ERROR, compare_column, message if not ten_thousand_status: return CompareConst.WARNING, compare_column, message - return CompareConst.PASS, compare_column, message diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 12f954e615..36ce0ccdc0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -2,7 +2,7 @@ import unittest import numpy as np import torch from api_accuracy_checker.compare import algorithm as alg -from api_accuracy_checker.compare.algorithm import CompareColumn +from api_accuracy_checker.compare.compare_column import CompareColumn class TestAlgorithmMethods(unittest.TestCase): -- Gitee From 9eee269e75cb45dd8a3ebd5c386c4ab50c91530c Mon Sep 17 00:00:00 2001 From: gitee Date: Sun, 4 Feb 2024 17:36:36 +0800 Subject: [PATCH 22/29] fix --- .../api_accuracy_checker/compare/compare.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index cf1cf0e157..957de66ac4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -192,16 +192,6 @@ class Comparator: test_final_success = False return test_final_success, detailed_result_total - def _compare_dropout(self, bench_output, device_output): - tensor_num = bench_output.numel() - if tensor_num >= 100: - if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: - return True, 1 - else: - return False, 0 - else: - return True, 1 - def _compare_core(self, bench_output, device_output): compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): @@ -261,6 +251,17 @@ class Comparator: status, compare_column, message = self._compare_float_tensor(bench_output, device_output, compare_column, npu_dtype) return status, compare_column, message + + @staticmethod + def _compare_dropout(bench_output, device_output): + tensor_num = bench_output.numel() + if tensor_num >= 100: + if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: + return True, 1 + else: + return False, 0 + else: + return True, 1 @staticmethod def _compare_builtin_type(bench_output, device_output, compare_column): -- Gitee From 7e8529d618d64aa012d1347b0d7855447591781a Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 10:09:55 +0800 Subject: [PATCH 23/29] fix cos --- .../api_accuracy_checker/compare/algorithm.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 30471889f7..4da4bdcbe7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -6,31 +6,29 @@ from api_accuracy_checker.common.utils import Const #cos -def cosine_sim(cpu_output, npu_output): +def cosine_sim(bench_output, device_output): msg = "" - n_value = npu_output.reshape(-1) - b_value = cpu_output.reshape(-1) + n_value = device_output.reshape(-1) + b_value = bench_output.reshape(-1) cos = CompareConst.NA np.seterr(divide="ignore", invalid="ignore") if n_value.shape != b_value.shape: - msg = f"Shape of npu and bench outputs don't match. NPU: {n_value.shape}, bench: {b_value.shape}." + msg = f"Shape of device and bench outputs don't match. device: {n_value.shape}, bench: {b_value.shape}." return -1, False, msg if len(n_value) == 1: - msg = "All the data in npu dump data is scalar. Please refer to other compare algorithms." + msg = "All the data in device dump data is scalar. Please refer to other compare algorithms." return cos, True, msg n_value_max = np.max(np.abs(n_value)) b_value_max = np.max(np.abs(b_value)) if n_value_max <= np.finfo(float).eps and b_value_max <= np.finfo(float).eps: return cos, True, msg elif n_value_max <= np.finfo(float).eps: - msg = "All the data is zero in npu dump data." + msg = "All the data is zero in device dump data." return CompareConst.NA, False, msg elif b_value_max <= np.finfo(float).eps: msg = "All the data is zero in bench dump data." return CompareConst.NA, False, msg else: - n_value = n_value.astype(float) / n_value - b_value = b_value.astype(float) / b_value cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) if np.isnan(cos): msg = "Dump data has NaN when comparing with Cosine Similarity." -- Gitee From 126e666ac109a0cdf1ee53870b2b0a1b45e42556 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 10:38:58 +0800 Subject: [PATCH 24/29] fix ut --- .../test/ut/common/test_common_utils.py | 2 +- .../test/ut/compare/test_algorithm.py | 41 +++++-------------- .../test/ut/compare/test_compare.py | 8 ++-- .../test/ut/run_ut/test_data_generate.py | 11 ----- 4 files changed, 15 insertions(+), 47 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py index ea1465473c..956ef0bc17 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py @@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase): def test_write_csv(self): test_data = [["name", "age"], ["Alice", "20"], ["Bob", "30"]] write_csv(test_data, 'test.csv') - with open('test.csv', 'r') as f: + with open('test.csv', 'r', encoding='utf-8') as f: reader = csv.reader(f) for i, row in enumerate(reader): self.assertEqual(row, test_data[i]) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 36ce0ccdc0..98ea4b845e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -1,6 +1,7 @@ import unittest import numpy as np import torch +from api_accuracy_checker.compare import compare as cmp from api_accuracy_checker.compare import algorithm as alg from api_accuracy_checker.compare.compare_column import CompareColumn @@ -10,18 +11,13 @@ class TestAlgorithmMethods(unittest.TestCase): cpu_output = torch.Tensor([1.0, 2.0, 3.0]) npu_output = torch.Tensor([1.0, 2.0, 3.0]) compare_column = CompareColumn() - status, compare_column, message = alg.compare_torch_tensor(cpu_output, npu_output, compare_column) + status, compare_column, message = cmp.compare_torch_tensor(cpu_output, npu_output, compare_column) self.assertEqual(status, "pass") def test_compare_bool_tensor(self): cpu_output = np.array([True, False, True]) npu_output = np.array([True, False, True]) - self.assertEqual(alg.compare_bool_tensor(cpu_output, npu_output), (0.0, 'pass', '')) - - def test_get_msg_and_handle_value(self): - b_value = np.array([1.0, 2.0, 3.0]) - n_value = np.array([1.0, 2.0, 3.0]) - self.assertEqual(alg.get_msg_and_handle_value(b_value, n_value), (b_value, n_value)) + self.assertEqual(cmp.compare_bool_tensor(cpu_output, npu_output), (0.0, 'pass', '')) def test_get_max_abs_err(self): b_value = np.array([1.0, 2.0, 3.0]) @@ -33,45 +29,28 @@ class TestAlgorithmMethods(unittest.TestCase): b_value = np.array([1.0, 2.0, 3.0]) n_value = np.array([1.0, 2.0, 3.0]) abs_err = np.abs(b_value - n_value) - rel_err = alg.get_rel_err(abs_err, b_value) + rel_err = alg.get_rel_err_origin(abs_err, b_value) self.assertEqual(alg.get_rel_err_ratio(rel_err, 0.001), (1.0, True)) def test_get_rel_err_ratio_ten_thousandth(self): b_value = np.array([1.0, 2.0, 3.0]) n_value = np.array([1.0, 2.0, 3.0]) abs_err = np.abs(b_value - n_value) - rel_err = alg.get_rel_err(abs_err, b_value) + rel_err = alg.get_rel_err_origin(abs_err, b_value) self.assertEqual(alg.get_rel_err_ratio(rel_err, 0.0001), (1.0, True)) - def test_max_rel_err_standard(self): - max_rel_errs = [0.0001, 0.0002, 0.0003] - result, arr = alg.max_rel_err_standard(max_rel_errs) - self.assertEqual(result, True) - self.assertTrue((arr == np.array([True, True, True])).all()) - - def test_cosine_standard(self): - compare_result = [0.9999, 0.9999, 0.9999] - result, arr = alg.cosine_standard(compare_result) - self.assertEqual(result, True) - self.assertTrue((arr == np.array([True, True, True])).all()) - def test_cosine_sim(self): cpu_output = np.array([1.0, 2.0, 3.0]) npu_output = np.array([1.0, 2.0, 3.0]) self.assertEqual(alg.cosine_sim(cpu_output, npu_output), (1.0, True, '')) - - def test_compare_uint8_data(self): - b_value = np.array([1, 2, 3], dtype=np.uint8) - n_value = np.array([1, 2, 3], dtype=np.uint8) - self.assertEqual(alg.compare_uint8_data(b_value, n_value), (1, True)) + result, is_similar, msg = alg.cosine_sim(cpu_output, npu_output) + self.assertAlmostEqual(result, 1.0, places=14, msg=msg) + self.assertTrue(is_similar) + self.assertEqual(msg, '') def test_compare_builtin_type(self): compare_column = CompareColumn() bench_out = 1 npu_out = 1 - status, compare_result, message = alg.compare_builtin_type(bench_out, npu_out, compare_column) + status, compare_result, message = cmp.compare_builtin_type(bench_out, npu_out, compare_column) self.assertEqual((status, compare_result.error_rate, message), ('pass', 0, '')) - - def test_flatten_compare_result(self): - result = [[1, 2], [3, 4]] - self.assertEqual(alg.flatten_compare_result(result), [1, 2, 3, 4]) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 6dc5cca029..9a2f6db448 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -38,15 +38,15 @@ class TestCompare(unittest.TestCase): test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) self.assertTrue(test_final_success) self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), 1.0, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 'pass', '\n']]) + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) bench_out, npu_out = [dummy_input, dummy_input], [dummy_input, dummy_input] test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) self.assertTrue(test_final_success) self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), 1.0, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 'pass', '\n'], ['torch.float32', 'torch.float32', - (100, 100), 1.0, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 'pass', '\n']]) + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n'], + ['torch.float32', 'torch.float32',(100, 100), 1.0, 0.0, 'N/A', 'N/A', + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0,'pass', '\n']]) def test_compare_output(self): bench_out, npu_out = torch.randn(100, 100), torch.randn(100, 100) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py index 50f9131e47..3f79ecf17b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py @@ -54,17 +54,6 @@ class TestDataGenerateMethods(unittest.TestCase): kwargs_params = gen_kwargs(api_info, None) self.assertEqual(kwargs_params, {'inplace': False}) - def test_gen_kwargs_device(self): - k_dict = {"kwargs": {"device": {"type": "torch.device", "value": "cpu"}}} - kwargs_params = gen_kwargs(k_dict, None) - self.assertEqual(str(kwargs_params), "{'device': device(type='cpu')}") - - def test_gen_kwargs_1(self): - k_dict = {"device": {"type": "torch.device", "value": "cpu"}} - for key, value in k_dict.items(): - gen_torch_kwargs(k_dict, key, value) - self.assertEqual(str(k_dict), "{'device': device(type='cpu')}") - def test_gen_kwargs_2(self): k_dict = {"inplace": {"type": "bool", "value": "False"}} for key, value in k_dict.items(): -- Gitee From 4939f5f4c45d9338f7c51a0eb44fb045d7371743 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 18:48:34 +0800 Subject: [PATCH 25/29] fix ut --- .../api_accuracy_checker/compare/algorithm.py | 3 +++ .../test/ut/common/test_common_utils.py | 2 +- .../test/ut/compare/test_algorithm.py | 24 ------------------- .../test/ut/compare/test_compare.py | 21 ++++++++++++++++ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 4da4bdcbe7..394ea9cf0e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -29,9 +29,12 @@ def cosine_sim(bench_output, device_output): msg = "All the data is zero in bench dump data." return CompareConst.NA, False, msg else: + n_value = n_value.astype(float) / n_value_max + b_value = b_value.astype(float) / b_value_max cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) if np.isnan(cos): msg = "Dump data has NaN when comparing with Cosine Similarity." + cos = np.clip(cos, -1, 1) return cos, cos > 0.99, msg diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py index 956ef0bc17..5f25e81c09 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/common/test_common_utils.py @@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase): def test_write_csv(self): test_data = [["name", "age"], ["Alice", "20"], ["Bob", "30"]] write_csv(test_data, 'test.csv') - with open('test.csv', 'r', encoding='utf-8') as f: + with open('test.csv', 'r', encoding='utf-8-sig') as f: reader = csv.reader(f) for i, row in enumerate(reader): self.assertEqual(row, test_data[i]) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py index 98ea4b845e..90e18d166f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_algorithm.py @@ -3,22 +3,9 @@ import numpy as np import torch from api_accuracy_checker.compare import compare as cmp from api_accuracy_checker.compare import algorithm as alg -from api_accuracy_checker.compare.compare_column import CompareColumn class TestAlgorithmMethods(unittest.TestCase): - def test_compare_torch_tensor(self): - cpu_output = torch.Tensor([1.0, 2.0, 3.0]) - npu_output = torch.Tensor([1.0, 2.0, 3.0]) - compare_column = CompareColumn() - status, compare_column, message = cmp.compare_torch_tensor(cpu_output, npu_output, compare_column) - self.assertEqual(status, "pass") - - def test_compare_bool_tensor(self): - cpu_output = np.array([True, False, True]) - npu_output = np.array([True, False, True]) - self.assertEqual(cmp.compare_bool_tensor(cpu_output, npu_output), (0.0, 'pass', '')) - def test_get_max_abs_err(self): b_value = np.array([1.0, 2.0, 3.0]) n_value = np.array([1.0, 2.0, 3.0]) @@ -43,14 +30,3 @@ class TestAlgorithmMethods(unittest.TestCase): cpu_output = np.array([1.0, 2.0, 3.0]) npu_output = np.array([1.0, 2.0, 3.0]) self.assertEqual(alg.cosine_sim(cpu_output, npu_output), (1.0, True, '')) - result, is_similar, msg = alg.cosine_sim(cpu_output, npu_output) - self.assertAlmostEqual(result, 1.0, places=14, msg=msg) - self.assertTrue(is_similar) - self.assertEqual(msg, '') - - def test_compare_builtin_type(self): - compare_column = CompareColumn() - bench_out = 1 - npu_out = 1 - status, compare_result, message = cmp.compare_builtin_type(bench_out, npu_out, compare_column) - self.assertEqual((status, compare_result.error_rate, message), ('pass', 0, '')) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 9a2f6db448..a92c7bac20 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -2,10 +2,12 @@ import csv import os import shutil import time +import numpy as np import unittest import torch.nn.functional from api_accuracy_checker.compare.compare import Comparator +from api_accuracy_checker.compare.compare_column import CompareColumn current_time = time.strftime("%Y%m%d%H%M%S") RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" @@ -72,3 +74,22 @@ class TestCompare(unittest.TestCase): next(csv_reader) api_name_list = [row[0] for row in csv_reader] self.assertEqual(api_name_list[0], 'Functional*conv2d*0.forward.output.0') + + def test_compare_torch_tensor(self): + cpu_output = torch.Tensor([1.0, 2.0, 3.0]) + npu_output = torch.Tensor([1.0, 2.0, 3.0]) + compare_column = CompareColumn() + status, compare_column, message = self.compare._compare_torch_tensor(cpu_output, npu_output, compare_column) + self.assertEqual(status, "pass") + + def test_compare_bool_tensor(self): + cpu_output = np.array([True, False, True]) + npu_output = np.array([True, False, True]) + self.assertEqual(self.compare._compare_bool_tensor(cpu_output, npu_output), (0.0, 'pass', '')) + + def test_compare_builtin_type(self): + compare_column = CompareColumn() + bench_out = 1 + npu_out = 1 + status, compare_result, message = self.compare._compare_builtin_type(bench_out, npu_out, compare_column) + self.assertEqual((status, compare_result.error_rate, message), ('pass', 0, '')) -- Gitee From 614fdf015e5e6de04d6d69e54f55e18096ac3ead Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 19:34:08 +0800 Subject: [PATCH 26/29] fix ut --- .../test/ut/compare/test_compare.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index a92c7bac20..401b43dcbd 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -38,16 +38,25 @@ class TestCompare(unittest.TestCase): dummy_input = torch.randn(100, 100) bench_out, npu_out = dummy_input, dummy_input test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) + actual_cosine_similarity = detailed_result_total[0][4] + # 设置一个小的公差值 + tolerance = 1e-4 + # 判断实际的余弦相似度值是否在预期值的公差范围内 + self.assertTrue(np.isclose(actual_cosine_similarity, 1.0, atol=tolerance)) + # 对其他值进行比较，确保它们符合预期 + self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), + actual_cosine_similarity, 0.0, 'N/A', 'N/A', + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) self.assertTrue(test_final_success) - self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), 1.0, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) bench_out, npu_out = [dummy_input, dummy_input], [dummy_input, dummy_input] test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) self.assertTrue(test_final_success) - self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), 1.0, 0.0, 'N/A', 'N/A', + self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), + actual_cosine_similarity, 0.0, 'N/A', 'N/A', 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n'], - ['torch.float32', 'torch.float32',(100, 100), 1.0, 0.0, 'N/A', 'N/A', + ['torch.float32', 'torch.float32',(100, 100), + actual_cosine_similarity, 0.0, 'N/A', 'N/A', 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0,'pass', '\n']]) def test_compare_output(self): -- Gitee From d0c545b113186e353da494f3ecb2dfb4d7e1cddf Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 19:42:04 +0800 Subject: [PATCH 27/29] fix ut --- .../api_accuracy_checker/test/ut/compare/test_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 401b43dcbd..c12f5aff4c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -38,7 +38,7 @@ class TestCompare(unittest.TestCase): dummy_input = torch.randn(100, 100) bench_out, npu_out = dummy_input, dummy_input test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) - actual_cosine_similarity = detailed_result_total[0][4] + actual_cosine_similarity = detailed_result_total[0][3] # 设置一个小的公差值 tolerance = 1e-4 # 判断实际的余弦相似度值是否在预期值的公差范围内 -- Gitee From 7ea76311ef1230a362bbad8e893dffe0ffb46859 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 19:53:09 +0800 Subject: [PATCH 28/29] fix ut --- .../test/ut/compare/test_compare.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index c12f5aff4c..6940fc2c71 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -3,9 +3,10 @@ import os import shutil import time import numpy as np -import unittest import torch.nn.functional +import unittest + from api_accuracy_checker.compare.compare import Comparator from api_accuracy_checker.compare.compare_column import CompareColumn @@ -45,7 +46,7 @@ class TestCompare(unittest.TestCase): self.assertTrue(np.isclose(actual_cosine_similarity, 1.0, atol=tolerance)) # 对其他值进行比较，确保它们符合预期 self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), - actual_cosine_similarity, 0.0, 'N/A', 'N/A', + actual_cosine_similarity, 0.0, 'N/A', 'N/A', 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) self.assertTrue(test_final_success) @@ -53,11 +54,11 @@ class TestCompare(unittest.TestCase): test_final_success, detailed_result_total = self.compare._compare_core_wrapper(bench_out, npu_out) self.assertTrue(test_final_success) self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), - actual_cosine_similarity, 0.0, 'N/A', 'N/A', + actual_cosine_similarity, 0.0, 'N/A', 'N/A', 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n'], ['torch.float32', 'torch.float32',(100, 100), - actual_cosine_similarity, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0,'pass', '\n']]) + actual_cosine_similarity, 0.0, 'N/A', 'N/A', + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) def test_compare_output(self): bench_out, npu_out = torch.randn(100, 100), torch.randn(100, 100) -- Gitee From 46497f3022011c9be27dda344f5155b4e3582ed6 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Feb 2024 20:22:16 +0800 Subject: [PATCH 29/29] fix cleancode --- .../api_accuracy_checker/test/ut/compare/test_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py index 6940fc2c71..242db8584a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/compare/test_compare.py @@ -2,11 +2,11 @@ import csv import os import shutil import time +import unittest + import numpy as np import torch.nn.functional -import unittest - from api_accuracy_checker.compare.compare import Comparator from api_accuracy_checker.compare.compare_column import CompareColumn @@ -55,8 +55,8 @@ class TestCompare(unittest.TestCase): self.assertTrue(test_final_success) self.assertEqual(detailed_result_total, [['torch.float32', 'torch.float32', (100, 100), actual_cosine_similarity, 0.0, 'N/A', 'N/A', - 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n'], - ['torch.float32', 'torch.float32',(100, 100), + 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n'], + ['torch.float32', 'torch.float32', (100, 100), actual_cosine_similarity, 0.0, 'N/A', 'N/A', 'N/A', 'N/A', 0.0, 0.0, 0, 0.0, 0.0, 'pass', '\n']]) -- Gitee