diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index bca77a887d5a17f3e1531ce65701649275cb601f..9dd204f5bc9a625170f6f5957cd7d9c3179ee694 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -10,7 +10,7 @@ def cosine_sim(bench_output, device_output): msg = "" n_value = device_output.reshape(-1) b_value = bench_output.reshape(-1) - cos = CompareConst.NA + cos = CompareConst.SPACE np.seterr(divide="ignore", invalid="ignore") if n_value.shape != b_value.shape: msg = f"Shape of device and bench outputs don't match. device: {n_value.shape}, bench: {b_value.shape}." @@ -25,10 +25,10 @@ def cosine_sim(bench_output, device_output): return cos, True, msg elif n_value_max <= np.finfo(float).eps: msg = "All the data is zero in device dump data." - return CompareConst.NA, False, msg + return CompareConst.SPACE, False, msg elif b_value_max <= np.finfo(float).eps: msg = "All the data is zero in bench dump data." - return CompareConst.NA, False, msg + return CompareConst.SPACE, False, msg else: n_value = n_value.astype(float) / n_value_max b_value = b_value.astype(float) / b_value_max diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index b86402d05ad104b36270e3c9c55d208de0e230da..8c98130ab61d27479817bc9c8c7eed0cc4f38361 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -11,7 +11,7 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST + ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST, convert_str_to_float from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -46,6 +46,26 @@ benchmark_algorithms_thresholds = { } +benchmark_message = { + "small_value_err_status": { + CompareConst.ERROR: "ERROR: 小值域错误比值超过阈值\n", + CompareConst.WARNING: "WARNING: 小值域错误比值超过阈值\n" + }, + "rmse_status": { + CompareConst.ERROR: "ERROR: 均方根误差比值超过阈值\n", + CompareConst.WARNING: "WARNING: 均方根误差比值超过阈值\n" + }, + "max_rel_err_status": { + CompareConst.ERROR: "ERROR: 相对误差最大值比值超过阈值\n", + CompareConst.WARNING: "WARNING: 相对误差最大值比值超过阈值\n" + }, + "mean_rel_err_status": { + CompareConst.ERROR: "ERROR: 相对误差平均值比值超过阈值\n", + CompareConst.WARNING: "WARNING: 相对误差平均值比值超过阈值\n" + } +} + + class BenchmarkStandard: def __init__(self, api_name, npu_precision, gpu_precision): self.api_name = api_name @@ -113,6 +133,7 @@ class BenchmarkStandard: @staticmethod def _calc_ratio(x, y, default_value=1.0): + x, y = convert_str_to_float(x), convert_str_to_float(y) if math.isclose(y, 0.0): return 1.0 if math.isclose(x, 0.0) else default_value else: @@ -170,14 +191,14 @@ def analyse_csv(npu_data, gpu_data, config): raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] #当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 - if pd.isna(row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]): + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): continue - _, dedicated_api_name, _ = full_api_name.split("*") - new_status = CompareConst.NA + _, api_name, _ = full_api_name.split("*") + new_status = CompareConst.SPACE compare_column.api_name = full_api_name_with_direction_status - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST: + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: new_status = record_binary_consistency_result(compare_column, row_npu) - elif dedicated_api_name in AbsoluteStandardApiName: + elif api_name in AbsoluteStandardApi: new_status = record_absolute_threshold_result(compare_column, row_npu) elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) @@ -222,13 +243,13 @@ def analyse_csv(npu_data, gpu_data, config): def check_error_rate(npu_error_rate): - return CompareConst.PASS if npu_error_rate == 0 else CompareConst.ERROR + return CompareConst.PASS if convert_str_to_float(npu_error_rate) == 0 else CompareConst.ERROR def get_absolute_threshold_result(row_npu): - inf_nan_error_ratio = row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] - rel_err_ratio = row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] - abs_err_ratio = row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] + inf_nan_error_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) + rel_err_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) + abs_err_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR @@ -252,7 +273,7 @@ def get_absolute_threshold_result(row_npu): def get_api_checker_result(status): if not status: - return CompareConst.NA + return CompareConst.SPACE for const in (CompareConst.ERROR, CompareConst.WARNING): if const in status: return const @@ -272,7 +293,11 @@ def record_binary_consistency_result(compare_column, row_npu): compare_column.error_rate = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] compare_column.error_rate_status = new_status compare_column.compare_result = new_status - compare_column.algorithm = "二进制一致法" + compare_column.compare_algorithm = "二进制一致法" + message = '' + if compare_column.error_rate_status == CompareConst.ERROR: + message += "ERROR: 二进制一致错误率超过阈值" + compare_column.compare_message = message return new_status @@ -285,7 +310,15 @@ def record_absolute_threshold_result(compare_column, row_npu): compare_column.abs_err_ratio = absolute_threshold_result.get("abs_err_ratio") compare_column.abs_err_ratio_status = absolute_threshold_result.get("abs_err_result") compare_column.compare_result = absolute_threshold_result.get("absolute_threshold_result") - compare_column.algorithm = "绝对阈值法" + compare_column.compare_algorithm = "绝对阈值法" + message = '' + if compare_column.inf_nan_error_ratio_status == CompareConst.ERROR: + message += "ERROR: inf/nan错误率超过阈值\n" + if compare_column.rel_err_ratio_status == CompareConst.ERROR: + message += "ERROR: 相对误差错误率超过阈值\n" + if compare_column.abs_err_ratio_status == CompareConst.ERROR: + message += "ERROR: 绝对误差错误率超过阈值\n" + compare_column.compare_message = message return compare_column.compare_result @@ -302,7 +335,13 @@ def record_benchmark_compare_result(compare_column, bs): compare_column.eb_ratio = bs.eb_ratio compare_column.eb_status = bs.eb_status compare_column.compare_result = bs.final_result - compare_column.algorithm = "标杆比对法" + compare_column.compare_algorithm = "标杆比对法" + message = '' + for status_attr, messages in benchmark_message.items(): + status_value = getattr(compare_column, status_attr) + if status_value in messages: + message += messages[status_value] + compare_column.compare_message = message return compare_column.compare_result diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml index 29665e91551704887621594721517a92d31129ce..ceccf65a46bdb757db4079c3e5e5bff71a5625b5 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml @@ -14,446 +14,94 @@ # limitations under the License. AbsoluteThreshStandard: - mul: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - mul_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __mul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __imul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __rmul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - add: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - add_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __add__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __iadd__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __radd__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - div: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - div_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __div__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __idiv__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - floor_divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - floor_divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - true_divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - true_divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - leaky_relu: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - leaky_relu_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - prelu: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - reciprocal: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - reciprocal_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsqrt: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsqrt_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - square: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - square_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - sub: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - sub_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsub: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __isub__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __sub__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - \ No newline at end of file + - mul + - mul_ + - __mul__ + - __imul__ + - __rmul__ + - add + - add_ + - __add__ + - __iadd__ + - __radd__ + - div + - div_ + - __div__ + - __idiv__ + - divide + - divide_ + - leaky_relu + - leaky_relu_ + - prelu + - reciprocal + - reciprocal_ + - rsqrt + - rsqrt_ + - square + - square_ + - sub + - sub_ + - rsub + - __isub__ + - __sub__ + +BinaryCompareStandard: + - abs + - abs_ + - absolute + - absolute_ + - argmin + - bitwise_and + - bitwise_and_ + - broadcast_to + - ceil + - ceil_ + - equal + - fill_ + - flatten + - floor + - floor_ + - gather + - greater + - greater_ + - greater_equal + - greater_equal_ + - isfinite + - isnan + - less + - less_ + - less_equal + - less_equal_ + - logical_and + - logical_and_ + - logical_not + - logical_not_ + - logical_or + - logical_or_ + - masked_fill + - masked_fill_ + - max_pool3d + - maximum + - minimum + - neg + - neg_ + - nonzero + - not_equal + - not_equal_ + - one_hot + - pad + - relu + - reshape + - round + - round_ + - select + - sign + - sign_ + - sort + - tile + - transpose + - transpose_ + - tril + - tril_ + - triu + - triu_ diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7565112da3122c636ac1e67a8494bbaed51d17c7 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml @@ -0,0 +1,390 @@ +mul: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +mul_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__mul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__imul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__rmul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +add: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +add_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__add__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__iadd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__radd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +div: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +div_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__div__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__idiv__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +leaky_relu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +leaky_relu_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +prelu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +reciprocal: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +reciprocal_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsqrt: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsqrt_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +square: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +square_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +sub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +sub_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__isub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__sub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 1906d6bbee0ba759f22533a39536f5093643bdc5..15bfb1904c810f756600daef1a90ab1d4176ac0d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -7,7 +7,7 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ - precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, AbsoluteStandardApiName + precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, apis_threshold from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ @@ -72,7 +72,7 @@ class Comparator: console.print(table_detail) def get_statistics_from_result_csv(self): - checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.NA, CompareConst.SKIP, "skip"] + checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.SPACE, CompareConst.SKIP, "skip"] self.test_result_cnt = { "success_num": 0, "warning_num": 0, "error_num": 0, "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, @@ -87,7 +87,7 @@ class Comparator: raise ValueError("The number of columns in %s is incorrect" % result_csv_name) if not all(item[i] and item[i] in checklist for i in (1, 2)): raise ValueError( - "The value in the 2nd or 3rd column of %s is wrong, it must be pass, error, warning, skip, or N/A" + "The value in the 2nd or 3rd column of %s is wrong, it must be pass, error, warning, skip, or SPACE" % result_csv_name) column1 = item[1] column2 = item[2] @@ -95,7 +95,7 @@ class Comparator: self.test_result_cnt["total_skip_num"] += 1 continue self.test_result_cnt["total_num"] += 1 - if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.NA]: + if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.SPACE]: self.test_result_cnt['success_num'] += 1 elif column1 == CompareConst.ERROR and column2 == CompareConst.ERROR: self.test_result_cnt['forward_and_backward_fail_num'] += 1 @@ -161,7 +161,7 @@ class Comparator: compare_func = self._compare_dropout if "dropout" in full_api_name else self._compare_core_wrapper fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(api_name, bench_grad[0], npu_grad[0]) if "dropout" in full_api_name else compare_func(api_name, bench_grad, npu_grad) - self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) + self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS def _compare_core_wrapper(self, api_name, bench_output, device_output): @@ -261,7 +261,10 @@ class Comparator: abs_err = get_abs_err(bench_output, device_output) if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - if api_name in AbsoluteStandardApiName: + if api_name in BinaryStandardApi: + err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output) + compare_column.error_rate = err_rate + elif api_name in AbsoluteStandardApi: small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( api_name, str(dtype)) rel_err = abs_err / abs_bench_with_eps @@ -285,13 +288,13 @@ class Comparator: compare_column.cosine_sim = cos_res message += msg + "\n" if not cos_status: - message += "Cosine similarity is less than 0.99, consider as error, skip other check and set to N/A.\n" + message += "Cosine similarity is less than 0.99, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message max_abs_res, max_abs_status = get_max_abs_err(abs_err) compare_column.max_abs_err = max_abs_res if max_abs_status: - message += "Max abs error is less than 0.001, consider as pass, skip other check and set to N/A.\n" + message += "Max abs error is less than 0.001, consider as pass, skip other check and set to SPACE.\n" return CompareConst.PASS, compare_column, message rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) @@ -299,24 +302,24 @@ class Comparator: hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) compare_column.rel_err_hundredth = hundred_res if not hundred_status: - message += "Relative error is greater than 0.01, consider as error, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.01, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, 0.001) compare_column.rel_err_thousandth = thousand_res if dtype in [torch.float16, torch.bfloat16]: if thousand_status: - message += "Relative error is less than 0.001, consider as pass, skip other check and set to N/A.\n" + message += "Relative error is less than 0.001, consider as pass, skip other check and set to SPACE.\n" return CompareConst.PASS, compare_column, message - message += "Relative error is greater than 0.001, consider as warning, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.001, consider as warning, skip other check and set to SPACE.\n" return CompareConst.WARNING, compare_column, message ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, 0.0001) compare_column.rel_err_ten_thousandth = ten_thousand_res if dtype in [torch.float32, torch.float64]: if not thousand_status: - message += "Relative error is greater than 0.001, consider as error, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.001, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message if not ten_thousand_status: - message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to SPACE.\n" return CompareConst.WARNING, compare_column, message message += "Relative error is less than 0.0001, consider as pass.\n" return CompareConst.PASS, compare_column, message @@ -353,7 +356,7 @@ class Comparator: @staticmethod def _get_absolute_threshold_attribute(api_name, dtype): - small_value_threshold = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value') - small_value_atol = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value_atol') - rtol = AbsoluteStandardApi.get(api_name).get(dtype).get('rtol') + small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value') + small_value_atol = apis_threshold.get(api_name).get(dtype).get('small_value_atol') + rtol = apis_threshold.get(api_name).get(dtype).get('rtol') return small_value_threshold, small_value_atol, rtol diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index a1b23467504b14159235d1973c30160fb87b4992..961fce6811efd34789cb06f19d894244da681c33 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -3,23 +3,23 @@ from api_accuracy_checker.compare.compare_utils import CompareConst class CompareColumn: def __init__(self): - self.bench_type = CompareConst.NA - self.npu_type = CompareConst.NA - self.shape = CompareConst.NA - self.cosine_sim = CompareConst.NA - self.max_abs_err = CompareConst.NA - self.rel_err_hundredth = CompareConst.NA - self.rel_err_thousandth = CompareConst.NA - self.rel_err_ten_thousandth = CompareConst.NA - self.error_rate = CompareConst.NA - self.EB = CompareConst.NA - self.RMSE = CompareConst.NA - self.small_value_err_ratio = CompareConst.NA - self.Max_rel_error = CompareConst.NA - self.Mean_rel_error = CompareConst.NA - self.inf_nan_error_ratio = CompareConst.NA - self.rel_err_ratio = CompareConst.NA - self.abs_err_ratio = CompareConst.NA + self.bench_type = CompareConst.SPACE + self.npu_type = CompareConst.SPACE + self.shape = CompareConst.SPACE + self.cosine_sim = CompareConst.SPACE + self.max_abs_err = CompareConst.SPACE + self.rel_err_hundredth = CompareConst.SPACE + self.rel_err_thousandth = CompareConst.SPACE + self.rel_err_ten_thousandth = CompareConst.SPACE + self.error_rate = CompareConst.SPACE + self.EB = CompareConst.SPACE + self.RMSE = CompareConst.SPACE + self.small_value_err_ratio = CompareConst.SPACE + self.Max_rel_error = CompareConst.SPACE + self.Mean_rel_error = CompareConst.SPACE + self.inf_nan_error_ratio = CompareConst.SPACE + self.rel_err_ratio = CompareConst.SPACE + self.abs_err_ratio = CompareConst.SPACE def to_column_value(self, is_pass, message): return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 34e4a6ba56a9784154c9d624559f8c80f6ed24c1..d711265cc781c634caa189692b5e0141eee2f83e 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -16,11 +16,16 @@ BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_C cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") -with FileOpen(yaml_path, 'r') as f: +standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") +with FileOpen(standard_yaml_path, 'r') as f: Apis = yaml.safe_load(f) AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') - AbsoluteStandardApiName = list(AbsoluteStandardApi.keys()) + BinaryStandardApi = Apis.get('BinaryCompareStandard') + + +threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml") +with FileOpen(threshold_yaml_path, 'r') as f: + apis_threshold = yaml.safe_load(f) DETAIL_TEST_ROWS = [[ @@ -162,3 +167,19 @@ def check_dtype_comparable(x, y): return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") return False + + +def convert_str_to_float(input_data): + if isinstance(input_data, str) and input_data.strip() == "": + msg = 'ERROR: Input data is an empty string' + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) + try: + float_data = float(input_data) + if str(float_data) in ('inf', '-inf', 'nan'): + msg = 'ERROR: Input data is either "inf", "-inf", "nan"' + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) + return float_data + except ValueError as e: + msg = 'ERROR: Input data cannot be converted to float' + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) from e + \ No newline at end of file