diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index a68cdce4a27f6e77bdd90a3e820c456eb6330818..43aaff0711d668785eb664ef8cc357d97b71fd63 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -12,8 +12,8 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, ULPStandardApi, BINARY_COMPARE_UNSUPPORT_LIST, \ - ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage + ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, ULPStandardApi, ThousandthStandardApi, \ + BINARY_COMPARE_UNSUPPORT_LIST, ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -244,7 +244,9 @@ def analyse_csv(npu_data, gpu_data, config): else: _, api_name, _ = full_api_name.split("*") compare_column.api_name = full_api_name_with_direction_status - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: + if api_name in ThousandthStandardApi: + new_status = record_thousandth_threshold_result(compare_column, row_npu) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: new_status = record_binary_consistency_result(api_name, compare_column, row_npu) elif api_name in AbsoluteStandardApi: new_status = record_absolute_threshold_result(compare_column, row_npu) @@ -418,6 +420,23 @@ def record_ulp_compare_result(compare_column, us): return compare_column.compare_result +def check_thousandth_rate(thousandth_rate): + return CompareConst.PASS if convert_str_to_float(thousandth_rate) >= 0.999 else CompareConst.ERROR + + +def record_thousandth_threshold_result(compare_column, row_npu): + new_status = check_thousandth_rate(row_npu[ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH]) + compare_column.rel_err_thousandth = row_npu[ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH] + compare_column.rel_err_thousandth_status = new_status + compare_column.compare_result = new_status + compare_column.compare_algorithm = "双千指标法" + message = '' + if compare_column.rel_err_thousandth_status == CompareConst.ERROR: + message += "ERROR: 双千指标不达标\n" + compare_column.compare_message = message + return compare_column.compare_result + + def _api_precision_compare(parser=None): if not parser: parser = argparse.ArgumentParser() diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml index 84d0675874b0016aaeb6955c42fb6b3a34a747d8..aaa01f7494cb8d8a61cd1e4fd911d1e1f6a9c737 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml @@ -123,4 +123,8 @@ ULPStandard: - mm - mv - smm - - sspaddmm \ No newline at end of file + - sspaddmm + +ThousandthStandard: + - conv1d + - conv2d diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index db549ec1d54060cb8a9f96ceeec39f90566b05ca..84007494261dbe20bb8c0199ea0aac33e1754c6c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -8,7 +8,7 @@ from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv, print_warn_log from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, ULPStandardApi, \ - apis_threshold + ThousandthStandardApi, apis_threshold from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ @@ -279,6 +279,10 @@ class Comparator: message = "" abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) + rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) + if api_name in ThousandthStandardApi: + thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, 0.001) + compare_column.rel_err_thousandth = thousand_res if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) if api_name in BinaryStandardApi: @@ -330,7 +334,6 @@ class Comparator: message += "Max abs error is less than 0.001, consider as pass, skip other check and set to SPACE.\n" return CompareConst.PASS, compare_column, message - rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) if dtype in [torch.float16, torch.bfloat16]: hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) compare_column.rel_err_hundredth = hundred_res diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index cbaa3cb512397522d41423492641179bfd72f5db..eef73648b25d6f2595fb2ec02e3062d1cf352eba 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -57,6 +57,8 @@ class ApiPrecisionOutputColumn: self.ulp_err_proportion = CompareConst.SPACE self.ulp_err_proportion_ratio = CompareConst.SPACE self.ulp_err_status = CompareConst.SPACE + self.rel_err_thousandth = CompareConst.SPACE + self.rel_err_thousandth_status = CompareConst.SPACE self.compare_result = CompareConst.SPACE self.compare_algorithm = CompareConst.SPACE self.compare_message = CompareConst.SPACE @@ -67,5 +69,5 @@ class ApiPrecisionOutputColumn: self.mean_rel_err_status, self.eb_ratio, self.eb_status, self.inf_nan_error_ratio, self.inf_nan_error_ratio_status, self.rel_err_ratio, self.rel_err_ratio_status, self.abs_err_ratio, self.abs_err_ratio_status, self.error_rate, self.error_rate_status, self.mean_ulp_err, - self.ulp_err_proportion, self.ulp_err_proportion_ratio, self.ulp_err_status, self.compare_result, - self.compare_algorithm, self.compare_message] + self.ulp_err_proportion, self.ulp_err_proportion_ratio, self.ulp_err_status, self.rel_err_thousandth, + self.rel_err_thousandth_status, self.compare_result, self.compare_algorithm, self.compare_message] diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index a1cc8caf8851ba13a313b16f049827e8397a8314..0be970747a0974170f9bf1c6356e7f12c12dfd17 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -23,6 +23,7 @@ with FileOpen(standard_yaml_path, 'r') as f: AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') BinaryStandardApi = Apis.get('BinaryCompareStandard') ULPStandardApi = Apis.get('ULPStandard') + ThousandthStandardApi = Apis.get('ThousandthStandard') threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml") @@ -155,6 +156,8 @@ class ApiPrecisionCompareColumn: ULP_ERR_PROPORTION = 'ULP误差大于阈值占比' ULP_ERR_PROPORTION_RATIO = 'ULP误差大于阈值占比比值' ULP_ERR_STATUS = 'ULP误差判定结果' + REL_ERR_THOUSANDTH = '双千指标' + REL_ERR_THOUSANDTH_STATUS = '双千指标判定结果' FINAL_RESULT = '比对结果' ALGORITHM = '比对算法' FORWWARD_STATUS = 'Forward Test Success' @@ -168,7 +171,8 @@ class ApiPrecisionCompareColumn: ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB, ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO, - ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION] + ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, + ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH] @staticmethod def get_detail_csv_title(): @@ -183,7 +187,8 @@ class ApiPrecisionCompareColumn: ApiPrecisionCompareColumn.ABS_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, ApiPrecisionCompareColumn.MEAN_ULP_ERR, ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, - ApiPrecisionCompareColumn.ULP_ERR_PROPORTION_RATIO, ApiPrecisionCompareColumn.ULP_ERR_STATUS, + ApiPrecisionCompareColumn.ULP_ERR_PROPORTION_RATIO, ApiPrecisionCompareColumn.ULP_ERR_STATUS, + ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH, ApiPrecisionCompareColumn.REL_ERR_THOUSANDTH_STATUS, ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE] @staticmethod diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py index 51fcfedefbb8a6ef079a2d26cdc7d9ca841092bf..67dc5ad2532143430319b7f203713b211e2fe172 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py @@ -22,6 +22,7 @@ import numpy from api_accuracy_checker.common.utils import Const, check_file_or_directory_path, check_object_type, print_warn_log, \ print_error_log, get_full_data_path, CompareException +from api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api TORCH_TYPE = ["torch.device", "torch.dtype"] TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] @@ -32,12 +33,13 @@ NUMPY_TYPE = ["numpy.int8", "numpy.int16", "numpy.int32", "numpy.int64", "numpy. "numpy.complex128", "numpy.complex256", "numpy.bool_", "numpy.string_", "numpy.bytes_", "numpy.unicode_"] -def gen_data(info, need_grad, convert_type, real_data_path=None): +def gen_data(info, api_name, need_grad, convert_type, real_data_path=None): """ Function Description: Based on arg basic information, generate arg data Parameter: info: arg basic information. Dict + api_name: API name need_grad: set Tensor grad for backward convert_type: convert ori_type to dist_type flag. """ @@ -50,6 +52,8 @@ def gen_data(info, need_grad, convert_type, real_data_path=None): data = gen_real_tensor(data_path, convert_type) else: data = gen_random_tensor(info, convert_type) + if api_name in hf_32_standard_api and data.dtype == torch.float32: + data = fp32_to_hf32_to_fp32(data) if info.get('requires_grad') and need_grad: data.requires_grad_(True) temp_data = data * 1 @@ -121,6 +125,17 @@ def gen_random_tensor(info, convert_type): return data +def fp32_to_hf32_to_fp32(input_tensor): + # 将输入的float32 tensor转为hf32 tensor，再转为float32 tensor + input_np = input_tensor.numpy() + input_int = input_np.view(numpy.int32) + input_int = numpy.right_shift(numpy.right_shift(input_int, 11) + 1, 1) + input_int = numpy.left_shift(input_int, 12) + input_fp32 = input_int.view(numpy.float32) + input_hf32 = torch.from_numpy(input_fp32) + return input_hf32 + + def gen_common_tensor(low_info, high_info, shape, data_dtype, convert_type): """ Function Description: @@ -209,12 +224,13 @@ def gen_bool_tensor(low, high, shape): return data -def gen_args(args_info, need_grad=True, convert_type=None, real_data_path=None): +def gen_args(args_info, api_name, need_grad=True, convert_type=None, real_data_path=None): """ Function Description: Based on API basic information, generate input parameters: args, for API forward running Parameter: api_info: API basic information. List + api_name: API name need_grad: set Tensor grad for backward convert_type: convert ori_type to dist_type flag. real_data_path: the root directory for storing real data. @@ -223,9 +239,9 @@ def gen_args(args_info, need_grad=True, convert_type=None, real_data_path=None): args_result = [] for arg in args_info: if isinstance(arg, (list, tuple)): - data = gen_args(arg, need_grad, convert_type, real_data_path) + data = gen_args(arg, api_name, need_grad, convert_type, real_data_path) elif isinstance(arg, dict): - data = gen_data(arg, need_grad, convert_type, real_data_path) + data = gen_data(arg, api_name, need_grad, convert_type, real_data_path) else: print_warn_log(f'Warning: {arg} is not supported') raise NotImplementedError() @@ -279,12 +295,13 @@ def gen_list_kwargs(kwargs_item_value, convert_type, real_data_path=None): return kwargs_item_result -def gen_api_params(api_info, need_grad=True, convert_type=None, real_data_path=None): +def gen_api_params(api_info, api_name, need_grad=True, convert_type=None, real_data_path=None): """ Function Description: Based on API basic information, generate input parameters: args, kwargs, for API forward running Parameter: api_info: API basic information. Dict + api_name: API name need_grad: set grad for backward convert_type: convert ori_type to dist_type flag. """ @@ -294,7 +311,7 @@ def gen_api_params(api_info, need_grad=True, convert_type=None, real_data_path=N raise CompareException(CompareException.INVALID_PARAM_ERROR, error_info) kwargs_params = gen_kwargs(api_info, convert_type, real_data_path) if api_info.get("args"): - args_params = gen_args(api_info.get("args"), need_grad, convert_type, real_data_path) + args_params = gen_args(api_info.get("args"), api_name, need_grad, convert_type, real_data_path) else: print_warn_log(f'Warning: No args in {api_info} ') args_params = [] diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py index 05bd4305a3b8ac1366596d2947cafd467a784b1d..79f402e33e7099ecf79a7b686841195294985d53 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut.py @@ -17,7 +17,7 @@ else: import torch from tqdm import tqdm from api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args -from api_accuracy_checker.run_ut.run_ut_utils import Backward_Message +from api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api from api_accuracy_checker.common.utils import print_info_log, print_warn_log, get_json_contents, api_info_preprocess, \ print_error_log, initialize_save_path, Const, create_directory from api_accuracy_checker.compare.compare import Comparator @@ -77,7 +77,18 @@ def deal_detach(arg, to_detach=True): return arg.detach() if to_detach else arg -def deal_dtype(arg, raise_dtype=None): +def raise_bench_data_dtype(api_name, arg, raise_dtype=None): + ''' + 将标杆数据的dtype转换为raise_dtype + 输入： + api_name：api名称 + arg：标杆输入 + raise_dtype：需要转换的dtype + 输出： + arg: 转换dtype的标杆输入 + ''' + if api_name in hf_32_standard_api and arg.dtype == torch.float32: + return arg if raise_dtype is None or arg.dtype not in Const.RAISE_PRECISION or raise_dtype == arg.dtype: return arg return arg.type(raise_dtype) @@ -112,13 +123,13 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): return type(arg_in)(recursive_arg_to_cpu(arg, to_detach, raise_dtype=raise_dtype) for arg in arg_in) elif isinstance(arg_in, torch.Tensor): if need_backward and arg_in.requires_grad: - arg_in = deal_detach(deal_dtype(arg_in.clone(), raise_dtype), to_detach).requires_grad_() + arg_in = deal_detach(raise_bench_data_dtype(api_name, arg_in.clone(), raise_dtype), to_detach).requires_grad_() temp_arg_in = arg_in * 1 arg_in = temp_arg_in.type_as(arg_in) arg_in.retain_grad() return arg_in else: - return deal_detach(deal_dtype(arg_in.clone(), raise_dtype=raise_dtype), to_detach) + return deal_detach(raise_bench_data_dtype(api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach) else: return arg_in @@ -204,11 +215,11 @@ def do_save_error_data(api_full_name, data_info, is_fwd_success, is_bwd_success) api_full_name = api_full_name.replace("*", ".") for element in data_info.in_fwd_data_list: UtAPIInfo(api_full_name + '.forward.input', element) - UtAPIInfo(api_full_name + '.forward.output.bench', data_info.bench_out) - UtAPIInfo(api_full_name + '.forward.output.device', data_info.device_out) + UtAPIInfo(api_full_name + '.forward.output.bench', data_info.bench_output) + UtAPIInfo(api_full_name + '.forward.output.device', data_info.device_output) UtAPIInfo(api_full_name + '.backward.input', data_info.grad_in) - UtAPIInfo(api_full_name + '.backward.output.bench', data_info.bench_grad_out) - UtAPIInfo(api_full_name + '.backward.output.device', data_info.device_grad_out) + UtAPIInfo(api_full_name + '.backward.output.bench', data_info.bench_grad) + UtAPIInfo(api_full_name + '.backward.output.device', data_info.device_grad) def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): @@ -246,7 +257,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict if need_backward: if need_to_backward(grad_index, out): backward_args = backward_content[api_full_name] - grad = gen_args(backward_args, real_data_path=real_data_path)[0] + grad = gen_args(backward_args, api_name, real_data_path=real_data_path)[0] bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) device_grad = grad.clone().detach().to(current_device) @@ -262,7 +273,7 @@ def get_api_info(api_info_dict, api_name, real_data_path): need_grad = True if api_info_dict.get("kwargs") and "out" in api_info_dict.get("kwargs"): need_grad = False - args, kwargs = gen_api_params(api_info_dict, need_grad, convert_type, real_data_path) + args, kwargs = gen_api_params(api_info_dict, api_name, need_grad, convert_type, real_data_path) return args, kwargs, need_grad diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py index 490d8a5db3d13678c4b86d042f2681322a88620d..d78642f214f904183af73f18e726ee8ad72ef2b4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/run_ut_utils.py @@ -1,3 +1,6 @@ +hf_32_standard_api = ["conv1d", "conv2d"] + + class Backward_Message: MULTIPLE_BACKWARD_MESSAGE = "Multiple backward is not supported." UNSUPPORT_BACKWARD_MESSAGE = "function with out=... arguments don't support automatic differentiation, skip backward." diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py index b98f84d516404665b5c3284f1e03f14eedddac55..ad6ef5b3970fccd4beb2bdc63a0fa3ad54b3ebc1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py @@ -30,7 +30,7 @@ class TestDataGenerateMethods(unittest.TestCase): self.assertEqual(kwargs_params, {'inplace': False}) def test_gen_args(self): - args_result = gen_args(api_info_dict.get('args'), real_data_path=None) + args_result = gen_args(api_info_dict.get('args'), "api_name", real_data_path=None) max_diff = abs(args_result[0].max() - max_value) min_diff = abs(args_result[0].min() - min_value) self.assertEqual(len(args_result), 1) @@ -40,7 +40,7 @@ class TestDataGenerateMethods(unittest.TestCase): self.assertEqual(args_result[0].shape, torch.Size([2, 2560, 24, 24])) def test_gen_data(self): - data = gen_data(api_info_dict.get('args')[0], True, None, None) + data = gen_data(api_info_dict.get('args')[0], "api_name", True, None, None) max_diff = abs(data.max() - max_value) min_diff = abs(data.min() - min_value) self.assertEqual(data.dtype, torch.float32)