diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 394ea9cf0e49158355dfce62c58f65fd1a3722b5..560d35da835bef2f91fce8e367e491371c090eef 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -119,3 +119,71 @@ def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold): small_value_mask = np.less_equal(abs_bench, small_value_threshold) small_value_mask = np.logical_and(small_value_mask, both_finite_mask) return small_value_mask + + +def get_abs_bench_with_eps(bench, dtype): + abs_bench = np.abs(bench) + eps = np.finfo(bench.dtype).eps if dtype != torch.bfloat16 else CompareConst.BFLOAT16_EPS + abs_bench_with_eps = abs_bench + eps + return abs_bench, abs_bench_with_eps + + +def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): + ''' + 新精度标准的绝对阈值法中，检查npu和golden输出的inf、nan是否一致 + 输入： + inf_nan_mask：npu输出和golden输出的inf、nan的mask + bench_output：golden输出 + device_output：npu输出 + dtype：npu输出的dtype + 输出： + inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例 + ''' + abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) + golden_same_dtype = bench_output.astype(device_output.dtype) + a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN + a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MAX + golden_clip = np.clip(golden_same_dtype, a_min, a_max) + npu_clip = np.clip(device_output, a_min, a_max) + clipped_abs_ae = np.abs(npu_clip - golden_clip) + clipped_re = clipped_abs_ae / abs_gpu_with_eps + pass_mask = np.less_equal(clipped_re, rtol) + both_nan_mask = np.logical_and(np.isnan(device_output), np.isnan(golden_clip)) + pass_mask = np.logical_or(pass_mask, both_nan_mask) + not_pass_mask = np.logical_not(pass_mask) + not_pass_mask = np.logical_and(not_pass_mask, inf_nan_mask) + + inf_nan_err_cnt = np.sum(not_pass_mask) + return 0 if np.sum(inf_nan_mask) == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) + + +def check_small_value(abs_err, small_value_mask, small_value_atol): + ''' + 新精度标准的相对阈值法中，检查npu和golden小值域输出的相对误差是否满足阈值 + 输入： + rel_err：npu输出和golden输出的相对误差 + normal_value_mask：npu输出和golden输出的正常值mask + rtol：相对误差的阈值 + 输出： + rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例 + ''' + greater_mask = np.greater(abs_err, small_value_atol) + err_mask = np.logical_and(greater_mask, small_value_mask) + err_cnt = np.sum(err_mask) + return 0 if np.sum(small_value_mask) == 0 else err_cnt / np.sum(small_value_mask) + + +def check_norm_value(normal_value_mask, rel_err, rtol): + ''' + 新精度标准的绝对阈值法中，检查npu和golden正常值输出的绝对误差是否满足阈值 + 输入： + abs_err：npu输出和golden输出的绝对误差 + normal_value_mask：npu输出和golden输出的正常值mask + atol：绝对误差的阈值 + 输出： + abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例 + ''' + err_mask = np.greater(rel_err, rtol) + err_mask = np.logical_and(err_mask, normal_value_mask) + err_cnt = np.sum(err_mask) + return 0 if np.sum(normal_value_mask) == 0 else err_cnt / np.sum(normal_value_mask) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/benchmark_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py similarity index 55% rename from debug/accuracy_tools/api_accuracy_checker/compare/benchmark_compare.py rename to debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index d42be6be91d038ff581be407e10faeaf5ae120ab..b86402d05ad104b36270e3c9c55d208de0e230da 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/benchmark_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -9,9 +9,10 @@ import pandas as pd from api_accuracy_checker.common.utils import print_info_log, print_warn_log, print_error_log, write_csv, \ CompareException, create_directory from api_accuracy_checker.common.config import msCheckerConfig -from api_accuracy_checker.compare.compare_utils import CompareConst, BENCHMARK_COMPARE_RESULT_FILE_NAME, \ -BENCHMARK_COMPARE_DETAILS_FILE_NAME, Benchmark_Compare_Support_List, Benchmark_Compare_Unsupport_List, \ - BenchmarkCompareColumn +from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ +API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ + ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST +from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create @@ -84,19 +85,19 @@ class BenchmarkStandard: def _compare_ratio(self): self.small_value_err_ratio = self._calc_ratio( - self.npu_precision.get(BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE), - self.gpu_precision.get(BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE)) - self.rmse_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.RMSE), - self.gpu_precision.get(BenchmarkCompareColumn.RMSE), 10000.0) - self.max_rel_err_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.MAX_REL_ERR), - self.gpu_precision.get(BenchmarkCompareColumn.MAX_REL_ERR), 10000.0) - self.mean_rel_err_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.MEAN_REL_ERR), - self.gpu_precision.get(BenchmarkCompareColumn.MEAN_REL_ERR)) - self.eb_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.EB), - self.gpu_precision.get(BenchmarkCompareColumn.EB)) + self.npu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE), + self.gpu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE)) + self.rmse_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.RMSE), + self.gpu_precision.get(ApiPrecisionCompareColumn.RMSE), 10000.0) + self.max_rel_err_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), 10000.0) + self.mean_rel_err_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR)) + self.eb_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.EB), + self.gpu_precision.get(ApiPrecisionCompareColumn.EB)) def to_column_value(self): - return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, + return [self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, self.mean_rel_err_status, self.eb_ratio, self.eb_status] @@ -126,8 +127,8 @@ def write_detail_csv(content, save_path): write_csv(rows, save_path) -def benchmark_compare(config): - print_info_log("start benchmark compare task") +def api_precision_compare(config): + print_info_log("Start compare task") print_info_log(f"Compare task result will be saved in {config.result_csv_path}") print_info_log(f"Compare task detail will be saved in {config.details_csv_path}") try: @@ -140,8 +141,8 @@ def benchmark_compare(config): except Exception as err: print_error_log(f"Open gpu csv Error: %s" % str(err)) check_csv_columns(gpu_data.columns, "gpu_csv") - detail_csv_title = [BenchmarkCompareColumn.get_detail_csv_title()] - result_csv_title = [BenchmarkCompareColumn.get_result_csv_title()] + detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] + result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] write_csv(result_csv_title, config.result_csv_path) write_csv(detail_csv_title, config.details_csv_path) try: @@ -154,30 +155,37 @@ def benchmark_compare(config): def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] - last_api_name = None - last_api_dtype = None + last_api_name, last_api_dtype = None, None for _, row_npu in npu_data.iterrows(): message = '' - part_api_name = row_npu[BenchmarkCompareColumn.API_NAME] - row_gpu = gpu_data[gpu_data[BenchmarkCompareColumn.API_NAME] == part_api_name] - api_name, direction_status, _, _ = part_api_name.split(".") - binary_consistency_check = False + compare_column = ApiPrecisionOutputColumn() + full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] + row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] + full_api_name, direction_status, _, _ = full_api_name_with_direction_status.split(".") if row_gpu.empty: - print_warn_log(f'This API : {part_api_name} does not exist in the GPU data.') + print_warn_log(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') continue if len(row_gpu) > 1: - msg = f'This API : {part_api_name} has multiple records in the GPU data.' + msg = f'This API : {full_api_name_with_direction_status} has multiple records in the GPU data.' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] - if row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] in Benchmark_Compare_Support_List: - bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) - bs.get_result() - write_detail_csv(bs.to_column_value(), config.details_csv_path) - else: - binary_consistency_check = True - - if last_api_name is not None and api_name != last_api_name: - if last_api_dtype in Benchmark_Compare_Unsupport_List: + #当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 + if pd.isna(row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]): + continue + _, dedicated_api_name, _ = full_api_name.split("*") + new_status = CompareConst.NA + compare_column.api_name = full_api_name_with_direction_status + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST: + new_status = record_binary_consistency_result(compare_column, row_npu) + elif dedicated_api_name in AbsoluteStandardApiName: + new_status = record_absolute_threshold_result(compare_column, row_npu) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: + bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) + new_status = record_benchmark_compare_result(compare_column, bs) + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) + + if last_api_name is not None and full_api_name != last_api_name: + if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) forward_status, backward_status = [], [] @@ -189,21 +197,13 @@ def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] message = '' - is_supported = row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] not in Benchmark_Compare_Unsupport_List - last_api_name = api_name - if pd.isna(row_npu[BenchmarkCompareColumn.DEVICE_DTYPE]): - continue - last_api_dtype = row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] + is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST + last_api_name = full_api_name + last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] if not is_supported: continue - - if binary_consistency_check: - new_status = check_error_rate(row_npu[BenchmarkCompareColumn.ERROR_RATE], - row_gpu[BenchmarkCompareColumn.ERROR_RATE]) - else: - new_status = bs.final_result - + if direction_status == 'forward': forward_status.append(new_status) elif direction_status == 'backward': @@ -212,7 +212,7 @@ def analyse_csv(npu_data, gpu_data, config): print_error_log(f"Invalid direction status: {direction_status}") if last_api_name is not None: - if last_api_dtype in Benchmark_Compare_Unsupport_List: + if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) else: @@ -221,8 +221,33 @@ def analyse_csv(npu_data, gpu_data, config): write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) -def check_error_rate(npu_error_rate, gpu_error_rate): - return CompareConst.PASS if npu_error_rate == 0 and gpu_error_rate == 0 else CompareConst.ERROR +def check_error_rate(npu_error_rate): + return CompareConst.PASS if npu_error_rate == 0 else CompareConst.ERROR + + +def get_absolute_threshold_result(row_npu): + inf_nan_error_ratio = row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] + rel_err_ratio = row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] + abs_err_ratio = row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] + + inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR + rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR + abs_err_result = CompareConst.PASS if abs_err_ratio == 0 else CompareConst.ERROR + + if CompareConst.ERROR in [inf_nan_result, rel_err_result, abs_err_result]: + absolute_threshold_result = CompareConst.ERROR + else: + absolute_threshold_result = CompareConst.PASS + + return { + "inf_nan_error_ratio": inf_nan_error_ratio, + "inf_nan_result": inf_nan_result, + "rel_err_ratio": rel_err_ratio, + "rel_err_result": rel_err_result, + "abs_err_ratio": abs_err_ratio, + "abs_err_result": abs_err_result, + "absolute_threshold_result": absolute_threshold_result, + } def get_api_checker_result(status): @@ -235,22 +260,61 @@ def get_api_checker_result(status): def check_csv_columns(columns, csv_type): - required_columns = BenchmarkCompareColumn.to_required_columns() + required_columns = ApiPrecisionCompareColumn.to_required_columns() missing_columns = [column for column in required_columns if column not in columns] if missing_columns: msg = f"The followint columns {','.join(missing_columns)} are missing in{csv_type}" raise CompareException(CompareException.INVALID_DATA_ERROR, msg) -def _benchmark_compare(parser=None): +def record_binary_consistency_result(compare_column, row_npu): + new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE]) + compare_column.error_rate = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] + compare_column.error_rate_status = new_status + compare_column.compare_result = new_status + compare_column.algorithm = "二进制一致法" + return new_status + + +def record_absolute_threshold_result(compare_column, row_npu): + absolute_threshold_result = get_absolute_threshold_result(row_npu) + compare_column.inf_nan_error_ratio = absolute_threshold_result.get("inf_nan_error_ratio") + compare_column.inf_nan_error_ratio_status = absolute_threshold_result.get("inf_nan_result") + compare_column.rel_err_ratio = absolute_threshold_result.get("rel_err_ratio") + compare_column.rel_err_ratio_status = absolute_threshold_result.get("rel_err_result") + compare_column.abs_err_ratio = absolute_threshold_result.get("abs_err_ratio") + compare_column.abs_err_ratio_status = absolute_threshold_result.get("abs_err_result") + compare_column.compare_result = absolute_threshold_result.get("absolute_threshold_result") + compare_column.algorithm = "绝对阈值法" + return compare_column.compare_result + + +def record_benchmark_compare_result(compare_column, bs): + bs.get_result() + compare_column.small_value_err_ratio = bs.small_value_err_ratio + compare_column.small_value_err_status = bs.small_value_err_status + compare_column.rmse_ratio = bs.rmse_ratio + compare_column.rmse_status = bs.rmse_status + compare_column.max_rel_err_ratio = bs.max_rel_err_ratio + compare_column.max_rel_err_status = bs.max_rel_err_status + compare_column.mean_rel_err_ratio = bs.mean_rel_err_ratio + compare_column.mean_rel_err_status = bs.mean_rel_err_status + compare_column.eb_ratio = bs.eb_ratio + compare_column.eb_status = bs.eb_status + compare_column.compare_result = bs.final_result + compare_column.algorithm = "标杆比对法" + return compare_column.compare_result + + +def _api_precision_compare(parser=None): if not parser: parser = argparse.ArgumentParser() - _benchmark_compare_parser(parser) + _api_precision_compare_parser(parser) args = parser.parse_args(sys.argv[1:]) - _benchmark_compare_command(args) + _api_precision_compare_command(args) -def _benchmark_compare_command(args): +def _api_precision_compare_command(args): npu_csv_path = get_validated_result_csv_path(args.npu_csv_path, 'detail') gpu_csv_path = get_validated_result_csv_path(args.gpu_csv_path, 'detail') out_path = os.path.realpath(args.out_path) if args.out_path else "./" @@ -258,13 +322,13 @@ def _benchmark_compare_command(args): create_directory(out_path) out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) out_path = out_path_checker.common_check() - result_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_RESULT_FILE_NAME) - details_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_DETAILS_FILE_NAME) + result_csv_path = os.path.join(out_path, API_PRECISION_COMPARE_RESULT_FILE_NAME) + details_csv_path = os.path.join(out_path, API_PRECISION_COMPARE_DETAILS_FILE_NAME) compare_config = CompareConfig(npu_csv_path, gpu_csv_path, result_csv_path, details_csv_path) - benchmark_compare(compare_config) + api_precision_compare(compare_config) -def _benchmark_compare_parser(parser): +def _api_precision_compare_parser(parser): parser.add_argument("-npu", "--npu_csv_path", dest="npu_csv_path", default="", type=str, help=" , Accuracy_checking_details.csv generated on the NPU by using the " "api_accuracy_checker tool.", @@ -274,11 +338,11 @@ def _benchmark_compare_parser(parser): "api_accuracy_checker tool.", required=False) parser.add_argument("-o", "--out_path", dest="out_path", default="", type=str, - help=" The benchmark compare task result out path.", + help=" The api precision compare task result out path.", required=False) if __name__ == '__main__': - _benchmark_compare() - print_info_log("Benchmark compare task completed.") + _api_precision_compare() + print_info_log("Compare task completed.") \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29665e91551704887621594721517a92d31129ce --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml @@ -0,0 +1,459 @@ +# Copyright (c) 2024 Huawei Technologies Co., Ltd +# All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AbsoluteThreshStandard: + mul: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + mul_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __mul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __imul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __rmul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + add: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + add_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __add__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __iadd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __radd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + div: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + div_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __div__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __idiv__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + floor_divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + floor_divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + true_divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + true_divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + leaky_relu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + leaky_relu_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + prelu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + reciprocal: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + reciprocal_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsqrt: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsqrt_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + square: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + square_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + sub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + sub_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __isub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __sub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index d9c94b78404c28a51f4a4c81aa1632b25aecbaef..0440b32f3da7c46828122d3c87bb78a3ff027521 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -7,11 +7,12 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ - precision_configs, Benchmark_Compare_Support_List + precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, AbsoluteStandardApiName from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ - get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask + get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \ + check_small_value, check_norm_value, get_abs_bench_with_eps from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -155,14 +156,15 @@ class Comparator: self.write_summary_csv(args) self.write_detail_csv(args) - def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): - compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper - fwd_success_status, fwd_compare_alg_results = compare_func(bench_output, device_output) - bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(bench_grad, npu_grad) - self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) + def compare_output(self, full_api_name, bench_output, device_output, bench_grad=None, npu_grad=None): + _, api_name, _ = full_api_name.split("*") + compare_func = self._compare_dropout if "dropout" in full_api_name else self._compare_core_wrapper + fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(api_name, bench_grad[0], npu_grad[0]) if "dropout" in full_api_name else compare_func(api_name, bench_grad, npu_grad) + self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS - def _compare_core_wrapper(self, bench_output, device_output): + def _compare_core_wrapper(self, api_name, bench_output, device_output): detailed_result_total = [] test_final_success = CompareConst.PASS if isinstance(bench_output, (list, tuple)): @@ -172,12 +174,12 @@ class Comparator: message = ["bench and npu output structure is different."] else: for b_out_i, n_out_i in zip(bench_output, device_output): - status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) + status_i, compare_result_i, message_i = self._compare_core(api_name, b_out_i, n_out_i) status.append(status_i) compare_result.append(compare_result_i) message.append(message_i) else: - status, compare_result, message = self._compare_core(bench_output, device_output) + status, compare_result, message = self._compare_core(api_name, bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) if status == CompareConst.ERROR: @@ -193,7 +195,7 @@ class Comparator: test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total - def _compare_core(self, bench_output, device_output): + def _compare_core(self, api_name, bench_output, device_output): compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): return CompareConst.ERROR, compare_column, "bench and npu output type is different." @@ -202,7 +204,7 @@ class Comparator: if b_keys != n_keys: return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." else: - status, compare_result, message = self._compare_core(list(bench_output.values()), + status, compare_result, message = self._compare_core(api_name, list(bench_output.values()), list(device_output.values())) elif isinstance(bench_output, torch.Tensor): copy_bench_out = bench_output.detach().clone() @@ -210,7 +212,7 @@ class Comparator: compare_column.bench_type = str(copy_bench_out.dtype) compare_column.npu_type = str(copy_device_output.dtype) compare_column.shape = tuple(device_output.shape) - status, compare_result, message = self._compare_torch_tensor(copy_bench_out, copy_device_output, + status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, compare_column) elif isinstance(bench_output, (bool, int, float, str)): compare_column.bench_type = str(type(bench_output)) @@ -224,7 +226,7 @@ class Comparator: return status, compare_result, message - def _compare_torch_tensor(self, bench_output, device_output, compare_column): + def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape npu_dtype = device_output.dtype @@ -249,58 +251,35 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - status, compare_column, message = self._compare_float_tensor(bench_output, device_output, + status, compare_column, message = self._compare_float_tensor(api_name, bench_output, device_output, compare_column, npu_dtype) return status, compare_column, message - @staticmethod - def _compare_dropout(bench_output, device_output): - tensor_num = bench_output.numel() - if tensor_num >= 100: - if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: - return CompareConst.PASS, 1 - else: - return CompareConst.ERROR, 0 - else: - return CompareConst.PASS, 1 - - @staticmethod - def _compare_builtin_type(bench_output, device_output, compare_column): - if not isinstance(bench_output, (bool, int, float, str)): - return CompareConst.PASS, compare_column, "" - if bench_output != device_output: - return CompareConst.ERROR, compare_column, "" - compare_column.error_rate = 0 - return CompareConst.PASS, compare_column, "" - - - @staticmethod - def _compare_bool_tensor(bench_output, device_output): - error_nums = (bench_output != device_output).sum() - if bench_output.size == 0: - return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result." - error_rate = float(error_nums / bench_output.size) - result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR - return error_rate, result, "" - - @staticmethod - def _compare_float_tensor(bench_output, device_output, compare_column, dtype): + def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype): message = "" - eps = np.finfo(bench_output.dtype).eps - abs_bench = np.abs(bench_output) - abs_bench_with_eps = abs_bench + eps + abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) - if str(dtype) in Benchmark_Compare_Support_List: - dtype_config = precision_configs.get(dtype) + if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) - abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) - compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) - rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) - compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) - compare_column.EB = get_error_balance(bench_output, device_output) - compare_column.Max_rel_error = get_max_rel_err(rel_err) - compare_column.Mean_rel_error = get_mean_rel_err(rel_err) + if api_name in AbsoluteStandardApiName: + small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( + api_name, str(dtype)) + rel_err = abs_err / abs_bench_with_eps + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold) + normal_value_mask = np.logical_and(both_finite_mask, np.logical_not(small_value_mask)) + compare_column.inf_nan_error_ratio = check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol) + compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol) + compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol) + else: + dtype_config = precision_configs.get(dtype) + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) + compare_column.EB = get_error_balance(bench_output, device_output) + compare_column.Max_rel_error = get_max_rel_err(rel_err) + compare_column.Mean_rel_error = get_mean_rel_err(rel_err) cos_res, cos_status, msg = cosine_sim(bench_output, device_output) compare_column.cosine_sim = cos_res @@ -333,3 +312,40 @@ class Comparator: if not ten_thousand_status: return CompareConst.WARNING, compare_column, message return CompareConst.PASS, compare_column, message + + @staticmethod + def _compare_dropout(api_name, bench_output, device_output): + tensor_num = bench_output.numel() + if tensor_num >= 100: + if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: + return CompareConst.PASS, 1 + else: + return CompareConst.ERROR, 0 + else: + return CompareConst.PASS, 1 + + @staticmethod + def _compare_builtin_type(bench_output, device_output, compare_column): + if not isinstance(bench_output, (bool, int, float, str)): + return CompareConst.PASS, compare_column, "" + if bench_output != device_output: + return CompareConst.ERROR, compare_column, "" + compare_column.error_rate = 0 + return CompareConst.PASS, compare_column, "" + + + @staticmethod + def _compare_bool_tensor(bench_output, device_output): + error_nums = (bench_output != device_output).sum() + if bench_output.size == 0: + return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result." + error_rate = float(error_nums / bench_output.size) + result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR + return error_rate, result, "" + + @staticmethod + def _get_absolute_threshold_attribute(api_name, dtype): + small_value_threshold = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value') + small_value_atol = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value_atol') + rtol = AbsoluteStandardApi.get(api_name).get(dtype).get('rtol') + return small_value_threshold, small_value_atol, rtol diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index 9579f6a915910533ddc14d60cdc41afb00f8ee67..a1b23467504b14159235d1973c30160fb87b4992 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -17,8 +17,47 @@ class CompareColumn: self.small_value_err_ratio = CompareConst.NA self.Max_rel_error = CompareConst.NA self.Mean_rel_error = CompareConst.NA + self.inf_nan_error_ratio = CompareConst.NA + self.rel_err_ratio = CompareConst.NA + self.abs_err_ratio = CompareConst.NA def to_column_value(self, is_pass, message): return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, - self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] \ No newline at end of file + self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, self.inf_nan_error_ratio, + self.rel_err_ratio, self.abs_err_ratio, is_pass, message] + + +class ApiPrecisionOutputColumn: + def __init__(self): + self.api_name = CompareConst.SPACE + self.small_value_err_ratio = CompareConst.SPACE + self.small_value_err_status = CompareConst.SPACE + self.rmse_ratio = CompareConst.SPACE + self.rmse_status = CompareConst.SPACE + self.max_rel_err_ratio = CompareConst.SPACE + self.max_rel_err_status = CompareConst.SPACE + self.mean_rel_err_ratio = CompareConst.SPACE + self.mean_rel_err_status = CompareConst.SPACE + self.eb_ratio = CompareConst.SPACE + self.eb_status = CompareConst.SPACE + self.inf_nan_error_ratio = CompareConst.SPACE + self.inf_nan_error_ratio_status = CompareConst.SPACE + self.rel_err_ratio = CompareConst.SPACE + self.rel_err_ratio_status = CompareConst.SPACE + self.abs_err_ratio = CompareConst.SPACE + self.abs_err_ratio_status = CompareConst.SPACE + self.error_rate = CompareConst.SPACE + self.error_rate_status = CompareConst.SPACE + self.compare_result = CompareConst.SPACE + self.compare_algorithm = CompareConst.SPACE + self.compare_message = CompareConst.SPACE + + def to_column_value(self): + return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, + self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, + self.mean_rel_err_status, self.eb_ratio, self.eb_status, self.inf_nan_error_ratio, + self.inf_nan_error_ratio_status, self.rel_err_ratio, self.rel_err_ratio_status, self.abs_err_ratio, + self.abs_err_ratio_status, self.error_rate, self.error_rate_status, self.compare_result, + self.compare_algorithm, self.compare_message] + \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index d96944bcec613009deafc1a5b4128511778e2730..34e4a6ba56a9784154c9d624559f8c80f6ed24c1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -1,19 +1,26 @@ import time +import os import numpy as np import torch +import yaml from api_accuracy_checker.common.utils import Const, print_warn_log +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen current_time = time.strftime("%Y%m%d%H%M%S") -BENCHMARK_COMPARE_RESULT_FILE_NAME = "benchmark_compare_result_" + current_time + ".csv" -BENCHMARK_COMPARE_DETAILS_FILE_NAME = "benchmark_compare_details_" + current_time + ".csv" -Benchmark_Compare_Support_List = ['torch.float16', 'torch.bfloat16', 'torch.float32'] -Benchmark_Compare_Unsupport_List = ['torch.float64'] -result_mapping = { - 'pass' : True, - 'warning': False, - 'error' : False -} +API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" +API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" +BENCHMARK_COMPARE_SUPPORT_LIST = ['torch.float16', 'torch.bfloat16', 'torch.float32'] +API_PRECISION_COMPARE_UNSUPPORT_LIST = ['torch.float64', 'torch.complex64', 'torch.complex128'] +BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_COMPARE_UNSUPPORT_LIST + + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") +with FileOpen(yaml_path, 'r') as f: + Apis = yaml.safe_load(f) + AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') + AbsoluteStandardApiName = list(AbsoluteStandardApi.keys()) DETAIL_TEST_ROWS = [[ @@ -23,12 +30,15 @@ DETAIL_TEST_ROWS = [[ "双百指标", "双千指标", "双万指标", - "错误率", + "二进制一致错误率", "误差均衡性", "均方根误差", "小值域错误占比", "相对误差最大值", "相对误差平均值", + "inf/nan错误率", + "相对误差错误率", + "绝对误差错误率", "Status", "Message" ]] @@ -71,9 +81,13 @@ class CompareConst: SKIP = 'SKIP' TRUE = 'TRUE' FALSE = 'FALSE' + BFLOAT16_MIN = -3.3895313892515355e+38 + BFLOAT16_MAX = 3.3895313892515355e+38 + BFLOAT16_EPS = 2 ** -8 + SPACE = " " -class BenchmarkCompareColumn: +class ApiPrecisionCompareColumn: API_NAME = 'API Name' DEVICE_DTYPE = 'DEVICE Dtype' SMALL_VALUE_ERROR_RATE = '小值域错误占比' @@ -91,31 +105,46 @@ class BenchmarkCompareColumn: MEAN_REL_ERR_STATUS = '相对误差平均值判定结果' EB_RATIO = '误差均衡性比值' EB_STATUS = '误差均衡性判定结果' - ERROR_RATE = '错误率' + ERROR_RATE = '二进制一致错误率' + ERROR_RATE_STATUS = '二进制一致错误率判定结果' + INF_NAN_ERROR_RATIO = 'inf/nan错误率' + INF_NAN_ERROR_RATIO_STATUS = 'inf/nan判定结果' + REL_ERR_RATIO = '相对误差错误率' + REL_ERR_RATIO_STATUS = '相对误差判定结果' + ABS_ERR_RATIO = '绝对误差错误率' + ABS_ERR_RATIO_STATUS = '绝对误差判定结果' + FINAL_RESULT = '比对结果' + ALGORITHM = '比对算法' FORWWARD_STATUS = 'Forward Test Success' BACKWARD_STATUS = 'Backward Test Success' MESSAGE = 'Message' @staticmethod def to_required_columns(): - return [BenchmarkCompareColumn.API_NAME, BenchmarkCompareColumn.DEVICE_DTYPE, - BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE, BenchmarkCompareColumn.RMSE, - BenchmarkCompareColumn.MAX_REL_ERR, BenchmarkCompareColumn.MEAN_REL_ERR, BenchmarkCompareColumn.EB, - BenchmarkCompareColumn.ERROR_RATE] - + return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.DEVICE_DTYPE, + ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE, ApiPrecisionCompareColumn.RMSE, + ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB, + ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, + ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO] + @staticmethod def get_detail_csv_title(): - return [BenchmarkCompareColumn.API_NAME, - BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATIO, BenchmarkCompareColumn.SMALL_VALUE_ERROR_STATUS, - BenchmarkCompareColumn.RMSE_RATIO, BenchmarkCompareColumn.RMSE_STATUS, - BenchmarkCompareColumn.MAX_REL_ERR_RATIO, BenchmarkCompareColumn.MAX_REL_ERR_STATUS, - BenchmarkCompareColumn.MEAN_REL_ERR_RATIO, BenchmarkCompareColumn.MEAN_REL_ERR_STATUS, - BenchmarkCompareColumn.EB_RATIO, BenchmarkCompareColumn.EB_STATUS] + return [ApiPrecisionCompareColumn.API_NAME, + ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATIO, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_STATUS, + ApiPrecisionCompareColumn.RMSE_RATIO, ApiPrecisionCompareColumn.RMSE_STATUS, + ApiPrecisionCompareColumn.MAX_REL_ERR_RATIO, ApiPrecisionCompareColumn.MAX_REL_ERR_STATUS, + ApiPrecisionCompareColumn.MEAN_REL_ERR_RATIO, ApiPrecisionCompareColumn.MEAN_REL_ERR_STATUS, + ApiPrecisionCompareColumn.EB_RATIO, ApiPrecisionCompareColumn.EB_STATUS, + ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO_STATUS, + ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO_STATUS, + ApiPrecisionCompareColumn.ABS_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, + ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, + ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE] @staticmethod def get_result_csv_title(): - return [BenchmarkCompareColumn.API_NAME, BenchmarkCompareColumn.FORWWARD_STATUS, - BenchmarkCompareColumn.BACKWARD_STATUS, BenchmarkCompareColumn.MESSAGE] + return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.FORWWARD_STATUS, + ApiPrecisionCompareColumn.BACKWARD_STATUS, ApiPrecisionCompareColumn.MESSAGE] def check_dtype_comparable(x, y): diff --git a/debug/accuracy_tools/atat/atat.py b/debug/accuracy_tools/atat/atat.py index 60e49cbe039cd2eb14e73d9bb71495b482e2c473..4f69afd2349f211d1c6e17ab9386de3c8fcd6909 100644 --- a/debug/accuracy_tools/atat/atat.py +++ b/debug/accuracy_tools/atat/atat.py @@ -18,7 +18,7 @@ import sys from api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command from ptdbg_ascend.src.python.ptdbg_ascend.parse_tool.cli import parse as cli_parse from api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut -from api_accuracy_checker.compare.benchmark_compare import _benchmark_compare_parser, _benchmark_compare_command +from api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, _api_precision_compare_command from api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, _run_overflow_check_command @@ -34,13 +34,13 @@ def main(): subparsers.add_parser('parse') run_ut_cmd_parser = subparsers.add_parser('run_ut') multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') - benchmark_compare_cmd_parser = subparsers.add_parser('benchmark_compare') + api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') _run_ut_parser(run_ut_cmd_parser) _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, help='Number of splits for parallel processing. Range: 1-64') - _benchmark_compare_parser(benchmark_compare_cmd_parser) + _api_precision_compare_parser(api_precision_compare_cmd_parser) _run_overflow_check_parser(run_overflow_check_cmd_parser) if len(sys.argv) == 1: parser.print_help() @@ -53,8 +53,8 @@ def main(): elif sys.argv[1] == "multi_run_ut": config = prepare_config(args) run_parallel_ut(config) - elif sys.argv[1] == "benchmark_compare": - _benchmark_compare_command(args) + elif sys.argv[1] == "api_precision_compare": + _api_precision_compare_command(args) elif sys.argv[1] == "run_overflow_check": _run_overflow_check_command(args)