From 0c27ec916330e7d65ff53b8265233a2aa596f64a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:32:07 +0000 Subject: [PATCH 01/25] update debug/accuracy_tools/atat/atat.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/atat/atat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/atat/atat.py b/debug/accuracy_tools/atat/atat.py index 60e49cbe03..4f69afd234 100644 --- a/debug/accuracy_tools/atat/atat.py +++ b/debug/accuracy_tools/atat/atat.py @@ -18,7 +18,7 @@ import sys from api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command from ptdbg_ascend.src.python.ptdbg_ascend.parse_tool.cli import parse as cli_parse from api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut -from api_accuracy_checker.compare.benchmark_compare import _benchmark_compare_parser, _benchmark_compare_command +from api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, _api_precision_compare_command from api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, _run_overflow_check_command @@ -34,13 +34,13 @@ def main(): subparsers.add_parser('parse') run_ut_cmd_parser = subparsers.add_parser('run_ut') multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') - benchmark_compare_cmd_parser = subparsers.add_parser('benchmark_compare') + api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') _run_ut_parser(run_ut_cmd_parser) _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, help='Number of splits for parallel processing. Range: 1-64') - _benchmark_compare_parser(benchmark_compare_cmd_parser) + _api_precision_compare_parser(api_precision_compare_cmd_parser) _run_overflow_check_parser(run_overflow_check_cmd_parser) if len(sys.argv) == 1: parser.print_help() @@ -53,8 +53,8 @@ def main(): elif sys.argv[1] == "multi_run_ut": config = prepare_config(args) run_parallel_ut(config) - elif sys.argv[1] == "benchmark_compare": - _benchmark_compare_command(args) + elif sys.argv[1] == "api_precision_compare": + _api_precision_compare_command(args) elif sys.argv[1] == "run_overflow_check": _run_overflow_check_command(args) -- Gitee From fd0783f7598df7ee5fba4cae77837649688519ae Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:32:35 +0000 Subject: [PATCH 02/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../compare/compare_utils.py | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index d96944bcec..79fd02edde 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -1,19 +1,26 @@ import time +import os import numpy as np import torch +import yaml from api_accuracy_checker.common.utils import Const, print_warn_log +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen current_time = time.strftime("%Y%m%d%H%M%S") -BENCHMARK_COMPARE_RESULT_FILE_NAME = "benchmark_compare_result_" + current_time + ".csv" -BENCHMARK_COMPARE_DETAILS_FILE_NAME = "benchmark_compare_details_" + current_time + ".csv" +BENCHMARK_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" +BENCHMARK_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" Benchmark_Compare_Support_List = ['torch.float16', 'torch.bfloat16', 'torch.float32'] -Benchmark_Compare_Unsupport_List = ['torch.float64'] -result_mapping = { - 'pass' : True, - 'warning': False, - 'error' : False -} +Api_Precision_Compare_Unsupport_List = ['torch.float64', 'torch.complex64', 'torch.complex128'] +Binary_Compare_Unsupport_List = Benchmark_Compare_Support_List + Api_Precision_Compare_Unsupport_List + + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") +with FileOpen(yaml_path, 'r') as f: + Apis = yaml.safe_load(f) + AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') + AbsoluteStandardApiName = list(AbsoluteStandardApi.keys()) DETAIL_TEST_ROWS = [[ @@ -29,6 +36,9 @@ DETAIL_TEST_ROWS = [[ "小值域错误占比", "相对误差最大值", "相对误差平均值", + "inf/nan错误率", + "相对误差错误率", + "绝对误差错误率", "Status", "Message" ]] @@ -73,7 +83,7 @@ class CompareConst: FALSE = 'FALSE' -class BenchmarkCompareColumn: +class ApiPrecisionCompareColumn: API_NAME = 'API Name' DEVICE_DTYPE = 'DEVICE Dtype' SMALL_VALUE_ERROR_RATE = '小值域错误占比' @@ -92,30 +102,43 @@ class BenchmarkCompareColumn: EB_RATIO = '误差均衡性比值' EB_STATUS = '误差均衡性判定结果' ERROR_RATE = '错误率' + ERROR_RATE_STATUS = '错误率判定结果' + INF_NAN_ERROR_RATIO = 'inf/nan错误率' + INF_NAN_ERROR_RATIO_STATUS = 'inf/nan判定结果' + REL_ERR_RATIO = '相对误差错误率' + REL_ERR_RATIO_STATUS = '相对误差判定结果' + ABS_ERR_RATIO = '绝对误差错误率' + ABS_ERR_RATIO_STATUS = '绝对误差判定结果' + FINAL_RESULT = '比对结果' + ALGORITHM = '比对算法' FORWWARD_STATUS = 'Forward Test Success' BACKWARD_STATUS = 'Backward Test Success' MESSAGE = 'Message' @staticmethod def to_required_columns(): - return [BenchmarkCompareColumn.API_NAME, BenchmarkCompareColumn.DEVICE_DTYPE, - BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE, BenchmarkCompareColumn.RMSE, - BenchmarkCompareColumn.MAX_REL_ERR, BenchmarkCompareColumn.MEAN_REL_ERR, BenchmarkCompareColumn.EB, - BenchmarkCompareColumn.ERROR_RATE] + return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.DEVICE_DTYPE, + ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE, ApiPrecisionCompareColumn.RMSE, + ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB, + ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, + ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO] @staticmethod def get_detail_csv_title(): - return [BenchmarkCompareColumn.API_NAME, - BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATIO, BenchmarkCompareColumn.SMALL_VALUE_ERROR_STATUS, - BenchmarkCompareColumn.RMSE_RATIO, BenchmarkCompareColumn.RMSE_STATUS, - BenchmarkCompareColumn.MAX_REL_ERR_RATIO, BenchmarkCompareColumn.MAX_REL_ERR_STATUS, - BenchmarkCompareColumn.MEAN_REL_ERR_RATIO, BenchmarkCompareColumn.MEAN_REL_ERR_STATUS, - BenchmarkCompareColumn.EB_RATIO, BenchmarkCompareColumn.EB_STATUS] + return [ApiPrecisionCompareColumn.API_NAME, + ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATIO, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_STATUS, + ApiPrecisionCompareColumn.RMSE_RATIO, ApiPrecisionCompareColumn.RMSE_STATUS, + ApiPrecisionCompareColumn.MAX_REL_ERR_RATIO, ApiPrecisionCompareColumn.MAX_REL_ERR_STATUS, + ApiPrecisionCompareColumn.MEAN_REL_ERR_RATIO, ApiPrecisionCompareColumn.MEAN_REL_ERR_STATUS, + ApiPrecisionCompareColumn.EB_RATIO, ApiPrecisionCompareColumn.EB_STATUS, + ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO_STATUS, ApiPrecisionCompareColumn.REL_ERR_RATIO_STATUS, + ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, + ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE] @staticmethod def get_result_csv_title(): - return [BenchmarkCompareColumn.API_NAME, BenchmarkCompareColumn.FORWWARD_STATUS, - BenchmarkCompareColumn.BACKWARD_STATUS, BenchmarkCompareColumn.MESSAGE] + return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.FORWWARD_STATUS, + ApiPrecisionCompareColumn.BACKWARD_STATUS, ApiPrecisionCompareColumn.MESSAGE] def check_dtype_comparable(x, y): -- Gitee From cc26472b1a8f5b96ece435f9acb401c7f96505ac Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:32:55 +0000 Subject: [PATCH 03/25] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=20debug/accur?= =?UTF-8?q?acy=5Ftools/api=5Faccuracy=5Fchecker/compare/benchmark=5Fcompar?= =?UTF-8?q?e.py=20=E4=B8=BA=20debug/accuracy=5Ftools/api=5Faccuracy=5Fchec?= =?UTF-8?q?ker/compare/api=5Fprecision=5Fcompare.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../compare/{benchmark_compare.py => api_precision_compare.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debug/accuracy_tools/api_accuracy_checker/compare/{benchmark_compare.py => api_precision_compare.py} (100%) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/benchmark_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py similarity index 100% rename from debug/accuracy_tools/api_accuracy_checker/compare/benchmark_compare.py rename to debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py -- Gitee From ae5607cb75a604d73e03b0ede300cc0060763700 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:33:15 +0000 Subject: [PATCH 04/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 118 +++++++++++------- 1 file changed, 72 insertions(+), 46 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index d42be6be91..b270317035 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -10,8 +10,8 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr CompareException, create_directory from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, BENCHMARK_COMPARE_RESULT_FILE_NAME, \ -BENCHMARK_COMPARE_DETAILS_FILE_NAME, Benchmark_Compare_Support_List, Benchmark_Compare_Unsupport_List, \ - BenchmarkCompareColumn +BENCHMARK_COMPARE_DETAILS_FILE_NAME, Benchmark_Compare_Support_List, Api_Precision_Compare_Unsupport_List, \ + ApiPrecisionCompareColumn, AbsoluteStandardApiName, Binary_Compare_Unsupport_List from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create @@ -84,16 +84,16 @@ class BenchmarkStandard: def _compare_ratio(self): self.small_value_err_ratio = self._calc_ratio( - self.npu_precision.get(BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE), - self.gpu_precision.get(BenchmarkCompareColumn.SMALL_VALUE_ERROR_RATE)) - self.rmse_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.RMSE), - self.gpu_precision.get(BenchmarkCompareColumn.RMSE), 10000.0) - self.max_rel_err_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.MAX_REL_ERR), - self.gpu_precision.get(BenchmarkCompareColumn.MAX_REL_ERR), 10000.0) - self.mean_rel_err_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.MEAN_REL_ERR), - self.gpu_precision.get(BenchmarkCompareColumn.MEAN_REL_ERR)) - self.eb_ratio = self._calc_ratio(self.npu_precision.get(BenchmarkCompareColumn.EB), - self.gpu_precision.get(BenchmarkCompareColumn.EB)) + self.npu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE), + self.gpu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE)) + self.rmse_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.RMSE), + self.gpu_precision.get(ApiPrecisionCompareColumn.RMSE), 10000.0) + self.max_rel_err_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), 10000.0) + self.mean_rel_err_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR)) + self.eb_ratio = self._calc_ratio(self.npu_precision.get(ApiPrecisionCompareColumn.EB), + self.gpu_precision.get(ApiPrecisionCompareColumn.EB)) def to_column_value(self): return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, @@ -127,7 +127,7 @@ def write_detail_csv(content, save_path): def benchmark_compare(config): - print_info_log("start benchmark compare task") + print_info_log("Start compare task") print_info_log(f"Compare task result will be saved in {config.result_csv_path}") print_info_log(f"Compare task detail will be saved in {config.details_csv_path}") try: @@ -140,13 +140,14 @@ def benchmark_compare(config): except Exception as err: print_error_log(f"Open gpu csv Error: %s" % str(err)) check_csv_columns(gpu_data.columns, "gpu_csv") - detail_csv_title = [BenchmarkCompareColumn.get_detail_csv_title()] - result_csv_title = [BenchmarkCompareColumn.get_result_csv_title()] + detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] + result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] write_csv(result_csv_title, config.result_csv_path) write_csv(detail_csv_title, config.details_csv_path) try: analyse_csv(npu_data, gpu_data, config) except Exception as err: + import traceback;traceback.print_exc() print_error_log(f"Analyse csv Error: %s" % str(err)) change_mode(config.result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) change_mode(config.details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) @@ -154,14 +155,13 @@ def benchmark_compare(config): def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] - last_api_name = None - last_api_dtype = None + last_api_name, last_api_dtype = None, None for _, row_npu in npu_data.iterrows(): message = '' - part_api_name = row_npu[BenchmarkCompareColumn.API_NAME] - row_gpu = gpu_data[gpu_data[BenchmarkCompareColumn.API_NAME] == part_api_name] + part_api_name = row_npu[ApiPrecisionCompareColumn.API_NAME] + row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == part_api_name] api_name, direction_status, _, _ = part_api_name.split(".") - binary_consistency_check = False + binary_consistency_check, absolute_threshold_check, benchmark_compare_check = False, False, False if row_gpu.empty: print_warn_log(f'This API : {part_api_name} does not exist in the GPU data.') continue @@ -169,15 +169,34 @@ def analyse_csv(npu_data, gpu_data, config): msg = f'This API : {part_api_name} has multiple records in the GPU data.' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] - if row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] in Benchmark_Compare_Support_List: + if pd.isna(row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]): + continue + _, dedicated_api_name, _ = api_name.split("*") + new_status = CompareConst.NA + detail_csv_content = [part_api_name] + [" " for _ in range(17)] + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in Binary_Compare_Unsupport_List: + new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE], + row_gpu[ApiPrecisionCompareColumn.ERROR_RATE]) + detail_csv_content[14] = new_status + detail_csv_content[15] = new_status + detail_csv_content[16] = "二进制一致法" + elif dedicated_api_name in AbsoluteStandardApiName: + inf_nan_result, rel_err_result, abs_err_result, new_status = get_absolute_threshold_result(row_npu, row_gpu) + detail_csv_content[11:14] = [inf_nan_result, rel_err_result, abs_err_result] + detail_csv_content[15] = new_status + detail_csv_content[16] = "绝对阈值法" + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in Benchmark_Compare_Support_List: bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) bs.get_result() - write_detail_csv(bs.to_column_value(), config.details_csv_path) - else: - binary_consistency_check = True + new_status = bs.final_result + detail_csv_content[0:11] = bs.to_column_value() + detail_csv_content[15] = new_status + detail_csv_content[16] = "标杆比对法" + + write_detail_csv(detail_csv_content, config.details_csv_path) if last_api_name is not None and api_name != last_api_name: - if last_api_dtype in Benchmark_Compare_Unsupport_List: + if last_api_dtype in Api_Precision_Compare_Unsupport_List: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) forward_status, backward_status = [], [] @@ -189,21 +208,13 @@ def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] message = '' - is_supported = row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] not in Benchmark_Compare_Unsupport_List + is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in Api_Precision_Compare_Unsupport_List last_api_name = api_name - if pd.isna(row_npu[BenchmarkCompareColumn.DEVICE_DTYPE]): - continue - last_api_dtype = row_npu[BenchmarkCompareColumn.DEVICE_DTYPE] + last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] if not is_supported: continue - - if binary_consistency_check: - new_status = check_error_rate(row_npu[BenchmarkCompareColumn.ERROR_RATE], - row_gpu[BenchmarkCompareColumn.ERROR_RATE]) - else: - new_status = bs.final_result - + if direction_status == 'forward': forward_status.append(new_status) elif direction_status == 'backward': @@ -212,7 +223,7 @@ def analyse_csv(npu_data, gpu_data, config): print_error_log(f"Invalid direction status: {direction_status}") if last_api_name is not None: - if last_api_dtype in Benchmark_Compare_Unsupport_List: + if last_api_dtype in Api_Precision_Compare_Unsupport_List: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) else: @@ -225,6 +236,21 @@ def check_error_rate(npu_error_rate, gpu_error_rate): return CompareConst.PASS if npu_error_rate == 0 and gpu_error_rate == 0 else CompareConst.ERROR +def get_absolute_threshold_result(row_npu, row_gpu): + inf_nan_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] == 0 and \ + row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] == 0 else CompareConst.ERROR + rel_err_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] == 0 and \ + row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] == 0 else CompareConst.ERROR + abs_err_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] == 0 and \ + row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] == 0 else CompareConst.ERROR + if CompareConst.ERROR not in [inf_nan_result, rel_err_result, abs_err_result]: + absolute_threshold_result = CompareConst.PASS + else: + absolute_threshold_result = CompareConst.ERROR + + return inf_nan_result, rel_err_result, abs_err_result, absolute_threshold_result + + def get_api_checker_result(status): if not status: return CompareConst.NA @@ -235,22 +261,22 @@ def get_api_checker_result(status): def check_csv_columns(columns, csv_type): - required_columns = BenchmarkCompareColumn.to_required_columns() + required_columns = ApiPrecisionCompareColumn.to_required_columns() missing_columns = [column for column in required_columns if column not in columns] if missing_columns: msg = f"The followint columns {','.join(missing_columns)} are missing in{csv_type}" raise CompareException(CompareException.INVALID_DATA_ERROR, msg) -def _benchmark_compare(parser=None): +def _api_precision_compare(parser=None): if not parser: parser = argparse.ArgumentParser() - _benchmark_compare_parser(parser) + _api_precision_compare_parser(parser) args = parser.parse_args(sys.argv[1:]) - _benchmark_compare_command(args) + _api_precision_compare_command(args) -def _benchmark_compare_command(args): +def _api_precision_compare_command(args): npu_csv_path = get_validated_result_csv_path(args.npu_csv_path, 'detail') gpu_csv_path = get_validated_result_csv_path(args.gpu_csv_path, 'detail') out_path = os.path.realpath(args.out_path) if args.out_path else "./" @@ -264,7 +290,7 @@ def _benchmark_compare_command(args): benchmark_compare(compare_config) -def _benchmark_compare_parser(parser): +def _api_precision_compare_parser(parser): parser.add_argument("-npu", "--npu_csv_path", dest="npu_csv_path", default="", type=str, help=" , Accuracy_checking_details.csv generated on the NPU by using the " "api_accuracy_checker tool.", @@ -274,11 +300,11 @@ def _benchmark_compare_parser(parser): "api_accuracy_checker tool.", required=False) parser.add_argument("-o", "--out_path", dest="out_path", default="", type=str, - help=" The benchmark compare task result out path.", + help=" The api precision compare task result out path.", required=False) if __name__ == '__main__': - _benchmark_compare() - print_info_log("Benchmark compare task completed.") + _api_precision_compare() + print_info_log("Compare task completed.") \ No newline at end of file -- Gitee From 522e9d89cfe2e297b364cf8633fd7dfb0fbcad3a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:33:34 +0000 Subject: [PATCH 05/25] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/algorithm.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 394ea9cf0e..9a98f45d9c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -119,3 +119,43 @@ def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold): small_value_mask = np.less_equal(abs_bench, small_value_threshold) small_value_mask = np.logical_and(small_value_mask, both_finite_mask) return small_value_mask + + +def get_abs_bench_with_eps(bench, dtype): + abs_bench = np.abs(bench) + eps = np.finfo(bench.dtype).eps if dtype != torch.bfloat16 else 2 ** -8 + abs_bench_with_eps = abs_bench + eps + return abs_bench, abs_bench_with_eps + + +def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): + abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) + golden_same_dtype = bench_output.astype(device_output.dtype) + a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else -3.3895313892515355e+38 + a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else 3.3895313892515355e+38 + golden_clip = np.clip(golden_same_dtype, a_min, a_max) + npu_clip = np.clip(device_output, a_min, a_max) + clipped_abs_ae = np.abs(npu_clip - golden_clip) + clipped_re = clipped_abs_ae / abs_gpu_with_eps + pass_mask = np.less_equal(clipped_re, rtol) + both_nan_mask = np.logical_and(np.isnan(device_output), np.isnan(golden_clip)) + pass_mask = np.logical_or(pass_mask, both_nan_mask) + not_pass_mask = np.logical_not(pass_mask) + not_pass_mask = np.logical_and(not_pass_mask, inf_nan_mask) + + inf_nan_err_cnt = np.sum(not_pass_mask) + return 0 if inf_nan_err_cnt == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) + + +def check_small_value(abs_err, small_value_mask, small_value_atol): + greater_mask = np.greater(abs_err, small_value_atol) + err_mask = np.logical_and(greater_mask, small_value_mask) + err_cnt = np.sum(err_mask) + return 0 if err_cnt == 0 else err_cnt / np.sum(small_value_mask) + + +def check_norm_value(normal_value_mask, rel_err, rtol): + err_mask = np.greater(rel_err, rtol) + err_mask = np.logical_and(err_mask, normal_value_mask) + err_cnt = np.sum(err_mask) + return 0 if err_cnt == 0 else err_cnt / np.sum(normal_value_mask) -- Gitee From c7a06261ce0a2da4deba4269aa1c87ee7ba0083d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:34:19 +0000 Subject: [PATCH 06/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_column.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index 9579f6a915..7c2b3cc76f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -17,8 +17,13 @@ class CompareColumn: self.small_value_err_ratio = CompareConst.NA self.Max_rel_error = CompareConst.NA self.Mean_rel_error = CompareConst.NA + self.inf_nan_error_ratio = CompareConst.NA + self.rel_err_ratio = CompareConst.NA + self.abs_err_ratio = CompareConst.NA def to_column_value(self, is_pass, message): return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, - self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] \ No newline at end of file + self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, self.inf_nan_error_ratio, + self.rel_err_ratio, self.abs_err_ratio, is_pass, message] + \ No newline at end of file -- Gitee From 92a4770ab2bc6cd5d81ca07a019cac74688e4df4 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 08:34:51 +0000 Subject: [PATCH 07/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 128 ++++++++++-------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index d9c94b7840..2603c7def3 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -7,11 +7,12 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ - precision_configs, Benchmark_Compare_Support_List + precision_configs, Benchmark_Compare_Support_List, AbsoluteStandardApi, AbsoluteStandardApiName from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ - get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask + get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \ + check_small_value, check_norm_value, get_abs_bench_with_eps from api_accuracy_checker.common.config import msCheckerConfig from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen @@ -157,12 +158,12 @@ class Comparator: def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper - fwd_success_status, fwd_compare_alg_results = compare_func(bench_output, device_output) - bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(bench_grad, npu_grad) + fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(api_name, bench_grad, npu_grad) self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS - def _compare_core_wrapper(self, bench_output, device_output): + def _compare_core_wrapper(self, api_name, bench_output, device_output): detailed_result_total = [] test_final_success = CompareConst.PASS if isinstance(bench_output, (list, tuple)): @@ -172,12 +173,12 @@ class Comparator: message = ["bench and npu output structure is different."] else: for b_out_i, n_out_i in zip(bench_output, device_output): - status_i, compare_result_i, message_i = self._compare_core(b_out_i, n_out_i) + status_i, compare_result_i, message_i = self._compare_core(api_name, b_out_i, n_out_i) status.append(status_i) compare_result.append(compare_result_i) message.append(message_i) else: - status, compare_result, message = self._compare_core(bench_output, device_output) + status, compare_result, message = self._compare_core(api_name, bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) if status == CompareConst.ERROR: @@ -193,7 +194,7 @@ class Comparator: test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total - def _compare_core(self, bench_output, device_output): + def _compare_core(self, api_name, bench_output, device_output): compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): return CompareConst.ERROR, compare_column, "bench and npu output type is different." @@ -202,7 +203,7 @@ class Comparator: if b_keys != n_keys: return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." else: - status, compare_result, message = self._compare_core(list(bench_output.values()), + status, compare_result, message = self._compare_core(api_name, list(bench_output.values()), list(device_output.values())) elif isinstance(bench_output, torch.Tensor): copy_bench_out = bench_output.detach().clone() @@ -210,7 +211,7 @@ class Comparator: compare_column.bench_type = str(copy_bench_out.dtype) compare_column.npu_type = str(copy_device_output.dtype) compare_column.shape = tuple(device_output.shape) - status, compare_result, message = self._compare_torch_tensor(copy_bench_out, copy_device_output, + status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, compare_column) elif isinstance(bench_output, (bool, int, float, str)): compare_column.bench_type = str(type(bench_output)) @@ -224,7 +225,7 @@ class Comparator: return status, compare_result, message - def _compare_torch_tensor(self, bench_output, device_output, compare_column): + def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape npu_dtype = device_output.dtype @@ -249,58 +250,36 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - status, compare_column, message = self._compare_float_tensor(bench_output, device_output, + status, compare_column, message = self._compare_float_tensor(api_name, bench_output, device_output, compare_column, npu_dtype) return status, compare_column, message - @staticmethod - def _compare_dropout(bench_output, device_output): - tensor_num = bench_output.numel() - if tensor_num >= 100: - if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: - return CompareConst.PASS, 1 - else: - return CompareConst.ERROR, 0 - else: - return CompareConst.PASS, 1 - - @staticmethod - def _compare_builtin_type(bench_output, device_output, compare_column): - if not isinstance(bench_output, (bool, int, float, str)): - return CompareConst.PASS, compare_column, "" - if bench_output != device_output: - return CompareConst.ERROR, compare_column, "" - compare_column.error_rate = 0 - return CompareConst.PASS, compare_column, "" - - - @staticmethod - def _compare_bool_tensor(bench_output, device_output): - error_nums = (bench_output != device_output).sum() - if bench_output.size == 0: - return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result." - error_rate = float(error_nums / bench_output.size) - result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR - return error_rate, result, "" - - @staticmethod - def _compare_float_tensor(bench_output, device_output, compare_column, dtype): + def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype): message = "" - eps = np.finfo(bench_output.dtype).eps - abs_bench = np.abs(bench_output) - abs_bench_with_eps = abs_bench + eps + abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) if str(dtype) in Benchmark_Compare_Support_List: - dtype_config = precision_configs.get(dtype) + _, dedicated_api_name, _ = api_name.split("*") both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) - abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) - compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) - rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) - compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) - compare_column.EB = get_error_balance(bench_output, device_output) - compare_column.Max_rel_error = get_max_rel_err(rel_err) - compare_column.Mean_rel_error = get_mean_rel_err(rel_err) + if dedicated_api_name in AbsoluteStandardApiName: + small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( + dedicated_api_name, str(dtype)) + rel_err = abs_err / abs_bench_with_eps + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold) + normal_value_mask = np.logical_and(both_finite_mask, np.logical_not(small_value_mask)) + compare_column.inf_nan_error_ratio = check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol) + compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol) + compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol) + else: + dtype_config = precision_configs.get(dtype) + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) + compare_column.EB = get_error_balance(bench_output, device_output) + compare_column.Max_rel_error = get_max_rel_err(rel_err) + compare_column.Mean_rel_error = get_mean_rel_err(rel_err) cos_res, cos_status, msg = cosine_sim(bench_output, device_output) compare_column.cosine_sim = cos_res @@ -333,3 +312,40 @@ class Comparator: if not ten_thousand_status: return CompareConst.WARNING, compare_column, message return CompareConst.PASS, compare_column, message + + @staticmethod + def _compare_dropout(api_name, bench_output, device_output): + tensor_num = bench_output.numel() + if tensor_num >= 100: + if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: + return CompareConst.PASS, 1 + else: + return CompareConst.ERROR, 0 + else: + return CompareConst.PASS, 1 + + @staticmethod + def _compare_builtin_type(bench_output, device_output, compare_column): + if not isinstance(bench_output, (bool, int, float, str)): + return CompareConst.PASS, compare_column, "" + if bench_output != device_output: + return CompareConst.ERROR, compare_column, "" + compare_column.error_rate = 0 + return CompareConst.PASS, compare_column, "" + + + @staticmethod + def _compare_bool_tensor(bench_output, device_output): + error_nums = (bench_output != device_output).sum() + if bench_output.size == 0: + return CompareConst.NAN, CompareConst.ERROR, "There is not bench calculation result." + error_rate = float(error_nums / bench_output.size) + result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR + return error_rate, result, "" + + @staticmethod + def _get_absolute_threshold_attribute(dedicated_api_name, dtype): + small_value_threshold = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('small_value') + small_value_atol = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('small_value_atol') + rtol = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('rtol') + return small_value_threshold, small_value_atol, rtol -- Gitee From f1eae6537b13c215a4248d8edb809efdd96781fb Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 09:14:01 +0000 Subject: [PATCH 08/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/api_precision_compare.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index b270317035..d891a2a782 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -147,7 +147,6 @@ def benchmark_compare(config): try: analyse_csv(npu_data, gpu_data, config) except Exception as err: - import traceback;traceback.print_exc() print_error_log(f"Analyse csv Error: %s" % str(err)) change_mode(config.result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) change_mode(config.details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From cf3004d2588fb14e796c240821ca97a690413dad Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 10:01:43 +0000 Subject: [PATCH 09/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index d891a2a782..55a0381b8b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -180,7 +180,7 @@ def analyse_csv(npu_data, gpu_data, config): detail_csv_content[15] = new_status detail_csv_content[16] = "二进制一致法" elif dedicated_api_name in AbsoluteStandardApiName: - inf_nan_result, rel_err_result, abs_err_result, new_status = get_absolute_threshold_result(row_npu, row_gpu) + inf_nan_result, rel_err_result, abs_err_result, new_status = get_absolute_threshold_result(row_npu) detail_csv_content[11:14] = [inf_nan_result, rel_err_result, abs_err_result] detail_csv_content[15] = new_status detail_csv_content[16] = "绝对阈值法" @@ -235,17 +235,23 @@ def check_error_rate(npu_error_rate, gpu_error_rate): return CompareConst.PASS if npu_error_rate == 0 and gpu_error_rate == 0 else CompareConst.ERROR -def get_absolute_threshold_result(row_npu, row_gpu): - inf_nan_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] == 0 and \ - row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] == 0 else CompareConst.ERROR - rel_err_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] == 0 and \ - row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] == 0 else CompareConst.ERROR - abs_err_result = CompareConst.PASS if row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] == 0 and \ - row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] == 0 else CompareConst.ERROR - if CompareConst.ERROR not in [inf_nan_result, rel_err_result, abs_err_result]: - absolute_threshold_result = CompareConst.PASS - else: - absolute_threshold_result = CompareConst.ERROR +def get_absolute_threshold_result(row_npu): + inf_nan_error_ratio = row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] + rel_err_ratio = row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] + abs_err_ratio = row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] + + inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR + rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR + abs_err_result = CompareConst.PASS if abs_err_ratio == 0 else CompareConst.ERROR + + absolute_threshold_result = CompareConst.PASS if all(result == CompareConst.PASS for result in [inf_nan_result, rel_err_result, abs_err_result]) else CompareConst.ERROR + + return [ + inf_nan_error_ratio, inf_nan_result, + rel_err_ratio, rel_err_result, + abs_err_ratio, abs_err_result, + absolute_threshold_result + ] return inf_nan_result, rel_err_result, abs_err_result, absolute_threshold_result -- Gitee From 24cc694ba7d3c19b764139eca92f48d96e58547a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 9 Mar 2024 10:10:41 +0000 Subject: [PATCH 10/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 55a0381b8b..72ab715fc1 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -126,7 +126,7 @@ def write_detail_csv(content, save_path): write_csv(rows, save_path) -def benchmark_compare(config): +def api_precision_compare(config): print_info_log("Start compare task") print_info_log(f"Compare task result will be saved in {config.result_csv_path}") print_info_log(f"Compare task detail will be saved in {config.details_csv_path}") @@ -172,25 +172,25 @@ def analyse_csv(npu_data, gpu_data, config): continue _, dedicated_api_name, _ = api_name.split("*") new_status = CompareConst.NA - detail_csv_content = [part_api_name] + [" " for _ in range(17)] + detail_csv_content = [part_api_name] + [" " for _ in range(21)] if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in Binary_Compare_Unsupport_List: - new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE], - row_gpu[ApiPrecisionCompareColumn.ERROR_RATE]) - detail_csv_content[14] = new_status - detail_csv_content[15] = new_status - detail_csv_content[16] = "二进制一致法" + new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE]) + detail_csv_content[18] = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] + detail_csv_content[19] = new_status + detail_csv_content[20] = "二进制一致法" elif dedicated_api_name in AbsoluteStandardApiName: - inf_nan_result, rel_err_result, abs_err_result, new_status = get_absolute_threshold_result(row_npu) - detail_csv_content[11:14] = [inf_nan_result, rel_err_result, abs_err_result] - detail_csv_content[15] = new_status - detail_csv_content[16] = "绝对阈值法" + absolute_threshold_result = get_absolute_threshold_result(row_npu) + new_status = absolute_threshold_result[-1] + detail_csv_content[11:14] = absolute_threshold_result[:-1] + detail_csv_content[19] = new_status + detail_csv_content[20] = "绝对阈值法" elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in Benchmark_Compare_Support_List: bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) bs.get_result() new_status = bs.final_result detail_csv_content[0:11] = bs.to_column_value() - detail_csv_content[15] = new_status - detail_csv_content[16] = "标杆比对法" + detail_csv_content[19] = new_status + detail_csv_content[20] = "标杆比对法" write_detail_csv(detail_csv_content, config.details_csv_path) @@ -231,8 +231,8 @@ def analyse_csv(npu_data, gpu_data, config): write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) -def check_error_rate(npu_error_rate, gpu_error_rate): - return CompareConst.PASS if npu_error_rate == 0 and gpu_error_rate == 0 else CompareConst.ERROR +def check_error_rate(npu_error_rate): + return CompareConst.PASS if npu_error_rate == 0 else CompareConst.ERROR def get_absolute_threshold_result(row_npu): @@ -244,7 +244,8 @@ def get_absolute_threshold_result(row_npu): rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR abs_err_result = CompareConst.PASS if abs_err_ratio == 0 else CompareConst.ERROR - absolute_threshold_result = CompareConst.PASS if all(result == CompareConst.PASS for result in [inf_nan_result, rel_err_result, abs_err_result]) else CompareConst.ERROR + absolute_threshold_result = CompareConst.PASS if all(result == CompareConst.PASS for result in + [inf_nan_result, rel_err_result, abs_err_result]) else CompareConst.ERROR return [ inf_nan_error_ratio, inf_nan_result, @@ -292,7 +293,7 @@ def _api_precision_compare_command(args): result_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_RESULT_FILE_NAME) details_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_DETAILS_FILE_NAME) compare_config = CompareConfig(npu_csv_path, gpu_csv_path, result_csv_path, details_csv_path) - benchmark_compare(compare_config) + api_precision_compare(compare_config) def _api_precision_compare_parser(parser): -- Gitee From bc201aaaea5b591a92ffc70870b19246766087c8 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 02:01:13 +0000 Subject: [PATCH 11/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 79fd02edde..0124ab12b2 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -30,7 +30,7 @@ DETAIL_TEST_ROWS = [[ "双百指标", "双千指标", "双万指标", - "错误率", + "二进制一致错误率", "误差均衡性", "均方根误差", "小值域错误占比", @@ -101,8 +101,8 @@ class ApiPrecisionCompareColumn: MEAN_REL_ERR_STATUS = '相对误差平均值判定结果' EB_RATIO = '误差均衡性比值' EB_STATUS = '误差均衡性判定结果' - ERROR_RATE = '错误率' - ERROR_RATE_STATUS = '错误率判定结果' + ERROR_RATE = '二进制一致错误率' + ERROR_RATE_STATUS = '二进制一致错误率判定结果' INF_NAN_ERROR_RATIO = 'inf/nan错误率' INF_NAN_ERROR_RATIO_STATUS = 'inf/nan判定结果' REL_ERR_RATIO = '相对误差错误率' -- Gitee From 88e22e27966ff1f55375987b72f6c80f38041d1a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 02:12:58 +0000 Subject: [PATCH 12/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/api_precision_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 72ab715fc1..baf5359b3f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -244,8 +244,10 @@ def get_absolute_threshold_result(row_npu): rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR abs_err_result = CompareConst.PASS if abs_err_ratio == 0 else CompareConst.ERROR - absolute_threshold_result = CompareConst.PASS if all(result == CompareConst.PASS for result in - [inf_nan_result, rel_err_result, abs_err_result]) else CompareConst.ERROR + if CompareConst.ERROR in [inf_nan_result, rel_err_result, abs_err_result]: + absolute_threshold_result = CompareConst.ERROR + else: + absolute_threshold_result = CompareConst.PASS return [ inf_nan_error_ratio, inf_nan_result, @@ -254,8 +256,6 @@ def get_absolute_threshold_result(row_npu): absolute_threshold_result ] - return inf_nan_result, rel_err_result, abs_err_result, absolute_threshold_result - def get_api_checker_result(status): if not status: -- Gitee From 3adf0c2afef8efc396d65cb146dd3b6f753e2983 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 08:27:58 +0000 Subject: [PATCH 13/25] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 9a98f45d9c..5fc8e024cf 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -129,7 +129,7 @@ def get_abs_bench_with_eps(bench, dtype): def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): - abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) + abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) golden_same_dtype = bench_output.astype(device_output.dtype) a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else -3.3895313892515355e+38 a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else 3.3895313892515355e+38 -- Gitee From 321bc5b7a733edf8cb21ddf85452de4ec5efd9ce Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 08:28:36 +0000 Subject: [PATCH 14/25] add debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml. Signed-off-by: jiangchangting1 --- .../compare/api_precision_standard.yaml | 459 ++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml new file mode 100644 index 0000000000..29665e9155 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml @@ -0,0 +1,459 @@ +# Copyright (c) 2024 Huawei Technologies Co., Ltd +# All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AbsoluteThreshStandard: + mul: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + mul_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __mul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __imul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __rmul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + add: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + add_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __add__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __iadd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __radd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + div: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + div_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __div__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __idiv__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + floor_divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + floor_divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + true_divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + true_divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + leaky_relu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + leaky_relu_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + prelu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + reciprocal: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + reciprocal_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsqrt: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsqrt_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + square: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + square_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + sub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + sub_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + rsub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __isub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + __sub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 + \ No newline at end of file -- Gitee From ca1f43daaa1483311acb56f1c6aacab6c6c98e0e Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Mar 2024 21:07:29 +0800 Subject: [PATCH 15/25] fix --- .../api_accuracy_checker/compare/algorithm.py | 12 +-- .../compare/api_precision_compare.py | 78 +++++++++++++------ .../api_accuracy_checker/compare/compare.py | 30 +++---- .../compare/compare_column.py | 36 ++++++++- .../compare/compare_utils.py | 10 ++- 5 files changed, 116 insertions(+), 50 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 5fc8e024cf..b9e58f9a3a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -123,7 +123,7 @@ def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold): def get_abs_bench_with_eps(bench, dtype): abs_bench = np.abs(bench) - eps = np.finfo(bench.dtype).eps if dtype != torch.bfloat16 else 2 ** -8 + eps = np.finfo(bench.dtype).eps if dtype != torch.bfloat16 else CompareConst.BFLOAT16_EPS abs_bench_with_eps = abs_bench + eps return abs_bench, abs_bench_with_eps @@ -131,8 +131,8 @@ def get_abs_bench_with_eps(bench, dtype): def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) golden_same_dtype = bench_output.astype(device_output.dtype) - a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else -3.3895313892515355e+38 - a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else 3.3895313892515355e+38 + a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN + a_max = np.finfo(device_output.dtype).max if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MAX golden_clip = np.clip(golden_same_dtype, a_min, a_max) npu_clip = np.clip(device_output, a_min, a_max) clipped_abs_ae = np.abs(npu_clip - golden_clip) @@ -144,18 +144,18 @@ def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): not_pass_mask = np.logical_and(not_pass_mask, inf_nan_mask) inf_nan_err_cnt = np.sum(not_pass_mask) - return 0 if inf_nan_err_cnt == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) + return 0 if np.sum(inf_nan_mask) == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) def check_small_value(abs_err, small_value_mask, small_value_atol): greater_mask = np.greater(abs_err, small_value_atol) err_mask = np.logical_and(greater_mask, small_value_mask) err_cnt = np.sum(err_mask) - return 0 if err_cnt == 0 else err_cnt / np.sum(small_value_mask) + return 0 if np.sum(small_value_mask) == 0 else err_cnt / np.sum(small_value_mask) def check_norm_value(normal_value_mask, rel_err, rtol): err_mask = np.greater(rel_err, rtol) err_mask = np.logical_and(err_mask, normal_value_mask) err_cnt = np.sum(err_mask) - return 0 if err_cnt == 0 else err_cnt / np.sum(normal_value_mask) + return 0 if np.sum(normal_value_mask) == 0 else err_cnt / np.sum(normal_value_mask) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index baf5359b3f..a1fb88fb8a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -10,8 +10,8 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr CompareException, create_directory from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, BENCHMARK_COMPARE_RESULT_FILE_NAME, \ -BENCHMARK_COMPARE_DETAILS_FILE_NAME, Benchmark_Compare_Support_List, Api_Precision_Compare_Unsupport_List, \ - ApiPrecisionCompareColumn, AbsoluteStandardApiName, Binary_Compare_Unsupport_List +BENCHMARK_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ + ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create @@ -96,7 +96,7 @@ class BenchmarkStandard: self.gpu_precision.get(ApiPrecisionCompareColumn.EB)) def to_column_value(self): - return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, + return [self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, self.mean_rel_err_status, self.eb_ratio, self.eb_status] @@ -157,10 +157,10 @@ def analyse_csv(npu_data, gpu_data, config): last_api_name, last_api_dtype = None, None for _, row_npu in npu_data.iterrows(): message = '' + compare_column = ApiPrecisionCompareColumn() part_api_name = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == part_api_name] api_name, direction_status, _, _ = part_api_name.split(".") - binary_consistency_check, absolute_threshold_check, benchmark_compare_check = False, False, False if row_gpu.empty: print_warn_log(f'This API : {part_api_name} does not exist in the GPU data.') continue @@ -172,30 +172,19 @@ def analyse_csv(npu_data, gpu_data, config): continue _, dedicated_api_name, _ = api_name.split("*") new_status = CompareConst.NA - detail_csv_content = [part_api_name] + [" " for _ in range(21)] - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in Binary_Compare_Unsupport_List: - new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE]) - detail_csv_content[18] = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] - detail_csv_content[19] = new_status - detail_csv_content[20] = "二进制一致法" + compare_column.api_name = part_api_name + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST: + new_status = record_binary_consistency_result(compare_column, row_npu) elif dedicated_api_name in AbsoluteStandardApiName: - absolute_threshold_result = get_absolute_threshold_result(row_npu) - new_status = absolute_threshold_result[-1] - detail_csv_content[11:14] = absolute_threshold_result[:-1] - detail_csv_content[19] = new_status - detail_csv_content[20] = "绝对阈值法" - elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in Benchmark_Compare_Support_List: + new_status = record_absolute_threshold_result(compare_column, row_npu) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) - bs.get_result() - new_status = bs.final_result - detail_csv_content[0:11] = bs.to_column_value() - detail_csv_content[19] = new_status - detail_csv_content[20] = "标杆比对法" + new_status = record_benchmark_compare_result(compare_column, bs) - write_detail_csv(detail_csv_content, config.details_csv_path) + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) if last_api_name is not None and api_name != last_api_name: - if last_api_dtype in Api_Precision_Compare_Unsupport_List: + if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) forward_status, backward_status = [], [] @@ -207,7 +196,7 @@ def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] message = '' - is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in Api_Precision_Compare_Unsupport_List + is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST last_api_name = api_name last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] @@ -222,7 +211,7 @@ def analyse_csv(npu_data, gpu_data, config): print_error_log(f"Invalid direction status: {direction_status}") if last_api_name is not None: - if last_api_dtype in Api_Precision_Compare_Unsupport_List: + if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) else: @@ -274,6 +263,45 @@ def check_csv_columns(columns, csv_type): raise CompareException(CompareException.INVALID_DATA_ERROR, msg) +def record_binary_consistency_result(compare_column, row_npu): + new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE]) + compare_column.error_rate = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] + compare_column.compare_result = new_status + compare_column.algorithm = "二进制一致法" + return new_status + + +def record_absolute_threshold_result(compare_column, row_npu): + absolute_threshold_result = get_absolute_threshold_result(row_npu) + compare_column.inf_nan_error_ratio = absolute_threshold_result[0] + compare_column.inf_nan_result = absolute_threshold_result[1] + compare_column.rel_err_ratio = absolute_threshold_result[2] + compare_column.rel_err_result = absolute_threshold_result[3] + compare_column.abs_err_ratio = absolute_threshold_result[4] + compare_column.abs_err_result = absolute_threshold_result[5] + new_status = absolute_threshold_result[6] + compare_column.compare_result = new_status + compare_column.algorithm = "绝对阈值法" + return new_status + + +def record_benchmark_compare_result(compare_column, bs): + bs.get_result() + compare_column.small_value_err_ratio = bs.small_value_err_ratio + compare_column.small_value_err_status = bs.small_value_err_status + compare_column.rmse_ratio = bs.rmse_ratio + compare_column.rmse_status = bs.rmse_status + compare_column.max_rel_err_ratio = bs.max_rel_err_ratio + compare_column.max_rel_err_status = bs.max_rel_err_status + compare_column.mean_rel_err_ratio = bs.mean_rel_err_ratio + compare_column.mean_rel_err_status = bs.mean_rel_err_status + compare_column.eb_ratio = bs.eb_ratio + compare_column.eb_status = bs.eb_status + compare_column.compare_result = bs.final_result + compare_column.algorithm = "标杆比对法" + return bs.final_result + + def _api_precision_compare(parser=None): if not parser: parser = argparse.ArgumentParser() diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 2603c7def3..f5e47e4e25 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -7,7 +7,7 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ - precision_configs, Benchmark_Compare_Support_List, AbsoluteStandardApi, AbsoluteStandardApiName + precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, AbsoluteStandardApiName from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ @@ -157,13 +157,14 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): + _, dedicated_api_name, _ = api_name.split("*") compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper - fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) - bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(api_name, bench_grad, npu_grad) + fwd_success_status, fwd_compare_alg_results = compare_func(dedicated_api_name, bench_output, device_output) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(dedicated_api_name,bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(dedicated_api_name, bench_grad, npu_grad) self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS - def _compare_core_wrapper(self, api_name, bench_output, device_output): + def _compare_core_wrapper(self, dedicated_api_name, bench_output, device_output): detailed_result_total = [] test_final_success = CompareConst.PASS if isinstance(bench_output, (list, tuple)): @@ -173,12 +174,12 @@ class Comparator: message = ["bench and npu output structure is different."] else: for b_out_i, n_out_i in zip(bench_output, device_output): - status_i, compare_result_i, message_i = self._compare_core(api_name, b_out_i, n_out_i) + status_i, compare_result_i, message_i = self._compare_core(dedicated_api_name, b_out_i, n_out_i) status.append(status_i) compare_result.append(compare_result_i) message.append(message_i) else: - status, compare_result, message = self._compare_core(api_name, bench_output, device_output) + status, compare_result, message = self._compare_core(dedicated_api_name, bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) if status == CompareConst.ERROR: @@ -194,7 +195,7 @@ class Comparator: test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total - def _compare_core(self, api_name, bench_output, device_output): + def _compare_core(self, dedicated_api_name, bench_output, device_output): compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): return CompareConst.ERROR, compare_column, "bench and npu output type is different." @@ -203,7 +204,7 @@ class Comparator: if b_keys != n_keys: return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." else: - status, compare_result, message = self._compare_core(api_name, list(bench_output.values()), + status, compare_result, message = self._compare_core(dedicated_api_name, list(bench_output.values()), list(device_output.values())) elif isinstance(bench_output, torch.Tensor): copy_bench_out = bench_output.detach().clone() @@ -211,7 +212,7 @@ class Comparator: compare_column.bench_type = str(copy_bench_out.dtype) compare_column.npu_type = str(copy_device_output.dtype) compare_column.shape = tuple(device_output.shape) - status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, + status, compare_result, message = self._compare_torch_tensor(dedicated_api_name, copy_bench_out, copy_device_output, compare_column) elif isinstance(bench_output, (bool, int, float, str)): compare_column.bench_type = str(type(bench_output)) @@ -225,7 +226,7 @@ class Comparator: return status, compare_result, message - def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column): + def _compare_torch_tensor(self, dedicated_api_name, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape npu_dtype = device_output.dtype @@ -250,16 +251,15 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - status, compare_column, message = self._compare_float_tensor(api_name, bench_output, device_output, + status, compare_column, message = self._compare_float_tensor(dedicated_api_name, bench_output, device_output, compare_column, npu_dtype) return status, compare_column, message - def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype): + def _compare_float_tensor(self, dedicated_api_name, bench_output, device_output, compare_column, dtype): message = "" abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) - if str(dtype) in Benchmark_Compare_Support_List: - _, dedicated_api_name, _ = api_name.split("*") + if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) if dedicated_api_name in AbsoluteStandardApiName: small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( @@ -314,7 +314,7 @@ class Comparator: return CompareConst.PASS, compare_column, message @staticmethod - def _compare_dropout(api_name, bench_output, device_output): + def _compare_dropout(dedicated_api_name, bench_output, device_output): tensor_num = bench_output.numel() if tensor_num >= 100: if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index 7c2b3cc76f..4a04c3592c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -26,4 +26,38 @@ class CompareColumn: self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, self.inf_nan_error_ratio, self.rel_err_ratio, self.abs_err_ratio, is_pass, message] - \ No newline at end of file + + +class ApiPrecisionCompareColumn: + def __init__(self): + self.api_name = CompareConst.SPACE + self.small_value_err_ratio = CompareConst.SPACE + self.small_value_err_status = CompareConst.SPACE + self.rmse_ratio = CompareConst.SPACE + self.rmse_status = CompareConst.SPACE + self.max_rel_err_ratio = CompareConst.SPACE + self.max_rel_err_status = CompareConst.SPACE + self.mean_rel_err_ratio = CompareConst.SPACE + self.mean_rel_err_status = CompareConst.SPACE + self.eb_ratio = CompareConst.SPACE + self.eb_status = CompareConst.SPACE + self.inf_nan_error_ratio = CompareConst.SPACE + self.inf_nan_error_ratio_status = CompareConst.SPACE + self.rel_err_ratio = CompareConst.SPACE + self.rel_err_ratio_status = CompareConst.SPACE + self.abs_err_ratio = CompareConst.SPACE + self.abs_err_ratio_status = CompareConst.SPACE + self.error_rate = CompareConst.SPACE + self.error_rate_status = CompareConst.SPACE + self.compare_result = CompareConst.SPACE + self.compare_algorithm = CompareConst.SPACE + self.compare_message = CompareConst.SPACE + + def to_column_value(self, is_pass, message): + return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, + self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, + self.mean_rel_err_status, self.eb_ratio, self.eb_status, self.inf_nan_error_ratio, + self.inf_nan_error_ratio_status, self.rel_err_ratio, self.rel_err_ratio_status, self.abs_err_ratio, + self.abs_err_ratio_status, self.error_rate, self.error_rate_status, self.compare_result, + self.compare_algorithm, self.compare_message] + \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 0124ab12b2..b4950aa48f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -10,9 +10,9 @@ from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen current_time = time.strftime("%Y%m%d%H%M%S") BENCHMARK_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" BENCHMARK_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" -Benchmark_Compare_Support_List = ['torch.float16', 'torch.bfloat16', 'torch.float32'] -Api_Precision_Compare_Unsupport_List = ['torch.float64', 'torch.complex64', 'torch.complex128'] -Binary_Compare_Unsupport_List = Benchmark_Compare_Support_List + Api_Precision_Compare_Unsupport_List +BENCHMARK_COMPARE_SUPPORT_LIST = ['torch.float16', 'torch.bfloat16', 'torch.float32'] +API_PRECISION_COMPARE_UNSUPPORT_LIST = ['torch.float64', 'torch.complex64', 'torch.complex128'] +BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_COMPARE_UNSUPPORT_LIST cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -81,6 +81,10 @@ class CompareConst: SKIP = 'SKIP' TRUE = 'TRUE' FALSE = 'FALSE' + BFLOAT16_MIN = -3.3895313892515355e+38 + BFLOAT16_MAX = 3.3895313892515355e+38 + BFLOAT16_EPS = 2 ** -8 + SPACE = " " class ApiPrecisionCompareColumn: -- Gitee From d50c4f30082b2c11d1a6a9aed82658b97fcc4cf4 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Mar 2024 21:20:47 +0800 Subject: [PATCH 16/25] FIX --- .../api_accuracy_checker/compare/api_precision_compare.py | 8 ++++---- .../api_accuracy_checker/compare/compare_utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index a1fb88fb8a..1c8a0b81b9 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -9,8 +9,8 @@ import pandas as pd from api_accuracy_checker.common.utils import print_info_log, print_warn_log, print_error_log, write_csv, \ CompareException, create_directory from api_accuracy_checker.common.config import msCheckerConfig -from api_accuracy_checker.compare.compare_utils import CompareConst, BENCHMARK_COMPARE_RESULT_FILE_NAME, \ -BENCHMARK_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ +from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ +API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -318,8 +318,8 @@ def _api_precision_compare_command(args): create_directory(out_path) out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) out_path = out_path_checker.common_check() - result_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_RESULT_FILE_NAME) - details_csv_path = os.path.join(out_path, BENCHMARK_COMPARE_DETAILS_FILE_NAME) + result_csv_path = os.path.join(out_path, API_PRECISION_COMPARE_RESULT_FILE_NAME) + details_csv_path = os.path.join(out_path, API_PRECISION_COMPARE_DETAILS_FILE_NAME) compare_config = CompareConfig(npu_csv_path, gpu_csv_path, result_csv_path, details_csv_path) api_precision_compare(compare_config) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index b4950aa48f..2f0ae5a8c0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -8,8 +8,8 @@ from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen current_time = time.strftime("%Y%m%d%H%M%S") -BENCHMARK_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" -BENCHMARK_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" +API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" +API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" BENCHMARK_COMPARE_SUPPORT_LIST = ['torch.float16', 'torch.bfloat16', 'torch.float32'] API_PRECISION_COMPARE_UNSUPPORT_LIST = ['torch.float64', 'torch.complex64', 'torch.complex128'] BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_COMPARE_UNSUPPORT_LIST -- Gitee From 09535a2dc1db35e68ff8aa0748d2480caf55b7bd Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 13:58:32 +0000 Subject: [PATCH 17/25] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 1c8a0b81b9..e10fc103ae 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -12,6 +12,7 @@ from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST +from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_path_before_create @@ -157,7 +158,7 @@ def analyse_csv(npu_data, gpu_data, config): last_api_name, last_api_dtype = None, None for _, row_npu in npu_data.iterrows(): message = '' - compare_column = ApiPrecisionCompareColumn() + compare_column = ApiPrecisionOutputColumn() part_api_name = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == part_api_name] api_name, direction_status, _, _ = part_api_name.split(".") @@ -180,7 +181,6 @@ def analyse_csv(npu_data, gpu_data, config): elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) new_status = record_benchmark_compare_result(compare_column, bs) - write_detail_csv(compare_column.to_column_value(), config.details_csv_path) if last_api_name is not None and api_name != last_api_name: @@ -266,6 +266,7 @@ def check_csv_columns(columns, csv_type): def record_binary_consistency_result(compare_column, row_npu): new_status = check_error_rate(row_npu[ApiPrecisionCompareColumn.ERROR_RATE]) compare_column.error_rate = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] + compare_column.error_rate_status = new_status compare_column.compare_result = new_status compare_column.algorithm = "二进制一致法" return new_status @@ -274,11 +275,11 @@ def record_binary_consistency_result(compare_column, row_npu): def record_absolute_threshold_result(compare_column, row_npu): absolute_threshold_result = get_absolute_threshold_result(row_npu) compare_column.inf_nan_error_ratio = absolute_threshold_result[0] - compare_column.inf_nan_result = absolute_threshold_result[1] + compare_column.inf_nan_error_ratio_status = absolute_threshold_result[1] compare_column.rel_err_ratio = absolute_threshold_result[2] - compare_column.rel_err_result = absolute_threshold_result[3] + compare_column.rel_err_ratio_status = absolute_threshold_result[3] compare_column.abs_err_ratio = absolute_threshold_result[4] - compare_column.abs_err_result = absolute_threshold_result[5] + compare_column.abs_err_ratio_status = absolute_threshold_result[5] new_status = absolute_threshold_result[6] compare_column.compare_result = new_status compare_column.algorithm = "绝对阈值法" -- Gitee From 2a3abbd1bc77e608dd83d124ffdfa7023f4d3f1d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 13:58:52 +0000 Subject: [PATCH 18/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_column.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index 4a04c3592c..89f1c63298 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -15,7 +15,7 @@ class CompareColumn: self.EB = CompareConst.NA self.RMSE = CompareConst.NA self.small_value_err_ratio = CompareConst.NA - self.Max_rel_error = CompareConst.NA + self.Max_rel_error = CompareConst.NASPACE self.Mean_rel_error = CompareConst.NA self.inf_nan_error_ratio = CompareConst.NA self.rel_err_ratio = CompareConst.NA @@ -28,7 +28,7 @@ class CompareColumn: self.rel_err_ratio, self.abs_err_ratio, is_pass, message] -class ApiPrecisionCompareColumn: +class ApiPrecisionOutputColumn: def __init__(self): self.api_name = CompareConst.SPACE self.small_value_err_ratio = CompareConst.SPACE @@ -53,7 +53,7 @@ class ApiPrecisionCompareColumn: self.compare_algorithm = CompareConst.SPACE self.compare_message = CompareConst.SPACE - def to_column_value(self, is_pass, message): + def to_column_value(self): return [self.api_name, self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, self.mean_rel_err_status, self.eb_ratio, self.eb_status, self.inf_nan_error_ratio, -- Gitee From f5c224a9106a42b8e8e3a2104bc6baa0390058bf Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 11 Mar 2024 13:59:21 +0000 Subject: [PATCH 19/25] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 2f0ae5a8c0..0906bfb814 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -126,8 +126,7 @@ class ApiPrecisionCompareColumn: ApiPrecisionCompareColumn.MAX_REL_ERR, ApiPrecisionCompareColumn.MEAN_REL_ERR, ApiPrecisionCompareColumn.EB, ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO] - - @staticmethod + def get_detail_csv_title(): return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATIO, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_STATUS, @@ -135,8 +134,10 @@ class ApiPrecisionCompareColumn: ApiPrecisionCompareColumn.MAX_REL_ERR_RATIO, ApiPrecisionCompareColumn.MAX_REL_ERR_STATUS, ApiPrecisionCompareColumn.MEAN_REL_ERR_RATIO, ApiPrecisionCompareColumn.MEAN_REL_ERR_STATUS, ApiPrecisionCompareColumn.EB_RATIO, ApiPrecisionCompareColumn.EB_STATUS, - ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO_STATUS, ApiPrecisionCompareColumn.REL_ERR_RATIO_STATUS, - ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, + ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO_STATUS, + ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO_STATUS, + ApiPrecisionCompareColumn.ABS_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO_STATUS, + ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.ERROR_RATE_STATUS, ApiPrecisionCompareColumn.FINAL_RESULT, ApiPrecisionCompareColumn.ALGORITHM, ApiPrecisionCompareColumn.MESSAGE] @staticmethod -- Gitee From 00b32150c0530527952abd1dafbdc9e06b3f1a7f Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 11 Mar 2024 22:06:32 +0800 Subject: [PATCH 20/25] note --- .../api_accuracy_checker/compare/algorithm.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index b9e58f9a3a..111692d59f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -128,6 +128,16 @@ def get_abs_bench_with_eps(bench, dtype): return abs_bench, abs_bench_with_eps +''' +功能：新精度标准的绝对阈值法中，检查npu和golden输出的inf、nan是否一致 +输入： + inf_nan_mask：npu输出和golden输出的inf、nan的mask + bench_output：golden输出 + device_output：npu输出 + dtype：npu输出的dtype +输出： + inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例 +''' def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) golden_same_dtype = bench_output.astype(device_output.dtype) @@ -147,6 +157,15 @@ def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): return 0 if np.sum(inf_nan_mask) == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) +''' +功能：新精度标准的相对阈值法中，检查npu和golden小值域输出的相对误差是否满足阈值 +输入： + rel_err：npu输出和golden输出的相对误差 + normal_value_mask：npu输出和golden输出的正常值mask + rtol：相对误差的阈值 +输出： + rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例 +''' def check_small_value(abs_err, small_value_mask, small_value_atol): greater_mask = np.greater(abs_err, small_value_atol) err_mask = np.logical_and(greater_mask, small_value_mask) @@ -154,6 +173,15 @@ def check_small_value(abs_err, small_value_mask, small_value_atol): return 0 if np.sum(small_value_mask) == 0 else err_cnt / np.sum(small_value_mask) +''' +功能：新精度标准的绝对阈值法中，检查npu和golden正常值输出的绝对误差是否满足阈值 +输入： + abs_err：npu输出和golden输出的绝对误差 + normal_value_mask：npu输出和golden输出的正常值mask + atol：绝对误差的阈值 +输出： + abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例 +''' def check_norm_value(normal_value_mask, rel_err, rtol): err_mask = np.greater(rel_err, rtol) err_mask = np.logical_and(err_mask, normal_value_mask) -- Gitee From dccc642021ec49a1788e08f5e5b4a96897dbc6b6 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 12 Mar 2024 09:19:45 +0800 Subject: [PATCH 21/25] fix --- .../api_accuracy_checker/compare/compare_column.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index 89f1c63298..eab610c79b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -15,7 +15,7 @@ class CompareColumn: self.EB = CompareConst.NA self.RMSE = CompareConst.NA self.small_value_err_ratio = CompareConst.NA - self.Max_rel_error = CompareConst.NASPACE + self.Max_rel_error = CompareConst.NA self.Mean_rel_error = CompareConst.NA self.inf_nan_error_ratio = CompareConst.NA self.rel_err_ratio = CompareConst.NA -- Gitee From 9fa59bbcf6108aa562678e0cb4c354f8b0813c2f Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 12 Mar 2024 10:18:36 +0800 Subject: [PATCH 22/25] cleancode --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- .../api_accuracy_checker/compare/compare_column.py | 2 +- .../api_accuracy_checker/compare/compare_utils.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index f5e47e4e25..343360ec09 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -160,7 +160,7 @@ class Comparator: _, dedicated_api_name, _ = api_name.split("*") compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper fwd_success_status, fwd_compare_alg_results = compare_func(dedicated_api_name, bench_output, device_output) - bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(dedicated_api_name,bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(dedicated_api_name, bench_grad, npu_grad) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(dedicated_api_name, bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(dedicated_api_name, bench_grad, npu_grad) self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index eab610c79b..a1b2346750 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -40,7 +40,7 @@ class ApiPrecisionOutputColumn: self.mean_rel_err_ratio = CompareConst.SPACE self.mean_rel_err_status = CompareConst.SPACE self.eb_ratio = CompareConst.SPACE - self.eb_status = CompareConst.SPACE + self.eb_status = CompareConst.SPACE self.inf_nan_error_ratio = CompareConst.SPACE self.inf_nan_error_ratio_status = CompareConst.SPACE self.rel_err_ratio = CompareConst.SPACE diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 0906bfb814..34e4a6ba56 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -127,6 +127,7 @@ class ApiPrecisionCompareColumn: ApiPrecisionCompareColumn.ERROR_RATE, ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO, ApiPrecisionCompareColumn.REL_ERR_RATIO, ApiPrecisionCompareColumn.ABS_ERR_RATIO] + @staticmethod def get_detail_csv_title(): return [ApiPrecisionCompareColumn.API_NAME, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATIO, ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_STATUS, -- Gitee From bbeb902d0dccc39f2635e66478dd5a21b17c523e Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 12 Mar 2024 14:33:32 +0800 Subject: [PATCH 23/25] fix --- .../api_accuracy_checker/compare/algorithm.py | 56 +++++++++---------- .../compare/api_precision_compare.py | 55 +++++++++--------- .../api_accuracy_checker/compare/compare.py | 44 +++++++-------- 3 files changed, 79 insertions(+), 76 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 111692d59f..334ad42f08 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -128,17 +128,17 @@ def get_abs_bench_with_eps(bench, dtype): return abs_bench, abs_bench_with_eps -''' -功能：新精度标准的绝对阈值法中，检查npu和golden输出的inf、nan是否一致 -输入： - inf_nan_mask：npu输出和golden输出的inf、nan的mask - bench_output：golden输出 - device_output：npu输出 - dtype：npu输出的dtype -输出： - inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例 -''' def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): + ''' + 新精度标准的绝对阈值法中，检查npu和golden输出的inf、nan是否一致 + 输入： + inf_nan_mask：npu输出和golden输出的inf、nan的mask + bench_output：golden输出 + device_output：npu输出 + dtype：npu输出的dtype + 输出： + inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例 +''' abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) golden_same_dtype = bench_output.astype(device_output.dtype) a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN @@ -157,32 +157,32 @@ def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): return 0 if np.sum(inf_nan_mask) == 0 else inf_nan_err_cnt / np.sum(inf_nan_mask) -''' -功能：新精度标准的相对阈值法中，检查npu和golden小值域输出的相对误差是否满足阈值 -输入： - rel_err：npu输出和golden输出的相对误差 - normal_value_mask：npu输出和golden输出的正常值mask - rtol：相对误差的阈值 -输出： - rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例 -''' def check_small_value(abs_err, small_value_mask, small_value_atol): + ''' + 新精度标准的相对阈值法中，检查npu和golden小值域输出的相对误差是否满足阈值 + 输入： + rel_err：npu输出和golden输出的相对误差 + normal_value_mask：npu输出和golden输出的正常值mask + rtol：相对误差的阈值 + 输出： + rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例 +''' greater_mask = np.greater(abs_err, small_value_atol) err_mask = np.logical_and(greater_mask, small_value_mask) err_cnt = np.sum(err_mask) return 0 if np.sum(small_value_mask) == 0 else err_cnt / np.sum(small_value_mask) -''' -功能：新精度标准的绝对阈值法中，检查npu和golden正常值输出的绝对误差是否满足阈值 -输入： - abs_err：npu输出和golden输出的绝对误差 - normal_value_mask：npu输出和golden输出的正常值mask - atol：绝对误差的阈值 -输出： - abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例 -''' def check_norm_value(normal_value_mask, rel_err, rtol): + ''' + 新精度标准的绝对阈值法中，检查npu和golden正常值输出的绝对误差是否满足阈值 + 输入： + abs_err：npu输出和golden输出的绝对误差 + normal_value_mask：npu输出和golden输出的正常值mask + atol：绝对误差的阈值 + 输出： + abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例 +''' err_mask = np.greater(rel_err, rtol) err_mask = np.logical_and(err_mask, normal_value_mask) err_cnt = np.sum(err_mask) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index e10fc103ae..ef93def686 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -159,31 +159,32 @@ def analyse_csv(npu_data, gpu_data, config): for _, row_npu in npu_data.iterrows(): message = '' compare_column = ApiPrecisionOutputColumn() - part_api_name = row_npu[ApiPrecisionCompareColumn.API_NAME] - row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == part_api_name] - api_name, direction_status, _, _ = part_api_name.split(".") + full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] + row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] + full_api_name, direction_status, _, _ = full_api_name_with_direction_status.split(".") if row_gpu.empty: - print_warn_log(f'This API : {part_api_name} does not exist in the GPU data.') + print_warn_log(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') continue if len(row_gpu) > 1: - msg = f'This API : {part_api_name} has multiple records in the GPU data.' + msg = f'This API : {full_api_name_with_direction_status} has multiple records in the GPU data.' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] + #当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 if pd.isna(row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]): continue - _, dedicated_api_name, _ = api_name.split("*") + _, dedicated_api_name, _ = full_api_name.split("*") new_status = CompareConst.NA - compare_column.api_name = part_api_name + compare_column.api_name = full_api_name_with_direction_status if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST: new_status = record_binary_consistency_result(compare_column, row_npu) elif dedicated_api_name in AbsoluteStandardApiName: new_status = record_absolute_threshold_result(compare_column, row_npu) elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: - bs = BenchmarkStandard(part_api_name, row_npu, row_gpu) + bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) new_status = record_benchmark_compare_result(compare_column, bs) write_detail_csv(compare_column.to_column_value(), config.details_csv_path) - if last_api_name is not None and api_name != last_api_name: + if last_api_name is not None and full_api_name != last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) @@ -197,7 +198,7 @@ def analyse_csv(npu_data, gpu_data, config): message = '' is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST - last_api_name = api_name + last_api_name = full_api_name last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] if not is_supported: @@ -238,12 +239,15 @@ def get_absolute_threshold_result(row_npu): else: absolute_threshold_result = CompareConst.PASS - return [ - inf_nan_error_ratio, inf_nan_result, - rel_err_ratio, rel_err_result, - abs_err_ratio, abs_err_result, - absolute_threshold_result - ] + return { + "inf_nan_error_ratio": inf_nan_error_ratio, + "inf_nan_result": inf_nan_result, + "rel_err_ratio": rel_err_ratio, + "rel_err_result": rel_err_result, + "abs_err_ratio": abs_err_ratio, + "abs_err_result": abs_err_result, + "absolute_threshold_result": absolute_threshold_result, + } def get_api_checker_result(status): @@ -274,16 +278,15 @@ def record_binary_consistency_result(compare_column, row_npu): def record_absolute_threshold_result(compare_column, row_npu): absolute_threshold_result = get_absolute_threshold_result(row_npu) - compare_column.inf_nan_error_ratio = absolute_threshold_result[0] - compare_column.inf_nan_error_ratio_status = absolute_threshold_result[1] - compare_column.rel_err_ratio = absolute_threshold_result[2] - compare_column.rel_err_ratio_status = absolute_threshold_result[3] - compare_column.abs_err_ratio = absolute_threshold_result[4] - compare_column.abs_err_ratio_status = absolute_threshold_result[5] - new_status = absolute_threshold_result[6] - compare_column.compare_result = new_status + compare_column.inf_nan_error_ratio = absolute_threshold_result["inf_nan_error_ratio"] + compare_column.inf_nan_error_ratio_status = absolute_threshold_result["inf_nan_result"] + compare_column.rel_err_ratio = absolute_threshold_result["rel_err_ratio"] + compare_column.rel_err_ratio_status = absolute_threshold_result["rel_err_result"] + compare_column.abs_err_ratio = absolute_threshold_result["abs_err_ratio"] + compare_column.abs_err_ratio_status = absolute_threshold_result["abs_err_result"] + compare_column.compare_result = absolute_threshold_result["absolute_threshold_result"] compare_column.algorithm = "绝对阈值法" - return new_status + return compare_column.compare_result def record_benchmark_compare_result(compare_column, bs): @@ -300,7 +303,7 @@ def record_benchmark_compare_result(compare_column, bs): compare_column.eb_status = bs.eb_status compare_column.compare_result = bs.final_result compare_column.algorithm = "标杆比对法" - return bs.final_result + return compare_column.compare_result def _api_precision_compare(parser=None): diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 343360ec09..0440b32f3d 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -156,15 +156,15 @@ class Comparator: self.write_summary_csv(args) self.write_detail_csv(args) - def compare_output(self, api_name, bench_output, device_output, bench_grad=None, npu_grad=None): - _, dedicated_api_name, _ = api_name.split("*") - compare_func = self._compare_dropout if "dropout" in api_name else self._compare_core_wrapper - fwd_success_status, fwd_compare_alg_results = compare_func(dedicated_api_name, bench_output, device_output) - bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(dedicated_api_name, bench_grad[0], npu_grad[0]) if "dropout" in api_name else compare_func(dedicated_api_name, bench_grad, npu_grad) - self.record_results(api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) + def compare_output(self, full_api_name, bench_output, device_output, bench_grad=None, npu_grad=None): + _, api_name, _ = full_api_name.split("*") + compare_func = self._compare_dropout if "dropout" in full_api_name else self._compare_core_wrapper + fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) + bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(api_name, bench_grad[0], npu_grad[0]) if "dropout" in full_api_name else compare_func(api_name, bench_grad, npu_grad) + self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS - def _compare_core_wrapper(self, dedicated_api_name, bench_output, device_output): + def _compare_core_wrapper(self, api_name, bench_output, device_output): detailed_result_total = [] test_final_success = CompareConst.PASS if isinstance(bench_output, (list, tuple)): @@ -174,12 +174,12 @@ class Comparator: message = ["bench and npu output structure is different."] else: for b_out_i, n_out_i in zip(bench_output, device_output): - status_i, compare_result_i, message_i = self._compare_core(dedicated_api_name, b_out_i, n_out_i) + status_i, compare_result_i, message_i = self._compare_core(api_name, b_out_i, n_out_i) status.append(status_i) compare_result.append(compare_result_i) message.append(message_i) else: - status, compare_result, message = self._compare_core(dedicated_api_name, bench_output, device_output) + status, compare_result, message = self._compare_core(api_name, bench_output, device_output) if not isinstance(status, list): detailed_result_total.append(compare_result.to_column_value(status, message)) if status == CompareConst.ERROR: @@ -195,7 +195,7 @@ class Comparator: test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total - def _compare_core(self, dedicated_api_name, bench_output, device_output): + def _compare_core(self, api_name, bench_output, device_output): compare_column = CompareColumn() if not isinstance(bench_output, type(device_output)): return CompareConst.ERROR, compare_column, "bench and npu output type is different." @@ -204,7 +204,7 @@ class Comparator: if b_keys != n_keys: return CompareConst.ERROR, compare_column, "bench and npu output dict keys are different." else: - status, compare_result, message = self._compare_core(dedicated_api_name, list(bench_output.values()), + status, compare_result, message = self._compare_core(api_name, list(bench_output.values()), list(device_output.values())) elif isinstance(bench_output, torch.Tensor): copy_bench_out = bench_output.detach().clone() @@ -212,7 +212,7 @@ class Comparator: compare_column.bench_type = str(copy_bench_out.dtype) compare_column.npu_type = str(copy_device_output.dtype) compare_column.shape = tuple(device_output.shape) - status, compare_result, message = self._compare_torch_tensor(dedicated_api_name, copy_bench_out, copy_device_output, + status, compare_result, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, compare_column) elif isinstance(bench_output, (bool, int, float, str)): compare_column.bench_type = str(type(bench_output)) @@ -226,7 +226,7 @@ class Comparator: return status, compare_result, message - def _compare_torch_tensor(self, dedicated_api_name, bench_output, device_output, compare_column): + def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape npu_dtype = device_output.dtype @@ -251,19 +251,19 @@ class Comparator: compare_column.error_rate = err_rate return status, compare_column, message else: - status, compare_column, message = self._compare_float_tensor(dedicated_api_name, bench_output, device_output, + status, compare_column, message = self._compare_float_tensor(api_name, bench_output, device_output, compare_column, npu_dtype) return status, compare_column, message - def _compare_float_tensor(self, dedicated_api_name, bench_output, device_output, compare_column, dtype): + def _compare_float_tensor(self, api_name, bench_output, device_output, compare_column, dtype): message = "" abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - if dedicated_api_name in AbsoluteStandardApiName: + if api_name in AbsoluteStandardApiName: small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( - dedicated_api_name, str(dtype)) + api_name, str(dtype)) rel_err = abs_err / abs_bench_with_eps small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold) normal_value_mask = np.logical_and(both_finite_mask, np.logical_not(small_value_mask)) @@ -314,7 +314,7 @@ class Comparator: return CompareConst.PASS, compare_column, message @staticmethod - def _compare_dropout(dedicated_api_name, bench_output, device_output): + def _compare_dropout(api_name, bench_output, device_output): tensor_num = bench_output.numel() if tensor_num >= 100: if abs((bench_output == 0).sum() - (device_output == 0).cpu().sum()) / tensor_num < 0.1: @@ -344,8 +344,8 @@ class Comparator: return error_rate, result, "" @staticmethod - def _get_absolute_threshold_attribute(dedicated_api_name, dtype): - small_value_threshold = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('small_value') - small_value_atol = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('small_value_atol') - rtol = AbsoluteStandardApi.get(dedicated_api_name).get(dtype).get('rtol') + def _get_absolute_threshold_attribute(api_name, dtype): + small_value_threshold = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value') + small_value_atol = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value_atol') + rtol = AbsoluteStandardApi.get(api_name).get(dtype).get('rtol') return small_value_threshold, small_value_atol, rtol -- Gitee From 30b5d62f8f7c9efc0ea709e79eb72eab487f9547 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 12 Mar 2024 14:51:39 +0800 Subject: [PATCH 24/25] fix --- .../api_accuracy_checker/compare/algorithm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 334ad42f08..560d35da83 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -138,7 +138,7 @@ def check_inf_nan_value(inf_nan_mask, bench_output, device_output, dtype, rtol): dtype：npu输出的dtype 输出： inf_nan_err_ratio：npu输出和golden输出的inf、nan不一致的比例 -''' + ''' abs_gpu, abs_gpu_with_eps = get_abs_bench_with_eps(bench_output, dtype) golden_same_dtype = bench_output.astype(device_output.dtype) a_min = np.finfo(device_output.dtype).min if dtype != torch.bfloat16 else CompareConst.BFLOAT16_MIN @@ -166,7 +166,7 @@ def check_small_value(abs_err, small_value_mask, small_value_atol): rtol：相对误差的阈值 输出： rel_err_ratio：npu输出和golden输出的相对误差不满足阈值的比例 -''' + ''' greater_mask = np.greater(abs_err, small_value_atol) err_mask = np.logical_and(greater_mask, small_value_mask) err_cnt = np.sum(err_mask) @@ -182,7 +182,7 @@ def check_norm_value(normal_value_mask, rel_err, rtol): atol：绝对误差的阈值 输出： abs_err_ratio：npu输出和golden输出的绝对误差不满足阈值的比例 -''' + ''' err_mask = np.greater(rel_err, rtol) err_mask = np.logical_and(err_mask, normal_value_mask) err_cnt = np.sum(err_mask) -- Gitee From f43ebef41fbe827063276edc324372b4c31c2e62 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 12 Mar 2024 15:01:46 +0800 Subject: [PATCH 25/25] fix --- .../compare/api_precision_compare.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index ef93def686..b86402d05a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -278,13 +278,13 @@ def record_binary_consistency_result(compare_column, row_npu): def record_absolute_threshold_result(compare_column, row_npu): absolute_threshold_result = get_absolute_threshold_result(row_npu) - compare_column.inf_nan_error_ratio = absolute_threshold_result["inf_nan_error_ratio"] - compare_column.inf_nan_error_ratio_status = absolute_threshold_result["inf_nan_result"] - compare_column.rel_err_ratio = absolute_threshold_result["rel_err_ratio"] - compare_column.rel_err_ratio_status = absolute_threshold_result["rel_err_result"] - compare_column.abs_err_ratio = absolute_threshold_result["abs_err_ratio"] - compare_column.abs_err_ratio_status = absolute_threshold_result["abs_err_result"] - compare_column.compare_result = absolute_threshold_result["absolute_threshold_result"] + compare_column.inf_nan_error_ratio = absolute_threshold_result.get("inf_nan_error_ratio") + compare_column.inf_nan_error_ratio_status = absolute_threshold_result.get("inf_nan_result") + compare_column.rel_err_ratio = absolute_threshold_result.get("rel_err_ratio") + compare_column.rel_err_ratio_status = absolute_threshold_result.get("rel_err_result") + compare_column.abs_err_ratio = absolute_threshold_result.get("abs_err_ratio") + compare_column.abs_err_ratio_status = absolute_threshold_result.get("abs_err_result") + compare_column.compare_result = absolute_threshold_result.get("absolute_threshold_result") compare_column.algorithm = "绝对阈值法" return compare_column.compare_result -- Gitee