diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_column.py index b1cbc3234682e9106a38301e0b8035cca74f010b..5058bf7e7f72a6607ad1f5525e0e153473a76664 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_column.py @@ -88,4 +88,9 @@ class ApiPrecisionOutputColumn: self.abs_err_ratio_status, self.error_rate, self.error_rate_status, self.mean_ulp_err, self.ulp_err_proportion, self.ulp_err_proportion_ratio, self.ulp_err_status, self.rel_err_thousandth, self.rel_err_thousandth_status, self.compare_result, self.compare_algorithm, self.compare_message] - \ No newline at end of file + + def update(self, metrics): + for key, value in metrics.items(): + if value is None: + continue + setattr(self, key, value) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/basecompare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/basecompare.py new file mode 100644 index 0000000000000000000000000000000000000000..2044742025df37c58c99aa75a81588438a83e7be --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/basecompare.py @@ -0,0 +1,166 @@ +import torch +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ApiPrecisionCompareColumn, convert_str_to_float, check_inf_or_nan, \ + is_inf_or_nan + +class BaseConfig: + _small_value = { + torch.float16: 1e-3, + torch.bfloat16: 1e-3, + torch.float32: 1e-6, + "default": 1e-6 + } + _small_value_atol = { + torch.float16: 1e-5, + torch.bfloat16: 1e-5, + torch.float32: 1e-9, + "default": 1e-9 + } + _rtol = { + torch.float16: 1e-3, + torch.bfloat16: 4e-3, + torch.float32: 1e-6, + "default": 1e-6 # 默认值也放在配置类中 + } + + _small_value_threshold = { + 'error_threshold': 2, + 'warning_threshold': 1, + "default": 1 + } + _rmse_threshold = { + 'error_threshold': 2, + 'warning_threshold': 1, + "default": 1 + } + _max_rel_err_threshold = { + 'error_threshold': 10, + 'warning_threshold': 1, + "default": 1 + } + _mean_rel_err_threshold = { + 'error_threshold': 2, + 'warning_threshold': 1, + "default": 1 + } + _eb_threshold ={ + 'error_threshold': 2, + 'warning_threshold': 1, + "default": 1 + } + + _fp32_mean_ulp_err_threshold = 64 + ulp_err_proportion_ratio = 1 + _fp32_ulp_err_proportion = 0.05 + _fp16_ulp_err_proportion = 0.001 + + @classmethod + def get_small_valuel(cls, dtype): + return cls._small_value.get(dtype, cls._small_value["default"]) + + @classmethod + def get_small_value_atol(cls, dtype): + return cls._small_value_atol.get(dtype, cls._small_value_atol["default"]) + + @classmethod + def get_rtol(cls, dtype): + return cls._rtol.get(dtype, cls._rtol["default"]) + + @classmethod + def get_small_value_threshold(cls, threshold_type): + return cls._small_value_threshold.get(threshold_type, cls._small_value_threshold["default"]) + + @classmethod + def get_rmse_threshold(cls, threshold_type): + return cls._rmse_threshold.get(threshold_type, cls._rmse_threshold["default"]) + + @classmethod + def get_max_rel_err_threshold(cls, threshold_type): + return cls._max_rel_err_threshold.get(threshold_type, cls._max_rel_err_threshold["default"]) + + @classmethod + def get_mean_rel_err_threshold(cls, threshold_type): + return cls._mean_rel_err_threshold.get(threshold_type, cls._mean_rel_err_threshold["default"]) + + @classmethod + def get_eb_threshold(cls, threshold_type): + return cls._eb_threshold.get(threshold_type, cls._eb_threshold["default"]) + + def get_benchmark_threshold(metric): + metric_threshold_functions = { + 'small_value': BaseConfig.get_small_value_threshold, + 'rmse': BaseConfig.get_rmse_threshold, + 'max_rel_err': BaseConfig.get_max_rel_err_threshold, + 'mean_rel_err': BaseConfig.get_mean_rel_err_threshold, + 'eb': BaseConfig.get_eb_threshold + } + + threshold_func = metric_threshold_functions.get(metric) + return threshold_func('error_threshold'), threshold_func('warning_threshold') + + @classmethod + def get_fp32_mean_ulp_err_threshold(cls): + return cls._fp32_mean_ulp_err_threshold + + @classmethod + def get_ulp_err_proportion_ratio_threshold(cls): + return cls.ulp_err_proportion_ratio + + @classmethod + def get_fp32_ulp_err_proportion_threshold(cls): + return cls._fp32_ulp_err_proportion + + @classmethod + def get_fp16_ulp_err_proportion_threshold(cls): + return cls._fp16_ulp_err_proportion + + def get_ulp_threshold(self, dtype): + if dtype == torch.float32: + mean_ulp_err_threshold = BaseConfig.get_fp32_mean_ulp_err_threshold() + ulp_err_proportion_threshold = BaseConfig.get_fp32_ulp_err_proportion_threshold() + ulp_err_proportion_ratio_threshold = BaseConfig.get_ulp_err_proportion_ratio_threshold() + return mean_ulp_err_threshold, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold + else: + ulp_err_proportion_threshold = BaseConfig.get_fp16_ulp_err_proportion_threshold() + ulp_err_proportion_ratio_threshold = BaseConfig.get_ulp_err_proportion_ratio_threshold() + return None, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold + + + +class BasePrecisionCompare: + def __init__(self, input_data): + self.api_name = input_data.api_name + self.npu_precision = input_data.npu_precision + self.gpu_precision = input_data.gpu_precision + self.compare_column = input_data.compare_column + + + def _get_and_convert_values(self, column_name): + npu_value = self.npu_precision.get(column_name) + gpu_value = self.gpu_precision.get(column_name) + npu_value = convert_str_to_float(npu_value) + gpu_value = convert_str_to_float(gpu_value) + return npu_value, gpu_value + + def get_status(self, metrics, inf_nan_consistency): + pass + + def get_final_status(self, status_list): + if CompareConst.ERROR in status_list: + compare_result = CompareConst.ERROR + elif CompareConst.WARNING in status_list: + compare_result = CompareConst.WARNING + return compare_result + + def compute_ratio(self): + pass + + def post_compare(self, metrics, inf_nan_consistency): + compare_result, status_dict = self.get_status(metrics, inf_nan_consistency) + metrics.update(status_dict) + metrics.update({'compare_result': compare_result}) + self.compare_column.update(metrics) + + def compare(self): + metrics, inf_nan_consistency = self.compute_ratio() + self.post_compare(metrics, inf_nan_consistency) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/benchmark.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..ff1f6b0a47e4d3ce182d95fe07e1cf33d498f084 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/benchmark.py @@ -0,0 +1,306 @@ +import math +from collections import namedtuple + +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.pytorch.api_accuracy_checker.standard.basecompare import BasePrecisionCompare, BaseConfig +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ApiPrecisionCompareColumn, convert_str_to_float, check_inf_or_nan, \ + is_inf_or_nan + + +benchmark_algorithms_thresholds = { + 'small_value': { + 'error_threshold': 2, + 'warning_threshold': 1 + }, + 'rmse': { + 'error_threshold': 2, + 'warning_threshold': 1 + }, + 'max_rel_err': { + 'error_threshold': 10, + 'warning_threshold': 1 + }, + 'mean_rel_err': { + 'error_threshold': 2, + 'warning_threshold': 1 + }, + 'eb': { + 'error_threshold': 2, + 'warning_threshold': 1 + } +} + +BenchmarkInfNanConsistency = namedtuple('BenchmarkInfNanConsistency', ['small_value_inf_nan_consistency', + 'rmse_inf_nan_consistency', + 'max_rel_inf_nan_consistency', + 'mean_rel_inf_nan_consistency', + 'eb_inf_nan_consistency']) + +class Standard: + @staticmethod + def _calc_ratio(column_name, x, y, default_value): + ''' + 计算npu侧和gpu侧统计量的比值 + 输入： + column_name：统计量名称 + x：npu侧统计量 + y：gpu侧统计量 + default：当x不接近0，y接近0，设置的比值默认值 + 输出： + ratio：统计量x和y的比值 + inf_nan_consistency：不出现inf或nan时为True，出现inf或nan时必须同时为inf或-inf或nan才为True，否则为False + message：当出现inf或nan时的提示信息 + ''' + x, y = convert_str_to_float(x), convert_str_to_float(y) + + if is_inf_or_nan(x) or is_inf_or_nan(y): + return check_inf_or_nan(x, y, column_name) + + inf_nan_consistency = True + message = "" + if math.isclose(y, 0.0): + if math.isclose(x, 0.0): + return 1.0, inf_nan_consistency, message + else: + return default_value, inf_nan_consistency, message + else: + return abs(x / y), inf_nan_consistency, message + + +class BenchmarkStandard(Standard): + def __init__(self, api_name, npu_precision, gpu_precision): + self.api_name = api_name + self.npu_precision = npu_precision + self.gpu_precision = gpu_precision + self.small_value_err_ratio = 1 + self.rmse_ratio = 1 + self.max_rel_err_ratio = 1 + self.mean_rel_err_ratio = 1 + self.eb_ratio = 1 + self.small_value_err_status = CompareConst.PASS + self.rmse_status = CompareConst.PASS + self.max_rel_err_status = CompareConst.PASS + self.mean_rel_err_status = CompareConst.PASS + self.eb_status = CompareConst.PASS + self.check_result_list = [] + self.final_result = CompareConst.PASS + self.compare_message = "" + + def __str__(self): + return "%s" % (self.api_name) + + @staticmethod + def _get_status(ratio, algorithm): + if math.isnan(ratio) or math.isinf(ratio): + return CompareConst.PASS + error_threshold = benchmark_algorithms_thresholds.get(algorithm, {}).get('error_threshold', DEFAULT_THRESHOLD) + warning_threshold = benchmark_algorithms_thresholds.get(algorithm, {}).get('warning_threshold', + DEFAULT_THRESHOLD) + if ratio > error_threshold: + return CompareConst.ERROR + elif ratio > warning_threshold: + return CompareConst.WARNING + return CompareConst.PASS + + def get_result(self): + inf_nan_consistency = self._compare_ratio() + small_value_inf_nan_consistency = inf_nan_consistency.small_value_inf_nan_consistency + rmse_inf_nan_consistency = inf_nan_consistency.rmse_inf_nan_consistency + max_rel_inf_nan_consistency = inf_nan_consistency.max_rel_inf_nan_consistency + mean_rel_inf_nan_consistency = inf_nan_consistency.mean_rel_inf_nan_consistency + eb_inf_nan_consistency = inf_nan_consistency.eb_inf_nan_consistency + self.small_value_err_status = self._get_status(self.small_value_err_ratio, 'small_value') if \ + small_value_inf_nan_consistency else CompareConst.ERROR + self.check_result_list.append(self.small_value_err_status) + self.rmse_status = self._get_status(self.rmse_ratio, 'rmse') if rmse_inf_nan_consistency \ + else CompareConst.ERROR + self.check_result_list.append(self.rmse_status) + self.max_rel_err_status = self._get_status( + self.max_rel_err_ratio, 'max_rel_err') if max_rel_inf_nan_consistency else CompareConst.ERROR + self.check_result_list.append(self.max_rel_err_status) + self.mean_rel_err_status = self._get_status( + self.mean_rel_err_ratio, 'mean_rel_err') if mean_rel_inf_nan_consistency else CompareConst.ERROR + self.check_result_list.append(self.mean_rel_err_status) + self.eb_status = self._get_status(self.eb_ratio, 'eb') + if CompareConst.ERROR in self.check_result_list: + self.final_result = CompareConst.ERROR + elif CompareConst.WARNING in self.check_result_list: + self.final_result = CompareConst.WARNING + + def to_column_value(self): + return [self.small_value_err_ratio, self.small_value_err_status, self.rmse_ratio, + self.rmse_status, self.max_rel_err_ratio, self.max_rel_err_status, self.mean_rel_err_ratio, + self.mean_rel_err_status, self.eb_ratio, self.eb_status] + + def _compare_ratio(self): + + self.small_value_err_ratio, small_value_inf_nan_consistency, small_value_message = self._calc_ratio( + ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE, + self.npu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE), + self.gpu_precision.get(ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE), 10000.0) + self.compare_message += small_value_message + self.rmse_ratio, rmse_inf_nan_consistency, rmse_message = self._calc_ratio(ApiPrecisionCompareColumn.RMSE, + self.npu_precision.get(ApiPrecisionCompareColumn.RMSE), + self.gpu_precision.get(ApiPrecisionCompareColumn.RMSE), 10000.0) + self.compare_message += rmse_message + self.max_rel_err_ratio, max_rel_inf_nan_consistency, max_rel_message = self._calc_ratio( + ApiPrecisionCompareColumn.MAX_REL_ERR, + self.npu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MAX_REL_ERR), 10000.0) + self.compare_message += max_rel_message + self.mean_rel_err_ratio, mean_rel_inf_nan_consistency, mean_rel_message = self._calc_ratio( + ApiPrecisionCompareColumn.MEAN_REL_ERR, + self.npu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR), + self.gpu_precision.get(ApiPrecisionCompareColumn.MEAN_REL_ERR), 10000.0) + self.compare_message += mean_rel_message + self.eb_ratio, eb_inf_nan_consistency, eb_message = self._calc_ratio(ApiPrecisionCompareColumn.EB, + self.npu_precision.get(ApiPrecisionCompareColumn.EB), + self.gpu_precision.get(ApiPrecisionCompareColumn.EB), 10000.0) + self.compare_message += eb_message + + return BenchmarkInfNanConsistency(small_value_inf_nan_consistency, rmse_inf_nan_consistency, + max_rel_inf_nan_consistency, mean_rel_inf_nan_consistency, + eb_inf_nan_consistency) + + +def _calc_ratio(x, y, default_value): + ''' + 计算npu侧和gpu侧统计量的比值 + 输入： + column_name：统计量名称 + x：npu侧统计量 + y：gpu侧统计量 + default：当x不接近0，y接近0，设置的比值默认值 + 输出： + ratio：统计量x和y的比值 + ''' + + message = "" + if math.isclose(y, 0.0): + if math.isclose(x, 0.0): + return 1.0 + else: + return default_value + else: + return abs(x / y) + +class BenchmarkStandard(BasePrecisionCompare): + def __init__(self, input_data): + super().__init__(input_data) + + + def _compute_small_value_err_ratio(self): + column_name = ApiPrecisionCompareColumn.SMALL_VALUE_ERROR_RATE + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_rmse_ratio(self): + column_name = ApiPrecisionCompareColumn.RMSE + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_max_rel_err_ratio(self): + column_name = ApiPrecisionCompareColumn.MAX_REL_ERR + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_mean_rel_err_ratio(self): + column_name = ApiPrecisionCompareColumn.MEAN_REL_ERR + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_eb_ratio(self): + column_name = ApiPrecisionCompareColumn.EB + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_ratio(self): + compare_message = "" + small_value_err_ratio, small_value_inf_nan_consistency, small_value_message = self._compute_small_value_err_ratio() + compare_message += small_value_message + rmse_ratio, rmse_inf_nan_consistency, rmse_message = self._compute_rmse_ratio() + compare_message += rmse_message + max_rel_err_ratio, max_rel_inf_nan_consistency, max_rel_message = self._compute_max_rel_err_ratio() + compare_message += max_rel_message + mean_rel_err_ratio, mean_rel_inf_nan_consistency, mean_rel_message = self._compute_mean_rel_err_ratio() + compare_message += mean_rel_message + eb_ratio, eb_inf_nan_consistency, eb_message = self._compute_eb_ratio() + compare_message += eb_message + + metrics = { + "small_value_err_ratio": small_value_err_ratio, + "rmse_ratio": rmse_ratio, + "max_rel_err_ratio": max_rel_err_ratio, + "mean_rel_err_ratio": mean_rel_err_ratio, + "eb_ratio": eb_ratio, + "compare_message": compare_message + } + + return metrics, \ + BenchmarkInfNanConsistency(small_value_inf_nan_consistency, rmse_inf_nan_consistency, + max_rel_inf_nan_consistency, mean_rel_inf_nan_consistency, + eb_inf_nan_consistency) + + + def _get_threshold(self, metric): + error_threshold, warning_threshold = BaseConfig.get_threshold(metric) + return error_threshold, warning_threshold + + + def _get_single_metric_status(self, ratio, metric): + if math.isnan(ratio) or math.isinf(ratio): + return CompareConst.PASS + error_threshold, warning_threshold = self._get_threshold(metric) + if ratio > error_threshold: + return CompareConst.ERROR + elif ratio > warning_threshold: + return CompareConst.WARNING + return CompareConst.PASS + + def _get_status(self, metrics, inf_nan_consistency): + small_value_inf_nan_consistency = inf_nan_consistency.small_value_inf_nan_consistency + rmse_inf_nan_consistency = inf_nan_consistency.rmse_inf_nan_consistency + max_rel_inf_nan_consistency = inf_nan_consistency.max_rel_inf_nan_consistency + mean_rel_inf_nan_consistency = inf_nan_consistency.mean_rel_inf_nan_consistency + eb_inf_nan_consistency = inf_nan_consistency.eb_inf_nan_consistency + small_value_err_ratio = metrics.get("small_value_err_ratio") + rmse_ratio = metrics.get("rmse_ratio") + max_rel_err_ratio = metrics.get("max_rel_err_ratio") + mean_rel_err_ratio = metrics.get("mean_rel_err_ratio") + eb_ratio = metrics.get("eb_ratio") + + small_value_err_status = self._get_single_metric_status(small_value_err_ratio, 'small_value') \ + if small_value_inf_nan_consistency else CompareConst.ERROR + rmse_status = self._get_single_metric_status(rmse_ratio, 'rmse') \ + if rmse_inf_nan_consistency else CompareConst.ERROR + max_rel_err_status = self._get_single_metric_status(max_rel_err_ratio,'max_rel_err') \ + if max_rel_inf_nan_consistency else CompareConst.ERROR + mean_rel_err_status = self._get_single_metric_status(mean_rel_err_ratio,'mean_rel_err') \ + if mean_rel_inf_nan_consistency else CompareConst.ERROR + eb_status = self._get_single_metric_status(eb_ratio, 'eb') \ + if eb_inf_nan_consistency else CompareConst.ERROR + status_list = [small_value_err_status, rmse_status, max_rel_err_status, mean_rel_err_status, eb_status] + compare_result = self.get_final_status(status_list) + status_dict = { + "small_value_err_status": small_value_err_status, + "rmse_status": rmse_status, + "max_rel_err_status": max_rel_err_status, + "mean_rel_err_status": mean_rel_err_status, + "eb_status": eb_status + } + return compare_result, status_dict diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/compare_input.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/compare_input.py new file mode 100644 index 0000000000000000000000000000000000000000..e63a129c1a136ef9043d768ec41d5b0e3f4f8db9 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/compare_input.py @@ -0,0 +1,10 @@ + + + +class PrecisionCompareInput: + def __init__(self, api_name, npu_precision, gpu_precision, compare_column): + self.api_name = api_name + self.npu_precision = npu_precision + self.gpu_precision = gpu_precision + self.compare_column = compare_column + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/register.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/register.py new file mode 100644 index 0000000000000000000000000000000000000000..4c0637f8e71afac90cd0e96c25c78d7954d4e2a0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/register.py @@ -0,0 +1,49 @@ + +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import absolute_standard_api, binary_standard_api, \ + ulp_standard_api, thousandth_standard_api + + +class ComparisonRegistry: + def __init__(self): + self.comparison_functions = {} + self.standard_categories = { + 'absolute_threshold': absolute_standard_api, + 'binary_consistency': binary_standard_api, + 'ulp_compare': ulp_standard_api, + 'thousandth_threshold': thousandth_standard_api + } + + def register(self, standard, func): + self.comparison_functions[standard] = func + + def get_standard_category(self, api_name): + # 遍历字典，确定api_name属于哪个类别 + for name, category in self.standard_categories.items(): + if api_name in category: + return name + return "benchmark" + + + def get_comparison_function(self, api_name): + standard = self.get_standard_category(api_name) + return self.comparison_functions.get(standard) + +# 创建一个比较注册器 +# registry = ComparisonRegistry() + +# # 注册比较函数 +# registry.register('thousandth', record_thousandth_threshold_result) +# registry.register('binary', record_binary_consistency_result) +# registry.register('absolute', record_absolute_threshold_result) +# registry.register('ulp', record_ulp_compare_result) + +# def compare_api(api_name, compare_column, row_npu): +# new_status = None + +# # 获取比较函数 +# comparison_func = registry.get_comparison_function(api_name) + +# if comparison_func: +# new_status = comparison_func(compare_column, row_npu) + +# return new_status \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/ulp.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/ulp.py new file mode 100644 index 0000000000000000000000000000000000000000..2e294967bcff5f30fba4a981f29ed963c0fe9a81 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/standard/ulp.py @@ -0,0 +1,146 @@ +import math +import torch +from collections import namedtuple + +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.pytorch.api_accuracy_checker.standard.basecompare import BasePrecisionCompare, BaseConfig +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ApiPrecisionCompareColumn, convert_str_to_float, check_inf_or_nan, \ + is_inf_or_nan + + +UlpInfNanConsistency = namedtuple('UlpInfNanConsistency', ['ulp_inf_nan_consistency']) + +class ULPStandard(Standard): + def __init__(self, api_name, npu_precision, gpu_precision): + self.api_name = api_name + self.npu_precision = npu_precision + self.gpu_precision = gpu_precision + self.mean_ulp_err = 0 + self.ulp_err_proportion = 0 + self.ulp_err_proportion_ratio = 1 + self.ulp_err_status = CompareConst.PASS + self.compare_message = "" + + def __str__(self): + return f"{self.api_name}" + + def get_result(self): + self.mean_ulp_err = convert_str_to_float(self.npu_precision.get(ApiPrecisionCompareColumn.MEAN_ULP_ERR)) + gpu_mean_ulp_err = convert_str_to_float(self.gpu_precision.get(ApiPrecisionCompareColumn.MEAN_ULP_ERR)) + inf_nan_consistency = True + if is_inf_or_nan(self.mean_ulp_err) or is_inf_or_nan(gpu_mean_ulp_err): + _, inf_nan_consistency, message = check_inf_or_nan(self.mean_ulp_err, gpu_mean_ulp_err, + ApiPrecisionCompareColumn.MEAN_ULP_ERR) + self.compare_message += message + self.ulp_err_proportion = convert_str_to_float( + self.npu_precision.get(ApiPrecisionCompareColumn.ULP_ERR_PROPORTION)) + self.ulp_err_proportion_ratio, ulp_inf_nan_consistency, message = self._calc_ratio( + ApiPrecisionCompareColumn.ULP_ERR_PROPORTION, + self.npu_precision.get(ApiPrecisionCompareColumn.ULP_ERR_PROPORTION), + self.gpu_precision.get(ApiPrecisionCompareColumn.ULP_ERR_PROPORTION), 10000.0) + inf_nan_consistency = inf_nan_consistency and ulp_inf_nan_consistency + self.compare_message += message + if inf_nan_consistency: + self.ulp_err_status = self._get_ulp_status(self.npu_precision.get(ApiPrecisionCompareColumn.DEVICE_DTYPE)) + else: + self.ulp_err_status = CompareConst.ERROR + + def _get_ulp_status(self, dtype): + if dtype == torch.float32: + if self.mean_ulp_err < 64: + return CompareConst.PASS + elif self.ulp_err_proportion < 0.05: + return CompareConst.PASS + elif self.ulp_err_proportion_ratio < 1: + return CompareConst.PASS + else: + self.compare_message += "ERROR: ULP误差不满足标准\n" + return CompareConst.ERROR + else: + if self.ulp_err_proportion < 0.001: + return CompareConst.PASS + elif self.ulp_err_proportion_ratio < 1: + return CompareConst.PASS + else: + self.compare_message += "ERROR: ULP误差不满足标准\n" + return CompareConst.ERROR + + + +class ULPStandard(): + def __init__(self, input_data): + super().__init__(input_data) + + def __str__(self): + return f"{self.api_name}" + + def _compute_ulp_err_proportion_ratio(self): + column_name = ApiPrecisionCompareColumn.ULP_ERR_PROPORTION + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _calc_ratio(npu_value, gpu_value, 10000.0), True, "" + + def _compute_ratio(self): + ulp_err_proportion_ratio, ulp_inf_nan_consistency, compare_message = self._compute_ulp_err_proportion_ratio() + metrics = { + "ulp_err_proportion_ratio": ulp_err_proportion_ratio, + "compare_message": compare_message + } + return metrics, UlpInfNanConsistency(ulp_inf_nan_consistency) + + def check_mean_ulp_err(self): + column_name = ApiPrecisionCompareColumn.MEAN_ULP_ERR + npu_value, gpu_value = self._get_and_convert_values(column_name) + if is_inf_or_nan(npu_value) or is_inf_or_nan(gpu_value): + return check_inf_or_nan(npu_value, gpu_value, column_name) + else: + return _, True, "" + + def get_status(self, metrics, inf_nan_consistency): + ulp_inf_nan_consistency = inf_nan_consistency.ulp_inf_nan_consistency + _, mean_ulp_err_inf_nan_consistency, mean_ulp_err_message = self.check_mean_ulp_err() + if not mean_ulp_err_inf_nan_consistency: + metrics['compare_message'] += mean_ulp_err_message + if not ulp_inf_nan_consistency or not mean_ulp_err_inf_nan_consistency: + status_dict = { + "ulp_err_status": CompareConst.ERROR + } + return CompareConst.ERROR, status_dict + + dtype = self.npu_precision.get(ApiPrecisionCompareColumn.DEVICE_DTYPE) + if dtype == torch.float32: + status, final_message = self.get_fp32_ulp_err_status() + else: + status, final_message = self.get_fp16_ulp_err_status() + metrics['compare_message'] += final_message + + status_dict = { + "ulp_err_status": status + } + return status, status_dict + + def get_fp32_ulp_err_status(self): + compare_message = "" + mean_ulp_err_threshold, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold = BaseConfig.get_ulp_threshold(torch.float32) + if self.mean_ulp_err < mean_ulp_err_threshold: + return CompareConst.PASS + elif self.ulp_err_proportion < ulp_err_proportion_threshold: + return CompareConst.PASS + elif self.ulp_err_proportion_ratio < ulp_err_proportion_ratio_threshold: + return CompareConst.PASS + else: + compare_message += "ERROR: ULP误差不满足标准\n" + return CompareConst.ERROR, compare_message + + def get_fp16_ulp_err_status(self): + compare_message = "" + _, ulp_err_proportion_threshold, ulp_err_proportion_ratio_threshold = BaseConfig.get_ulp_threshold(torch.float16) + if self.ulp_err_proportion < ulp_err_proportion_threshold: + return CompareConst.PASS + elif self.ulp_err_proportion_ratio < ulp_err_proportion_ratio_threshold: + return CompareConst.PASS + else: + compare_message += "ERROR: ULP误差不满足标准\n" + return CompareConst.ERROR, compare_message