diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index d5b5381ef3fb9b22186fe6e704313af36b51efd5..75d1dadf0e7c9fd7f47ad94411e18e81acc74bb4 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -15,6 +15,7 @@ import multiprocessing import os +import math import pandas as pd from tqdm import tqdm from msprobe.core.common.file_utils import load_json @@ -61,19 +62,26 @@ class Comparator: start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) warning_flag = False for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): - if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): + if (isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)) and + not type(npu_val) is bool and not type(bench_val) is bool): diff = npu_val - bench_val - if bench_val != 0: - relative = str(abs((diff / bench_val) * 100)) + '%' + if math.isnan(diff): + diff = CompareConst.NAN + relative = CompareConst.NAN else: - relative = "N/A" + if bench_val != 0: + relative = str(abs((diff / bench_val) * 100)) + '%' + else: + relative = CompareConst.N_A + magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) + if magnitude_diff > 0.5: + warning_flag = True + result_item[start_idx + i] = diff result_item[start_idx + i + 4] = relative - magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) - if magnitude_diff > 0.5: - warning_flag = True else: - result_item[start_idx + i] = CompareConst.NONE + result_item[start_idx + i] = CompareConst.N_A + result_item[start_idx + i + 4] = CompareConst.N_A accuracy_check = CompareConst.WARNING if warning_flag else "" err_msg += "Need double check api accuracy." if warning_flag else "" for i in range(start_idx, len(result_item)): diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index c4b84163bfae8a2ab7cb01398a6ef0e9bf3aa7e6..0646d24c01c9be9fdf8180ee040ff69bbfe70630 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -167,7 +167,7 @@ class GetCosineSimilarity(TensorComparisonBasic): if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, '' if n_value == CompareConst.NAN: - return "N/A", '' + return CompareConst.N_A, '' if not n_value.shape: return CompareConst.UNSUPPORTED, '' @@ -205,10 +205,13 @@ class GetMaxAbsErr(TensorComparisonBasic): if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, "" if n_value == CompareConst.NAN: - return "N/A", "" + return CompareConst.N_A, "" temp_res = n_value - b_value max_value = np.max(np.abs(temp_res)) + if np.isnan(max_value): + message = 'Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data.' + return CompareConst.NAN, message return format_value(max_value), "" @@ -235,13 +238,13 @@ class GetMaxRelativeErr(TensorComparisonBasic): if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, '' if n_value == CompareConst.NAN: - return "N/A", '' + return CompareConst.N_A, '' if relative_err is None: relative_err = get_relative_err(n_value, b_value) max_relative_err = np.max(np.abs(relative_err)) if np.isnan(max_relative_err): - message = 'Cannot compare by MaxRelativeError, the data contains nan in dump data.' + message = 'Cannot compare by MaxRelativeError, the data contains nan/inf/-inf in dump data.' return CompareConst.NAN, message return format_value(max_relative_err), '' @@ -257,7 +260,7 @@ class GetThousandErrRatio(TensorComparisonBasic): if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, "" if n_value == CompareConst.NAN: - return "N/A", "" + return CompareConst.N_A, "" if not n_value.shape: return CompareConst.NAN, "" @@ -279,7 +282,7 @@ class GetFiveThousandErrRatio(TensorComparisonBasic): if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, "" if n_value == CompareConst.NAN: - return "N/A", "" + return CompareConst.N_A, "" if not n_value.shape: return CompareConst.NAN, "" diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 4963d40978f3bddd7f6a17d3a268467ef20fbc9e..b59cf300482d9f01ac710558f374954e8591f08d 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -15,6 +15,7 @@ import os import re +import math import numpy as np from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger @@ -274,27 +275,36 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals ] npu_summary_data = n_dict.get(CompareConst.SUMMARY)[n_start + index] - result_item.extend(npu_summary_data) + result_item.extend([CompareConst.NAN if isinstance(x, float) and math.isnan(x) + else x for x in npu_summary_data]) bench_summary_data = b_dict.get(CompareConst.SUMMARY)[b_start + index] - result_item.extend(bench_summary_data) + result_item.extend([CompareConst.NAN if isinstance(x, float) and math.isnan(x) + else x for x in bench_summary_data]) if summary_compare: start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) warning_flag = False for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): - if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): + if (isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)) and + not type(npu_val) is bool and not type(bench_val) is bool): diff = npu_val - bench_val - if bench_val != 0: - relative = str(abs((diff / bench_val) * 100)) + '%' + if math.isnan(diff): + diff = CompareConst.NAN + relative = CompareConst.NAN else: - relative = CompareConst.N_A + if bench_val != 0: + relative = str(abs((diff / bench_val) * 100)) + '%' + else: + relative = CompareConst.N_A + magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) + if magnitude_diff > 0.5: + warning_flag = True + result_item[start_idx + i] = diff result_item[start_idx + i + 4] = relative - magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) - if magnitude_diff > 0.5: - warning_flag = True else: - result_item[start_idx + i] = CompareConst.NONE + result_item[start_idx + i] = CompareConst.N_A + result_item[start_idx + i + 4] = CompareConst.N_A accuracy_check = CompareConst.WARNING if warning_flag else "" err_msg += "Need double check api accuracy." if warning_flag else "" for i in range(start_idx, len(result_item)):