diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py index bd91f33a3ed70ee35cc54fc618d714fc562fd9dd..6f34129aba0ab15e2817ba4edf8352e9b9b73c2f 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py @@ -20,7 +20,7 @@ import pandas as pd from .advisor_result import AdvisorResult from .advisor_const import AdvisorConst -from ..common.utils import CompareException, CompareConst +from ..common.utils import CompareException, CompareConst, Const from ..common.utils import print_info_log, print_warn_log, print_error_log from ..common.file_check_util import FileChecker, FileCheckConst @@ -42,9 +42,14 @@ class Advisor: % (self.input_file, str(os_err))) raise CompareException(CompareException.PARSE_FILE_ERROR) from os_err data_columns = df.columns.values - if not {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - print_error_log('Compare result file does not contain %s, %s columns.' % (CompareConst.ACCURACY, - CompareConst.NPU_NAME)) + if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): + self.file_type = Const.ALL + elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): + self.file_type = Const.MD5 + elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): + self.file_type = Const.SUMMARY + else: + print_error_log('Compare result file does not meet the required conditions.') raise CompareException(CompareException.INVALID_FILE_ERROR) df.reset_index(inplace=True) # The value of index is consistent with the line number of csv, csv file first line is 2 @@ -87,15 +92,18 @@ class Advisor: message = AdvisorConst.BATCH_NORM_SUGGEST return message - @staticmethod - def analyze_unmatched(analyze_data): - accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + def analyze_unmatched(self, analyze_data): + if self.file_type == Const.ALL: + accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + else: + accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | + (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] num_unmatch = len(accuracy_unmatched) if num_unmatch != 0: for i in range(len(accuracy_unmatched)): - item = analyze_data.iloc[i] + item = accuracy_unmatched.iloc[i] print_warn_log("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) + .format(item[CompareConst.NPU_NAME])) def gen_advisor_result(self, pd_data): first_failing_data = pd_data.iloc[0] @@ -111,7 +119,12 @@ class Advisor: analyze_data = self._parse_input_file() print_info_log("Start analyzing the comparison result: %s" % self.input_file) self.analyze_unmatched(analyze_data) - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + if self.file_type == Const.ALL: + failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + elif self.file_type == Const.MD5: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] + elif self.file_type == Const.SUMMARY: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] if failing_data.empty: print_info_log("All data from api input/output accuracy reached") result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 01759056764959d8ac9bbef3b2b19f41ddb961fa..d3762a1171202bca13937b12fa5bf5d225016ded 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -162,7 +162,7 @@ class CompareConst: SUMMARY_COMPARE_RESULT_HEADER = [ NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF, - NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, ACCURACY, ERROR_MESSAGE + NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, RESULT, ERROR_MESSAGE ] MD5_COMPARE_RESULT_HEADER = [ @@ -173,6 +173,9 @@ class CompareConst: NAN = 'Nan' SHAPE_UNMATCH = 'shape unmatched' DTYPE_UNMATCH = 'dtype unmatched' + PASS = 'Pass' + WARNING = 'Warning' + DIFF = 'Different' # accuracy standards COS_THRESHOLD = 0.99 diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index 68fe6c0d7ea6442bc01dfea19cbc97295e174016..087e062e206599d35cc45d453cadd17ed6619187 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -313,7 +313,7 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals err_msg = "" if md5_compare: result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - n_struct[2], b_struct[2], n_struct[2] == b_struct[2]] + n_struct[2], b_struct[2], CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) result.append(result_item) @@ -333,16 +333,21 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if summary_compare: start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) - for i, val in enumerate(zip(npu_summery_data, bench_summery_data)): - if all(isinstance(value, (float, int)) for value in val): - result_item[start_idx + i] = val[0] - val[1] + warning_flag = False + for i, (npu_val, bench_val) in enumerate(zip(npu_summery_data, bench_summery_data)): + if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): + diff = npu_val - bench_val + result_item[start_idx + i] = diff + magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) + if magnitude_diff > 0.5: + warning_flag = True else: result_item[start_idx + i] = CompareConst.NAN - replace_res = map(lambda x: f'{str(x)}\t' if str(x) in ('inf', '-inf', 'nan') else x, - result_item[start_idx:]) - result_item[start_idx:] = list(replace_res) + accuracy_check = CompareConst.WARNING if warning_flag else "" + err_msg += "Need double check api accuracy." if warning_flag else "" + result_item[start_idx:] = [f'{str(x)}\t' if str(x) in ('inf', '-inf', 'nan') else x for x in result_item[start_idx:]] - result_item.append(CompareConst.ACCURACY_CHECK_YES) + result_item.append(accuracy_check if summary_compare else CompareConst.ACCURACY_CHECK_YES) result_item.append(err_msg) if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) @@ -619,9 +624,8 @@ def compare_core(input_parma, output_path, stack_mode=False, auto_analyze=True, compare_process([npu_pkl, bench_pkl, fout], stack_mode, fuzzy_match, summary_compare, md5_compare) if summary_compare: print_info_log(f"Summary compare result is {file_path}") - return - if not md5_compare: + if not md5_compare and not summary_compare: _do_multi_process(input_parma, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) if auto_analyze: @@ -702,10 +706,6 @@ def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False if stack_mode: header.append(CompareConst.STACK) result_df = pd.DataFrame(result, columns=header) - if summary_compare and not md5_compare: - header.remove(CompareConst.ACCURACY) - header.remove(CompareConst.ERROR_MESSAGE) - result_df = result_df[header] result_df.to_csv(output_csv_handle, index=False)