diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py index bd91f33a3ed70ee35cc54fc618d714fc562fd9dd..c23a5c094542c74bb56f80f6d0aaae7854f872c2 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py @@ -42,9 +42,14 @@ class Advisor: % (self.input_file, str(os_err))) raise CompareException(CompareException.PARSE_FILE_ERROR) from os_err data_columns = df.columns.values - if not {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - print_error_log('Compare result file does not contain %s, %s columns.' % (CompareConst.ACCURACY, - CompareConst.NPU_NAME)) + if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): + self.file_type = 'accuracy' + elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): + self.file_type = 'md5' + elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): + self.file_type = 'summary' + else: + print_error_log('Compare result file does not meet the required conditions.') raise CompareException(CompareException.INVALID_FILE_ERROR) df.reset_index(inplace=True) # The value of index is consistent with the line number of csv, csv file first line is 2 @@ -87,15 +92,19 @@ class Advisor: message = AdvisorConst.BATCH_NORM_SUGGEST return message - @staticmethod - def analyze_unmatched(analyze_data): - accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + def analyze_unmatched(self, analyze_data): + if self.file_type == 'accuracy': + accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + elif self.file_type == 'md5': + accuracy_unmatched = analyze_data[analyze_data[CompareConst.RESULT] == False] + elif self.file_type == 'summary': + accuracy_unmatched = analyze_data[analyze_data[CompareConst.RESULT] == "warning"] num_unmatch = len(accuracy_unmatched) if num_unmatch != 0: for i in range(len(accuracy_unmatched)): - item = analyze_data.iloc[i] + item = accuracy_unmatched.iloc[i] print_warn_log("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) + .format(item[CompareConst.NPU_NAME])) def gen_advisor_result(self, pd_data): first_failing_data = pd_data.iloc[0] @@ -111,7 +120,12 @@ class Advisor: analyze_data = self._parse_input_file() print_info_log("Start analyzing the comparison result: %s" % self.input_file) self.analyze_unmatched(analyze_data) - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + if self.file_type == 'accuracy': + failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + elif self.file_type == 'md5': + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == False] + elif self.file_type == 'summary': + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == "warning"] if failing_data.empty: print_info_log("All data from api input/output accuracy reached") result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 01759056764959d8ac9bbef3b2b19f41ddb961fa..10d8d4a563f6481dcb6dc53ceba61bd7af9cf8e0 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -162,7 +162,7 @@ class CompareConst: SUMMARY_COMPARE_RESULT_HEADER = [ NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF, - NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, ACCURACY, ERROR_MESSAGE + NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, RESULT, ERROR_MESSAGE ] MD5_COMPARE_RESULT_HEADER = [ diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index 68fe6c0d7ea6442bc01dfea19cbc97295e174016..db21db9ed8f2be2d5d1299450e522abbd43201ec 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -333,16 +333,26 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if summary_compare: start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) + warning_flag = False for i, val in enumerate(zip(npu_summery_data, bench_summery_data)): if all(isinstance(value, (float, int)) for value in val): - result_item[start_idx + i] = val[0] - val[1] + diff = val[0] - val[1] + result_item[start_idx + i] = diff + magnitude_diff = abs(diff) / max(abs(val[0]), abs(val[1])) + 1e-6 + if magnitude_diff > 0.1: + warning_flag = True else: result_item[start_idx + i] = CompareConst.NAN + if warning_flag: + accuracy_check = "warning" + err_msg += "Need double check api accuracy." + else: + accuracy_check = "" replace_res = map(lambda x: f'{str(x)}\t' if str(x) in ('inf', '-inf', 'nan') else x, result_item[start_idx:]) result_item[start_idx:] = list(replace_res) - result_item.append(CompareConst.ACCURACY_CHECK_YES) + result_item.append(accuracy_check if summary_compare else "") result_item.append(err_msg) if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) @@ -619,9 +629,8 @@ def compare_core(input_parma, output_path, stack_mode=False, auto_analyze=True, compare_process([npu_pkl, bench_pkl, fout], stack_mode, fuzzy_match, summary_compare, md5_compare) if summary_compare: print_info_log(f"Summary compare result is {file_path}") - return - if not md5_compare: + if not md5_compare and not summary_compare: _do_multi_process(input_parma, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) if auto_analyze: @@ -702,10 +711,6 @@ def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False if stack_mode: header.append(CompareConst.STACK) result_df = pd.DataFrame(result, columns=header) - if summary_compare and not md5_compare: - header.remove(CompareConst.ACCURACY) - header.remove(CompareConst.ERROR_MESSAGE) - result_df = result_df[header] result_df.to_csv(output_csv_handle, index=False)