From 550a589655a01aa0c64bca50d67d297b47931bb4 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Mon, 1 Apr 2024 16:10:04 +0800 Subject: [PATCH 1/2] summary compare support advisor --- .../python/ptdbg_ascend/advisor/advisor.py | 32 +++++++++++++------ .../ptdbg_ascend/compare/acc_compare.py | 3 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py index bd91f33a3e..2f3732d9fa 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py @@ -42,9 +42,14 @@ class Advisor: % (self.input_file, str(os_err))) raise CompareException(CompareException.PARSE_FILE_ERROR) from os_err data_columns = df.columns.values - if not {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - print_error_log('Compare result file does not contain %s, %s columns.' % (CompareConst.ACCURACY, - CompareConst.NPU_NAME)) + if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): + self.file_type = 'accuracy' + elif "Result" in data_columns and CompareConst.NPU_NAME in data_columns and CompareConst.ACCURACY not in data_columns: + self.file_type = 'md5' + elif {"Max diff", "Min diff", "Mean diff", CompareConst.NPU_NAME}.issubset(data_columns) and "Result" not in data_columns: + self.file_type = 'summary' + else: + print_error_log('Compare result file does not meet the required conditions.') raise CompareException(CompareException.INVALID_FILE_ERROR) df.reset_index(inplace=True) # The value of index is consistent with the line number of csv, csv file first line is 2 @@ -87,15 +92,19 @@ class Advisor: message = AdvisorConst.BATCH_NORM_SUGGEST return message - @staticmethod - def analyze_unmatched(analyze_data): - accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + def analyze_unmatched(self, analyze_data): + if self.file_type == 'accuracy': + accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + elif self.file_type == 'md5': + accuracy_unmatched = analyze_data[analyze_data["Result"] == False] + elif self.file_type == 'summary': + accuracy_unmatched = analyze_data[(analyze_data["Max diff"] != 0) | (analyze_data["Min diff"] != 0) | (analyze_data["Mean diff"] != 0)] num_unmatch = len(accuracy_unmatched) if num_unmatch != 0: for i in range(len(accuracy_unmatched)): - item = analyze_data.iloc[i] + item = accuracy_unmatched.iloc[i] print_warn_log("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) + .format(item[CompareConst.NPU_NAME])) def gen_advisor_result(self, pd_data): first_failing_data = pd_data.iloc[0] @@ -111,7 +120,12 @@ class Advisor: analyze_data = self._parse_input_file() print_info_log("Start analyzing the comparison result: %s" % self.input_file) self.analyze_unmatched(analyze_data) - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + if self.file_type == 'accuracy': + failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + elif self.file_type == 'md5': + failing_data = analyze_data[analyze_data["Result"] == False] + elif self.file_type == 'summary': + failing_data = analyze_data[(analyze_data["Max diff"] != 0) | (analyze_data["Min diff"] != 0) | (analyze_data["Mean diff"] != 0)] if failing_data.empty: print_info_log("All data from api input/output accuracy reached") result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index 68fe6c0d7e..b7e67fab18 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -619,9 +619,8 @@ def compare_core(input_parma, output_path, stack_mode=False, auto_analyze=True, compare_process([npu_pkl, bench_pkl, fout], stack_mode, fuzzy_match, summary_compare, md5_compare) if summary_compare: print_info_log(f"Summary compare result is {file_path}") - return - if not md5_compare: + if not md5_compare and not summary_compare: _do_multi_process(input_parma, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) if auto_analyze: -- Gitee From bcedf6a797bb76eff7aeb8d7311d5fcbe0680742 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Wed, 3 Apr 2024 15:24:39 +0800 Subject: [PATCH 2/2] add summary result --- .../src/python/ptdbg_ascend/advisor/advisor.py | 12 ++++++------ .../src/python/ptdbg_ascend/common/utils.py | 2 +- .../python/ptdbg_ascend/compare/acc_compare.py | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py index 2f3732d9fa..c23a5c0945 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/advisor/advisor.py @@ -44,9 +44,9 @@ class Advisor: data_columns = df.columns.values if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): self.file_type = 'accuracy' - elif "Result" in data_columns and CompareConst.NPU_NAME in data_columns and CompareConst.ACCURACY not in data_columns: + elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): self.file_type = 'md5' - elif {"Max diff", "Min diff", "Mean diff", CompareConst.NPU_NAME}.issubset(data_columns) and "Result" not in data_columns: + elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): self.file_type = 'summary' else: print_error_log('Compare result file does not meet the required conditions.') @@ -96,9 +96,9 @@ class Advisor: if self.file_type == 'accuracy': accuracy_unmatched = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] elif self.file_type == 'md5': - accuracy_unmatched = analyze_data[analyze_data["Result"] == False] + accuracy_unmatched = analyze_data[analyze_data[CompareConst.RESULT] == False] elif self.file_type == 'summary': - accuracy_unmatched = analyze_data[(analyze_data["Max diff"] != 0) | (analyze_data["Min diff"] != 0) | (analyze_data["Mean diff"] != 0)] + accuracy_unmatched = analyze_data[analyze_data[CompareConst.RESULT] == "warning"] num_unmatch = len(accuracy_unmatched) if num_unmatch != 0: for i in range(len(accuracy_unmatched)): @@ -123,9 +123,9 @@ class Advisor: if self.file_type == 'accuracy': failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] elif self.file_type == 'md5': - failing_data = analyze_data[analyze_data["Result"] == False] + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == False] elif self.file_type == 'summary': - failing_data = analyze_data[(analyze_data["Max diff"] != 0) | (analyze_data["Min diff"] != 0) | (analyze_data["Mean diff"] != 0)] + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == "warning"] if failing_data.empty: print_info_log("All data from api input/output accuracy reached") result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 0175905676..10d8d4a563 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -162,7 +162,7 @@ class CompareConst: SUMMARY_COMPARE_RESULT_HEADER = [ NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, MAX_DIFF, MIN_DIFF, MEAN_DIFF, NORM_DIFF, - NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, ACCURACY, ERROR_MESSAGE + NPU_MAX, NPU_MIN, NPU_MEAN, NPU_NORM, BENCH_MAX, BENCH_MIN, BENCH_MEAN, BENCH_NORM, RESULT, ERROR_MESSAGE ] MD5_COMPARE_RESULT_HEADER = [ diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py index b7e67fab18..db21db9ed8 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/compare/acc_compare.py @@ -333,16 +333,26 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals if summary_compare: start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) + warning_flag = False for i, val in enumerate(zip(npu_summery_data, bench_summery_data)): if all(isinstance(value, (float, int)) for value in val): - result_item[start_idx + i] = val[0] - val[1] + diff = val[0] - val[1] + result_item[start_idx + i] = diff + magnitude_diff = abs(diff) / max(abs(val[0]), abs(val[1])) + 1e-6 + if magnitude_diff > 0.1: + warning_flag = True else: result_item[start_idx + i] = CompareConst.NAN + if warning_flag: + accuracy_check = "warning" + err_msg += "Need double check api accuracy." + else: + accuracy_check = "" replace_res = map(lambda x: f'{str(x)}\t' if str(x) in ('inf', '-inf', 'nan') else x, result_item[start_idx:]) result_item[start_idx:] = list(replace_res) - result_item.append(CompareConst.ACCURACY_CHECK_YES) + result_item.append(accuracy_check if summary_compare else "") result_item.append(err_msg) if has_stack and index == 0 and key == "input_struct": result_item.extend(npu_stack_info) @@ -701,10 +711,6 @@ def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False if stack_mode: header.append(CompareConst.STACK) result_df = pd.DataFrame(result, columns=header) - if summary_compare and not md5_compare: - header.remove(CompareConst.ACCURACY) - header.remove(CompareConst.ERROR_MESSAGE) - result_df = result_df[header] result_df.to_csv(output_csv_handle, index=False) -- Gitee