From 9b42c33358ad7afaa8df37a03b84642f19ab6856 Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Thu, 7 Nov 2024 20:47:28 +0800 Subject: [PATCH 1/9] temporary modify --- .../msprobe/core/compare/utils.py | 3 +- .../msprobe/mindspore/compare/ms_compare.py | 101 +++++++++++++++--- 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index b23840c3c..e43956d5e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -151,7 +151,7 @@ def op_item_parse(item, op_name, index, item_list=None, top_bool=True, depth=0): if isinstance(item, dict): if 'type' not in item: for kwarg in item: - kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None, depth=depth + 1) + kwarg_parsed_list = op_item_parse(item[kwarg], full_op_name + Const.SEP + kwarg, None, depth=depth + 1) item_list += kwarg_parsed_list kwarg_parsed_list.clear() elif 'dtype' in item: @@ -459,6 +459,7 @@ def merge_tensor(tensor_list, dump_mode): def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) parser.add_argument("-o", "--output_path", dest="output_path", type=str, diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 77d30c539..9cab35541 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -16,6 +16,7 @@ import copy import os import re +import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException @@ -24,10 +25,11 @@ from msprobe.core.common.file_utils import (FileOpen, create_directory, from msprobe.core.common.log import logger from msprobe.core.common.utils import (CompareException, check_compare_param, check_configuration_param, - get_dump_mode, set_dump_path) + get_dump_mode, set_dump_path, check_op_str_pattern_valid) from msprobe.core.compare.acc_compare import Comparator from msprobe.core.compare.check import check_struct_match, fuzzy_check_op from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping +from msprobe.core.common.file_utils import load_json class MSComparator(Comparator): @@ -66,14 +68,13 @@ class MSComparator(Comparator): return mapping_dict def process_cell_mapping(self, npu_op_name): - npu_op_name = [op_name.replace("Cell", "Module", 1) for op_name in npu_op_name] + npu_op_name = npu_op_name.replace("Cell", "Module", 1) if self.cell_mapping_dict: - for index, op_name in enumerate(npu_op_name): - # get cell name & class name from op_name - # Cell.fc1.Dense.forward.0.input.0 - cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] - if cell_name in self.cell_mapping_dict: - npu_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) + # get cell name & class name from op_name + # Cell.fc1.Dense.forward.0.input.0 + cell_name = npu_op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] + if cell_name in self.cell_mapping_dict: + npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) return npu_op_name def check_op(self, npu_dict, bench_dict, fuzzy_match): @@ -82,7 +83,7 @@ class MSComparator(Comparator): if self.cell_mapping is not None: npu_op_name = self.process_cell_mapping(npu_op_name) if self.api_mapping is not None: - npu_op_name = self.process_internal_api_mapping(npu_op_name, bench_op_name) + npu_op_name = self.process_internal_api_mapping(npu_op_name) if isinstance(self.api_mapping, str): npu_dict_new, bench_dict_new, target_dict = self.transform_user_mapping_api(npu_dict_new, bench_dict_new) @@ -119,19 +120,17 @@ class MSComparator(Comparator): npu_op_name[idx] = npu_op_name[idx].replace(target, para) return npu_op_name - def process_internal_api_mapping(self, npu_op_name, bench_op_name): + def process_internal_api_mapping(self, npu_op_name): # get api name & class name from op_name # Functional.addcmul.0.forward.input.0 - npu_op_name, bench_op_name = npu_op_name.copy(), bench_op_name.copy() - ms_api_name = self.get_api_name(npu_op_name[0].split(Const.SEP)) - pt_api_name = self.get_api_name(bench_op_name[0].split(Const.SEP)) + ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) class_name = ms_api_name.split(Const.SEP)[0] if class_name == "Mint": - return self.api_replace(npu_op_name, "Mint", "Torch") + return npu_op_name.replace("Mint", "Torch") elif class_name == "MintFunctional": - return self.api_replace(npu_op_name, "MintFunctional", "Functional") - elif self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: - return self.api_replace(npu_op_name, ms_api_name, pt_api_name) + return npu_op_name.replace("MintFunctional", "Functional") + elif self.ms_to_pt_mapping.get(ms_api_name): + return npu_op_name.replace(ms_api_name, self.ms_to_pt_mapping.get(ms_api_name)) else: return npu_op_name @@ -245,6 +244,74 @@ class MSComparator(Comparator): del_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) return del_bench_dict + + def compare_process(self, file_lists, stack_mode, fuzzy_match, dump_mode): + npu_json_path, bench_json_path, stack_json_path = file_lists + npu_json_data = load_json(npu_json_path) + bench_json_data = load_json(bench_json_path) + stack_json_data = load_json(stack_json_path) + + if fuzzy_match: + logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + npu_df = self.gen_data_df(npu_json_data, stack_json_data, dump_mode) + bench_df = self.gen_data_df(bench_json_data, stack_json_data, dump_mode) + if self.cell_mapping: + npu_df['compare_key'] = npu_df.apply(lambda row: self.process_cell_mapping(row['op_name']), axis=1) + elif self.api_mapping: + npu_df['compare_key'] = npu_df.apply(lambda row: self.process_internal_api_mapping(row['op_name']), axis=1) + else: + npu_df['compare_key'] = npu_df['op_name'] + match_result = pd.merge(npu_df, bench_df, on=['compare_key', Const.SHAPE], how='outer') + condition = (match_result['dtype_x'] == match_result['dtype_y']) | \ + ((match_result['dtype_x'] == Const.FLOAT16) & (match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.FLOAT16) & (match_result['dtype_x'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['dtype_x'] == Const.TORCH_FLOAT16) & (match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & (match_result['dtype_x'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + (match_result['op_name_x'].notna() & match_result['op_name_y'].isna()) + match_result = match_result[((condition))].fillna(CompareConst.N_A) + return MSComparator.make_result_table(match_result, stack_mode, dump_mode) + + + def gen_data_df(self, data_json, stack_json, dump_mode): + result = { + 'op_name': [], + Const.DTYPE: [], + Const.SHAPE: [], + Const.SUMMARY: [], + 'stack_info': [] + } + if dump_mode == Const.ALL: + result['data_name'] = [] + for data in data_json['data']: + check_op_str_pattern_valid(data) + merge_list = self.gen_merge_list(data_json, data, stack_json, dump_mode) + for op_name in merge_list['op_name']: + result['op_name'].append(op_name) + if Const.SEP + Const.INPUT + Const.SEP in op_name: + struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) + else: + struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) + result[Const.DTYPE].append(struct[0]) + result[Const.SHAPE].append(struct[1]) + result[Const.SUMMARY].append(merge_list[Const.SUMMARY].pop(0)) + result['stack_info'].append(merge_list['stack_info'][0]) + if dump_mode == Const.ALL: + result['data_name'].append(merge_list['data_name'].pop(0)) + return pd.DataFrame(result) + + @classmethod + def make_result_table(cls, result, stack_mode, dump_mode): + header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] + + if dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) + if stack_mode: + header.append(CompareConst.STACK) + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, + 'op_name_y': CompareConst.BENCH_NAME, + 'dtype_x': CompareConst.NPU_DTYPE, + 'dtype_y': CompareConst.BENCH_DTYPE, + Const.SHAPE: CompareConst.NPU_SHAPE,}) def check_cross_framework(bench_json_path): -- Gitee From f446c71358dc0788b252e5c7083a8ec4ec1cdc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Sun, 10 Nov 2024 20:52:24 +0800 Subject: [PATCH 2/9] temporary commit --- .../msprobe/mindspore/compare/ms_compare.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 5e6b0a714..322a309f9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -27,7 +27,7 @@ from msprobe.core.common.utils import (CompareException, check_compare_param, check_configuration_param, get_dump_mode, set_dump_path, check_op_str_pattern_valid) from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.compare.check import check_struct_match, fuzzy_check_op +from msprobe.core.compare.check import check_struct_match, fuzzy_check_op, dtype_mapping from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping from msprobe.core.common.file_utils import load_json @@ -266,16 +266,25 @@ class MSComparator(Comparator): npu_df['compare_key'] = npu_df.apply(lambda row: self.process_internal_api_mapping(row['op_name']), axis=1) else: npu_df['compare_key'] = npu_df['op_name'] - match_result = pd.merge(npu_df, bench_df, on=['compare_key', Const.SHAPE], how='outer') - condition = (match_result['dtype_x'] == match_result['dtype_y']) | \ - ((match_result['dtype_x'] == Const.FLOAT16) & (match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ - ((match_result['dtype_y'] == Const.FLOAT16) & (match_result['dtype_x'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ - ((match_result['dtype_x'] == Const.TORCH_FLOAT16) & (match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ - ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & (match_result['dtype_x'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ - (match_result['op_name_x'].notna() & match_result['op_name_y'].isna()) - match_result = match_result[((condition))].fillna(CompareConst.N_A) + npu_df['compare_shape'] = npu_df[Const.SHAPE].apply(str) + bench_df['compare_shape'] = bench_df[Const.SHAPE].apply(str) + bench_df['compare_key'] = bench_df['op_name'] + match_result = pd.merge(npu_df, bench_df, on=['compare_key', 'compare_shape'], how='outer') + match_result = match_result[self.gen_compare_condition(match_result)].fillna(CompareConst.N_A) return MSComparator.make_result_table(match_result, stack_mode, dump_mode) + def gen_compare_condition(self, match_result): + if self.cross_frame: + match_result['compare_dtype'] = match_result['dtype_x'].apply(lambda row: dtype_mapping.get(row, row)) + else: + match_result['compare_dtype'] = match_result['dtype_x'] + return (match_result['compare_dtype'] == match_result['dtype_y']) | \ + ((match_result['compare_dtype'] == Const.FLOAT16) & (match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.FLOAT16) & (match_result['compare_dtype'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['compare_dtype'] == Const.TORCH_FLOAT16) & (match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & (match_result['compare_dtype'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + (match_result['op_name_x'].notna() & match_result['op_name_y'].isna()) + def gen_data_df(self, data_json, stack_json, dump_mode): result = { -- Gitee From 63c069cff2ef9920056d835a9a95a4168a9b36e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Sun, 10 Nov 2024 21:22:49 +0800 Subject: [PATCH 3/9] temporary commit --- .../msprobe/mindspore/compare/ms_compare.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 322a309f9..60430c11b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -325,7 +325,17 @@ class MSComparator(Comparator): 'op_name_y': CompareConst.BENCH_NAME, 'dtype_x': CompareConst.NPU_DTYPE, 'dtype_y': CompareConst.BENCH_DTYPE, - Const.SHAPE: CompareConst.NPU_SHAPE,}) + 'shape_x': CompareConst.NPU_SHAPE, + 'shape_y': CompareConst.BENCH_DTYPE, + 'Max_x': CompareConst.NPU_MAX, + 'Min_x': CompareConst.NPU_MIN, + 'Mean_x': CompareConst.NPU_MEAN, + 'Norm_x': CompareConst.NPU_NORM, + 'Max_y': CompareConst.BENCH_MAX, + 'Min_y': CompareConst.BENCH_MIN, + 'Mean_y': CompareConst.BENCH_MEAN, + 'Norm_y': CompareConst.BENCH_NORM}, inplace=True) + def check_cross_framework(bench_json_path): -- Gitee From 9cf22eb85898a27e43d70dc104c0238d9d3e009c Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 20:45:22 +0800 Subject: [PATCH 4/9] temporary modify --- .../msprobe/mindspore/compare/ms_compare.py | 264 ++++++++---------- 1 file changed, 114 insertions(+), 150 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 60430c11b..81c8361e2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -13,23 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy +import math import os import re import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import (FileOpen, create_directory, +from msprobe.core.common.file_utils import (FileOpen, create_directory, load_json, load_npy, load_yaml) from msprobe.core.common.log import logger from msprobe.core.common.utils import (CompareException, check_compare_param, check_configuration_param, get_dump_mode, set_dump_path, check_op_str_pattern_valid) from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.compare.check import check_struct_match, fuzzy_check_op, dtype_mapping +from msprobe.core.compare.check import dtype_mapping from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping -from msprobe.core.common.file_utils import load_json class MSComparator(Comparator): @@ -76,31 +75,6 @@ class MSComparator(Comparator): if cell_name in self.cell_mapping_dict: npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) return npu_op_name - - def check_op(self, npu_dict, bench_dict, fuzzy_match): - npu_dict_new, bench_dict_new = copy.deepcopy(npu_dict), copy.deepcopy(bench_dict) - npu_op_name, bench_op_name = npu_dict_new.get(CompareConst.OP_NAME), bench_dict_new.get(CompareConst.OP_NAME) - if self.cell_mapping is not None: - npu_op_name = self.process_cell_mapping(npu_op_name) - if self.api_mapping is not None: - npu_op_name = self.process_internal_api_mapping(npu_op_name) - if isinstance(self.api_mapping, str): - npu_dict_new, bench_dict_new, target_dict = self.transform_user_mapping_api(npu_dict_new, - bench_dict_new) - if target_dict: - bench_dict = self.reconstitution_bench_dict(npu_dict, copy.deepcopy(bench_dict_new), target_dict) - npu_op_name = npu_dict_new.get(CompareConst.OP_NAME) - bench_op_name = bench_dict_new.get(CompareConst.OP_NAME) - struct_match = check_struct_match(npu_dict_new, bench_dict_new, cross_frame=self.cross_frame) - if not fuzzy_match: - return npu_op_name == bench_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(npu_op_name, bench_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name)) - is_match = False - return is_match and struct_match def read_npy_data(self, dir_path, file_name, load_pt_file=False): data_path = os.path.join(dir_path, file_name) @@ -147,108 +121,40 @@ class MSComparator(Comparator): raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error return api_name - def transform_user_mapping_api(self, new_npu_dict, new_bench_dict): - """ - Transform user mapping API based on new NPU and benchmark dictionaries. - Parameters: - new_npu_dict (dict): New NPU operation dictionary. - new_bench_dict (dict): New benchmark operation dictionary. - Returns: - tuple: Updated NPU and benchmark dictionaries, along with the target dictionary. - """ - npu_op_name, bench_op_name = new_npu_dict.get(CompareConst.OP_NAME), new_bench_dict.get(CompareConst.OP_NAME) - npu_struct_in = new_npu_dict.get(CompareConst.INPUT_STRUCT) - bench_struct_in = new_bench_dict.get(CompareConst.INPUT_STRUCT) - npu_struct_out = new_npu_dict.get(CompareConst.OUTPUT_STRUCT) - bench_struct_out = new_bench_dict.get(CompareConst.OUTPUT_STRUCT) - npu_summary, bench_summary = new_npu_dict.get(CompareConst.SUMMARY), new_bench_dict.get(CompareConst.SUMMARY) - npu_in_len, bench_in_len = len(npu_struct_in), len(bench_struct_in) - npu_out_len, bench_out_len = len(npu_struct_out), len(bench_struct_out) - ms_api_list, pt_api_list = npu_op_name[0].split(Const.SEP), bench_op_name[0].split(Const.SEP) - ms_api_name = self.get_api_name(ms_api_list) - pt_api_name = self.get_api_name(pt_api_list) - target_dict = {} - for api_dict in self.api_mapping_dict: - if api_dict.get("pt_api") == pt_api_name and api_dict.get("ms_api") == ms_api_name: - ms_user_args_len, pt_user_args_len = len(api_dict.get("ms_args")), len(api_dict.get("pt_args")) - ms_user_output_len, pt_user_output_len = len(api_dict.get("ms_output")), len(api_dict.get("pt_output")) - if ms_user_args_len != pt_user_args_len or ms_user_output_len != pt_user_output_len: - logger.warning("The user-defined mapping table is incorrect,\ - make sure that the number of parameters is equal") - break - ms_out_list = api_dict.get("ms_output", []) - for idx in reversed(range(npu_out_len)): - if idx not in ms_out_list: - del npu_struct_out[idx] - if idx + npu_in_len < len(npu_summary) and idx + npu_in_len < len(npu_op_name): - del npu_summary[idx + npu_in_len] - del npu_op_name[idx + npu_in_len] - pt_out_list = api_dict.get("pt_output", []) - for idx in reversed(range(bench_out_len)): - if idx not in pt_out_list: - del bench_struct_out[idx] - if idx + bench_in_len < len(bench_summary) and idx + bench_in_len < len(bench_op_name): - del bench_summary[idx + bench_in_len] - del bench_op_name[idx + bench_in_len] - ms_para_list = api_dict.get("ms_args", []) - for idx in reversed(range(npu_in_len)): - if idx not in ms_para_list: - self.remove_element(npu_op_name, npu_struct_in, npu_summary, idx) - pt_para_list = api_dict.get("pt_args", []) - for idx in reversed(range(bench_in_len)): - if idx not in pt_para_list: - self.remove_element(bench_op_name, bench_struct_in, bench_summary, idx) - npu_op_name = self.api_replace(npu_op_name, ms_api_name, pt_api_name) - if len(npu_op_name) != len(bench_op_name): - logger.warning( - "The total number of input and output parameters of \ - npu_op_name and bench_op_name are not equal.") - break - npu_op_name = self.para_sequence_update(npu_op_name, bench_op_name) - target_dict = api_dict - break - if target_dict: - new_npu_dict.update({CompareConst.OP_NAME: npu_op_name, CompareConst.INPUT_STRUCT: npu_struct_in, - CompareConst.OUTPUT_STRUCT: npu_struct_out, CompareConst.SUMMARY: npu_summary}) - new_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, - CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) - return new_npu_dict, new_bench_dict, target_dict - - def para_sequence_update(self, npu_op_name, bench_op_name): - for idx, _ in enumerate(npu_op_name): - bench_op_name_list = bench_op_name[idx].rsplit(Const.SEP, 1) - if len(bench_op_name_list) != 0: - npu_op_name[idx] = npu_op_name[idx].rsplit(Const.SEP, 1)[0] + Const.SEP + bench_op_name_list[-1] - return npu_op_name + @staticmethod + def calc_accuracy(row, dump_mode, header): + def calc_summary_diff(data_type: str): + need_warning = False + ms_val, pt_val = row['NPU ' + data_type], row['Bench ' + data_type] + if all(isinstance(val, (float, int)) and not isinstance(val, bool) for val in [ms_val, pt_val]): + diff = ms_val - pt_val + if math.isnan(diff): + row[data_type.capitalize() + ' diff'] = CompareConst.NAN + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.NAN + else: + if pt_val != 0: + row[data_type.capitalize() + 'RelativeErr'] = str(abs((diff / pt_val) * 100)) + '%' + else: + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.N_A + magnitude_diff = abs(diff) / (max(abs(ms_val), abs(pt_val)) + CompareConst.EPSILON) + need_warning = magnitude_diff > CompareConst.MAGNITUDE + return need_warning - def reconstitution_bench_dict(self, npu_dict, del_bench_dict, api_dict): - ms_user_args_list = api_dict.get("ms_args", []) - ms_user_output_list = api_dict.get("ms_output", []) - npu_struct_in = npu_dict.get(CompareConst.INPUT_STRUCT) - npu_struct_out = npu_dict.get(CompareConst.OUTPUT_STRUCT) - npu_in_len = len(npu_struct_in) - npu_out_len = len(npu_struct_out) - if npu_in_len == len(ms_user_args_list) and npu_out_len == len(ms_user_output_list): - return del_bench_dict - ms_input_args_list = [i for i in range(npu_in_len)] - input_sub_list = list(set(ms_input_args_list) - set(ms_user_args_list)) - ms_output_args_list = [i for i in range(npu_out_len)] - output_sub_list = list(set(ms_output_args_list) - set(ms_user_output_list)) - bench_op_name = del_bench_dict.get(CompareConst.OP_NAME, []) - bench_struct_in = del_bench_dict.get(CompareConst.INPUT_STRUCT, []) - bench_struct_out = del_bench_dict.get(CompareConst.OUTPUT_STRUCT, []) - bench_summary = del_bench_dict.get(CompareConst.SUMMARY, []) - for idx in input_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(idx, CompareConst.N_A) - bench_struct_in.insert(idx, CompareConst.N_A) - bench_summary.insert(idx, CompareConst.N_A) - for idx in output_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(npu_in_len + idx, CompareConst.N_A) - bench_struct_out.insert(idx, CompareConst.N_A) - bench_summary.insert(npu_in_len + idx, CompareConst.N_A) - del_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, - CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) - return del_bench_dict + if dump_mode == Const.MD5: + row[CompareConst.RESULT] = CompareConst.PASS if row[CompareConst.NPU_MD5] == row[CompareConst.BENCH_MD5] else CompareConst.DIFF + elif dump_mode == Const.SUMMARY: + warning_flag = any([calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']]) + row[CompareConst.RESULT] = CompareConst.WARNING if warning_flag else "" + row[CompareConst.ERROR_MESSAGE] = "Need double check api accuracy." if warning_flag else "" + else: + row[CompareConst.COSINE] = '' + row[CompareConst.MAX_ABS_ERR] = '' + row[CompareConst.MAX_RELATIVE_ERR] = '' + row[CompareConst.ONE_THOUSANDTH_ERR_RATIO] = '' + row[CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = '' + row[CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES + row[CompareConst.ERROR_MESSAGE] = '' + return row[header] def compare_process(self, file_lists, stack_mode, fuzzy_match, dump_mode): npu_json_path, bench_json_path, stack_json_path = file_lists @@ -256,14 +162,14 @@ class MSComparator(Comparator): bench_json_data = load_json(bench_json_path) stack_json_data = load_json(stack_json_path) - if fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") npu_df = self.gen_data_df(npu_json_data, stack_json_data, dump_mode) bench_df = self.gen_data_df(bench_json_data, stack_json_data, dump_mode) if self.cell_mapping: npu_df['compare_key'] = npu_df.apply(lambda row: self.process_cell_mapping(row['op_name']), axis=1) elif self.api_mapping: npu_df['compare_key'] = npu_df.apply(lambda row: self.process_internal_api_mapping(row['op_name']), axis=1) + if isinstance(self.api_mapping, str): + self.modify_compare_data_with_user_mapping(npu_df, bench_df) else: npu_df['compare_key'] = npu_df['op_name'] npu_df['compare_shape'] = npu_df[Const.SHAPE].apply(str) @@ -272,20 +178,65 @@ class MSComparator(Comparator): match_result = pd.merge(npu_df, bench_df, on=['compare_key', 'compare_shape'], how='outer') match_result = match_result[self.gen_compare_condition(match_result)].fillna(CompareConst.N_A) return MSComparator.make_result_table(match_result, stack_mode, dump_mode) + + def modify_compare_data_with_user_mapping(self, npu_df, bench_df): + def get_api_indices_dict(op_name_df): + api_indices_dict = {} + for op_index, name in enumerate(op_name_df['op_name']): + api = self.get_api_name(name.split(Const.SEP)) + if api in api_indices_dict: + api_indices_dict[api].append(op_index) + else: + api_indices_dict[api] = [op_index] + return api_indices_dict + + ms_api_indices_dict = get_api_indices_dict(npu_df) + pt_api_indices_dict = get_api_indices_dict(bench_df) + + for mapping_dict in self.api_mapping_dict: + if (len(mapping_dict.get('ms_args')) != len(mapping_dict.get('pt_args')) or + len(mapping_dict.get('ms_output')) != len(mapping_dict.get('pt_output'))): + logger.warning('The user-defined mapping table is incorrect,\ + make sure that the number of parameters is equal') + continue + ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') + if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: + continue + for index in ms_api_indices_dict.get(ms_api): + op_name = npu_df.loc[index, 'op_name'].replace(ms_api, pt_api, 1) + is_abandoned = True + if '.input.' in op_name: + for i, prefix in enumerate(mapping_dict.get('ms_args')): + if op_name.split('.input.')[1].startswith(str(prefix)): + npu_df.loc[index, 'compare_key'] = ( + op_name.replace('.input.' + str(prefix), + '.input.' + str(mapping_dict.get('pt_args')[i]))) + is_abandoned = False + else: + for i, prefix in enumerate(mapping_dict.get('ms_output')): + if op_name.split('.output.')[1].startswith(str(prefix)): + npu_df.loc[index, 'compare_key'] = ( + op_name.replace('.output.' + str(prefix), + '.output.' + str(mapping_dict.get('pt_output')[i]))) + is_abandoned = False + if is_abandoned: + npu_df.loc[index, 'compare_key'] = op_name + 'abandoned' def gen_compare_condition(self, match_result): + match_result['compare_dtype'] = match_result['dtype_x'] if self.cross_frame: - match_result['compare_dtype'] = match_result['dtype_x'].apply(lambda row: dtype_mapping.get(row, row)) - else: - match_result['compare_dtype'] = match_result['dtype_x'] + match_result['compare_dtype'] = match_result['dtype_x'].apply(lambda row: dtype_mapping.get(row, row)) return (match_result['compare_dtype'] == match_result['dtype_y']) | \ - ((match_result['compare_dtype'] == Const.FLOAT16) & (match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ - ((match_result['dtype_y'] == Const.FLOAT16) & (match_result['compare_dtype'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ - ((match_result['compare_dtype'] == Const.TORCH_FLOAT16) & (match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ - ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & (match_result['compare_dtype'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + ((match_result['compare_dtype'] == Const.FLOAT16) & ( + match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.FLOAT16) & ( + match_result['compare_dtype'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['compare_dtype'] == Const.TORCH_FLOAT16) & ( + match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & ( + match_result['compare_dtype'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ (match_result['op_name_x'].notna() & match_result['op_name_y'].isna()) - def gen_data_df(self, data_json, stack_json, dump_mode): result = { 'op_name': [], @@ -296,6 +247,8 @@ class MSComparator(Comparator): } if dump_mode == Const.ALL: result['data_name'] = [] + elif dump_mode == Const.MD5: + result[Const.MD5] = [] for data in data_json['data']: check_op_str_pattern_valid(data) merge_list = self.gen_merge_list(data_json, data, stack_json, dump_mode) @@ -307,6 +260,8 @@ class MSComparator(Comparator): struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) result[Const.DTYPE].append(struct[0]) result[Const.SHAPE].append(struct[1]) + if dump_mode == Const.MD5: + result[Const.MD5].append(struct[2]) result[Const.SUMMARY].append(merge_list[Const.SUMMARY].pop(0)) result['stack_info'].append(merge_list['stack_info'][0]) if dump_mode == Const.ALL: @@ -326,16 +281,25 @@ class MSComparator(Comparator): 'dtype_x': CompareConst.NPU_DTYPE, 'dtype_y': CompareConst.BENCH_DTYPE, 'shape_x': CompareConst.NPU_SHAPE, - 'shape_y': CompareConst.BENCH_DTYPE, - 'Max_x': CompareConst.NPU_MAX, - 'Min_x': CompareConst.NPU_MIN, - 'Mean_x': CompareConst.NPU_MEAN, - 'Norm_x': CompareConst.NPU_NORM, - 'Max_y': CompareConst.BENCH_MAX, - 'Min_y': CompareConst.BENCH_MIN, - 'Mean_y': CompareConst.BENCH_MEAN, - 'Norm_y': CompareConst.BENCH_NORM}, inplace=True) - + 'shape_y': CompareConst.BENCH_SHAPE, + 'md5_x': CompareConst.NPU_MD5, + 'md5_y': CompareConst.BENCH_MD5, + 'data_name_x': CompareConst.DATA_NAME, + 'stack_info_x': CompareConst.STACK}, inplace=True) + result[CompareConst.NPU_MAX] = result['summary_x'].apply(lambda row: row[0]) + result[CompareConst.NPU_MIN] = result['summary_x'].apply(lambda row: row[1]) + result[CompareConst.NPU_MEAN] = result['summary_x'].apply(lambda row: row[2]) + result[CompareConst.NPU_NORM] = result['summary_x'].apply(lambda row: row[3]) + result[CompareConst.BENCH_MAX] = result['summary_y'].apply(lambda row: row[0] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MIN] = result['summary_y'].apply(lambda row: row[1] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MEAN] = result['summary_y'].apply(lambda row: row[2] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_NORM] = result['summary_y'].apply(lambda row: row[3] if row != CompareConst.N_A else row) + result_df = pd.DataFrame(columns=header) + for h in header: + if h in result.columns: + result_df[h] = result[h] + result_df = result_df.apply(lambda row: cls.calc_accuracy(row, dump_mode, header), axis=1) + return result_df def check_cross_framework(bench_json_path): -- Gitee From edbb99a0b5a6fe94540288ff69bebfe899a0d2bf Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 20:52:04 +0800 Subject: [PATCH 5/9] temporary modify --- debug/accuracy_tools/msprobe/core/compare/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index b57266ae7..55209b33e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -137,7 +137,6 @@ def op_item_parse(op_data, op_name: str, depth: int = 0) -> list: return [gen_op_item(op_data, op_name)] for sub_name, sub_data in op_data.items(): item_list.extend(op_item_parse(sub_data, op_name + Const.SEP + str(sub_name), depth + 1)) - return item_list @@ -465,7 +464,6 @@ def merge_tensor(tensor_list, dump_mode): def _compare_parser(parser): - parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) parser.add_argument("-o", "--output_path", dest="output_path", type=str, -- Gitee From 5319832fd73e98dbd91663fa13f26a5edb0aad20 Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 21:14:20 +0800 Subject: [PATCH 6/9] temporary modify --- .../msprobe/mindspore/compare/ms_compare.py | 136 +++++++++--------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 81c8361e2..c93d392ea 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -53,6 +53,74 @@ class MSComparator(Comparator): else: raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " f"{type(self.data_mapping)}") + + @classmethod + def calc_accuracy(cls, row, dump_mode, header): + def calc_summary_diff(data_type: str): + need_warning = False + ms_val, pt_val = row['NPU ' + data_type], row['Bench ' + data_type] + if all(isinstance(val, (float, int)) and not isinstance(val, bool) for val in [ms_val, pt_val]): + diff = ms_val - pt_val + if math.isnan(diff): + row[data_type.capitalize() + ' diff'] = CompareConst.NAN + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.NAN + else: + if pt_val != 0: + row[data_type.capitalize() + 'RelativeErr'] = str(abs((diff / pt_val) * 100)) + '%' + else: + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.N_A + magnitude_diff = abs(diff) / (max(abs(ms_val), abs(pt_val)) + CompareConst.EPSILON) + need_warning = magnitude_diff > CompareConst.MAGNITUDE + return need_warning + + if dump_mode == Const.MD5: + row[CompareConst.RESULT] = CompareConst.PASS if row[CompareConst.NPU_MD5] == row[CompareConst.BENCH_MD5] else CompareConst.DIFF + elif dump_mode == Const.SUMMARY: + warning_flag = any([calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']]) + row[CompareConst.RESULT] = CompareConst.WARNING if warning_flag else "" + row[CompareConst.ERROR_MESSAGE] = "Need double check api accuracy." if warning_flag else "" + else: + row[CompareConst.COSINE] = '' + row[CompareConst.MAX_ABS_ERR] = '' + row[CompareConst.MAX_RELATIVE_ERR] = '' + row[CompareConst.ONE_THOUSANDTH_ERR_RATIO] = '' + row[CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = '' + row[CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES + row[CompareConst.ERROR_MESSAGE] = '' + return row[header] + + @classmethod + def make_result_table(cls, result, stack_mode, dump_mode): + header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] + + if dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) + if stack_mode: + header.append(CompareConst.STACK) + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, + 'op_name_y': CompareConst.BENCH_NAME, + 'dtype_x': CompareConst.NPU_DTYPE, + 'dtype_y': CompareConst.BENCH_DTYPE, + 'shape_x': CompareConst.NPU_SHAPE, + 'shape_y': CompareConst.BENCH_SHAPE, + 'md5_x': CompareConst.NPU_MD5, + 'md5_y': CompareConst.BENCH_MD5, + 'data_name_x': CompareConst.DATA_NAME, + 'stack_info_x': CompareConst.STACK}, inplace=True) + result[CompareConst.NPU_MAX] = result['summary_x'].apply(lambda row: row[0]) + result[CompareConst.NPU_MIN] = result['summary_x'].apply(lambda row: row[1]) + result[CompareConst.NPU_MEAN] = result['summary_x'].apply(lambda row: row[2]) + result[CompareConst.NPU_NORM] = result['summary_x'].apply(lambda row: row[3]) + result[CompareConst.BENCH_MAX] = result['summary_y'].apply(lambda row: row[0] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MIN] = result['summary_y'].apply(lambda row: row[1] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MEAN] = result['summary_y'].apply(lambda row: row[2] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_NORM] = result['summary_y'].apply(lambda row: row[3] if row != CompareConst.N_A else row) + result_df = pd.DataFrame(columns=header) + for h in header: + if h in result.columns: + result_df[h] = result[h] + result_df = result_df.apply(lambda row: cls.calc_accuracy(row, dump_mode, header), axis=1) + return result_df def load_internal_api(self): cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -121,41 +189,6 @@ class MSComparator(Comparator): raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error return api_name - @staticmethod - def calc_accuracy(row, dump_mode, header): - def calc_summary_diff(data_type: str): - need_warning = False - ms_val, pt_val = row['NPU ' + data_type], row['Bench ' + data_type] - if all(isinstance(val, (float, int)) and not isinstance(val, bool) for val in [ms_val, pt_val]): - diff = ms_val - pt_val - if math.isnan(diff): - row[data_type.capitalize() + ' diff'] = CompareConst.NAN - row[data_type.capitalize() + 'RelativeErr'] = CompareConst.NAN - else: - if pt_val != 0: - row[data_type.capitalize() + 'RelativeErr'] = str(abs((diff / pt_val) * 100)) + '%' - else: - row[data_type.capitalize() + 'RelativeErr'] = CompareConst.N_A - magnitude_diff = abs(diff) / (max(abs(ms_val), abs(pt_val)) + CompareConst.EPSILON) - need_warning = magnitude_diff > CompareConst.MAGNITUDE - return need_warning - - if dump_mode == Const.MD5: - row[CompareConst.RESULT] = CompareConst.PASS if row[CompareConst.NPU_MD5] == row[CompareConst.BENCH_MD5] else CompareConst.DIFF - elif dump_mode == Const.SUMMARY: - warning_flag = any([calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']]) - row[CompareConst.RESULT] = CompareConst.WARNING if warning_flag else "" - row[CompareConst.ERROR_MESSAGE] = "Need double check api accuracy." if warning_flag else "" - else: - row[CompareConst.COSINE] = '' - row[CompareConst.MAX_ABS_ERR] = '' - row[CompareConst.MAX_RELATIVE_ERR] = '' - row[CompareConst.ONE_THOUSANDTH_ERR_RATIO] = '' - row[CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = '' - row[CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES - row[CompareConst.ERROR_MESSAGE] = '' - return row[header] - def compare_process(self, file_lists, stack_mode, fuzzy_match, dump_mode): npu_json_path, bench_json_path, stack_json_path = file_lists npu_json_data = load_json(npu_json_path) @@ -268,39 +301,6 @@ class MSComparator(Comparator): result['data_name'].append(merge_list['data_name'].pop(0)) return pd.DataFrame(result) - @classmethod - def make_result_table(cls, result, stack_mode, dump_mode): - header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] - - if dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) - if stack_mode: - header.append(CompareConst.STACK) - result.rename(columns={'op_name_x': CompareConst.NPU_NAME, - 'op_name_y': CompareConst.BENCH_NAME, - 'dtype_x': CompareConst.NPU_DTYPE, - 'dtype_y': CompareConst.BENCH_DTYPE, - 'shape_x': CompareConst.NPU_SHAPE, - 'shape_y': CompareConst.BENCH_SHAPE, - 'md5_x': CompareConst.NPU_MD5, - 'md5_y': CompareConst.BENCH_MD5, - 'data_name_x': CompareConst.DATA_NAME, - 'stack_info_x': CompareConst.STACK}, inplace=True) - result[CompareConst.NPU_MAX] = result['summary_x'].apply(lambda row: row[0]) - result[CompareConst.NPU_MIN] = result['summary_x'].apply(lambda row: row[1]) - result[CompareConst.NPU_MEAN] = result['summary_x'].apply(lambda row: row[2]) - result[CompareConst.NPU_NORM] = result['summary_x'].apply(lambda row: row[3]) - result[CompareConst.BENCH_MAX] = result['summary_y'].apply(lambda row: row[0] if row != CompareConst.N_A else row) - result[CompareConst.BENCH_MIN] = result['summary_y'].apply(lambda row: row[1] if row != CompareConst.N_A else row) - result[CompareConst.BENCH_MEAN] = result['summary_y'].apply(lambda row: row[2] if row != CompareConst.N_A else row) - result[CompareConst.BENCH_NORM] = result['summary_y'].apply(lambda row: row[3] if row != CompareConst.N_A else row) - result_df = pd.DataFrame(columns=header) - for h in header: - if h in result.columns: - result_df[h] = result[h] - result_df = result_df.apply(lambda row: cls.calc_accuracy(row, dump_mode, header), axis=1) - return result_df - def check_cross_framework(bench_json_path): pattern = r'"data_name":\s*"[^"]+\.pt"' -- Gitee From 78e27dd0ff1944cba36d8e6effd7027f1bdb6cde Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 21:31:37 +0800 Subject: [PATCH 7/9] temporary modify --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 18d45228d..9f3c8701f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -329,7 +329,7 @@ class Comparator: bench_ops_all = self.merge_data(bench_json_data, stack_json_data, dump_mode) result = self.get_accuracy(npu_ops_all, bench_ops_all, dump_mode) - result_df = self.make_result_table(result, stack_mode, dump_mode) + result_df = Comparator.make_result_table(result, stack_mode, dump_mode) return result_df def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): -- Gitee From b81aa938a674f0fcb652b600960353385c887004 Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 22:09:19 +0800 Subject: [PATCH 8/9] temporary modify --- .../accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index c93d392ea..919b9a642 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -93,10 +93,10 @@ class MSComparator(Comparator): def make_result_table(cls, result, stack_mode, dump_mode): header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] - if dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) if stack_mode: header.append(CompareConst.STACK) + if dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) result.rename(columns={'op_name_x': CompareConst.NPU_NAME, 'op_name_y': CompareConst.BENCH_NAME, 'dtype_x': CompareConst.NPU_DTYPE, @@ -285,6 +285,8 @@ class MSComparator(Comparator): for data in data_json['data']: check_op_str_pattern_valid(data) merge_list = self.gen_merge_list(data_json, data, stack_json, dump_mode) + if not merge_list: + continue for op_name in merge_list['op_name']: result['op_name'].append(op_name) if Const.SEP + Const.INPUT + Const.SEP in op_name: -- Gitee From 8b954e4167fe734eea0ec14db3ff1282fa076411 Mon Sep 17 00:00:00 2001 From: wangchao426 Date: Mon, 11 Nov 2024 22:09:19 +0800 Subject: [PATCH 9/9] temporary modify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: wangchao426 修改提交人 --- .../accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index c93d392ea..919b9a642 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -93,10 +93,10 @@ class MSComparator(Comparator): def make_result_table(cls, result, stack_mode, dump_mode): header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] - if dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) if stack_mode: header.append(CompareConst.STACK) + if dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) result.rename(columns={'op_name_x': CompareConst.NPU_NAME, 'op_name_y': CompareConst.BENCH_NAME, 'dtype_x': CompareConst.NPU_DTYPE, @@ -285,6 +285,8 @@ class MSComparator(Comparator): for data in data_json['data']: check_op_str_pattern_valid(data) merge_list = self.gen_merge_list(data_json, data, stack_json, dump_mode) + if not merge_list: + continue for op_name in merge_list['op_name']: result['op_name'].append(op_name) if Const.SEP + Const.INPUT + Const.SEP in op_name: -- Gitee