diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 18d45228d1458b683e460198ce31c88923ff788e..9f3c8701f8af9e64df13dd64ea31be6bb3d7d72b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -329,7 +329,7 @@ class Comparator: bench_ops_all = self.merge_data(bench_json_data, stack_json_data, dump_mode) result = self.get_accuracy(npu_ops_all, bench_ops_all, dump_mode) - result_df = self.make_result_table(result, stack_mode, dump_mode) + result_df = Comparator.make_result_table(result, stack_mode, dump_mode) return result_df def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index db1a594da25a03a8848ab999785eaf2c5db40ffa..919b9a6425c83d6edc6371a2c86bd3ee61703af5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -13,20 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy +import math import os import re +import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import (FileOpen, create_directory, +from msprobe.core.common.file_utils import (FileOpen, create_directory, load_json, load_npy, load_yaml) from msprobe.core.common.log import logger from msprobe.core.common.utils import (CompareException, check_compare_param, check_configuration_param, - get_dump_mode, set_dump_path) + get_dump_mode, set_dump_path, check_op_str_pattern_valid) from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.compare.check import check_struct_match, fuzzy_check_op +from msprobe.core.compare.check import dtype_mapping from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping @@ -52,6 +53,74 @@ class MSComparator(Comparator): else: raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " f"{type(self.data_mapping)}") + + @classmethod + def calc_accuracy(cls, row, dump_mode, header): + def calc_summary_diff(data_type: str): + need_warning = False + ms_val, pt_val = row['NPU ' + data_type], row['Bench ' + data_type] + if all(isinstance(val, (float, int)) and not isinstance(val, bool) for val in [ms_val, pt_val]): + diff = ms_val - pt_val + if math.isnan(diff): + row[data_type.capitalize() + ' diff'] = CompareConst.NAN + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.NAN + else: + if pt_val != 0: + row[data_type.capitalize() + 'RelativeErr'] = str(abs((diff / pt_val) * 100)) + '%' + else: + row[data_type.capitalize() + 'RelativeErr'] = CompareConst.N_A + magnitude_diff = abs(diff) / (max(abs(ms_val), abs(pt_val)) + CompareConst.EPSILON) + need_warning = magnitude_diff > CompareConst.MAGNITUDE + return need_warning + + if dump_mode == Const.MD5: + row[CompareConst.RESULT] = CompareConst.PASS if row[CompareConst.NPU_MD5] == row[CompareConst.BENCH_MD5] else CompareConst.DIFF + elif dump_mode == Const.SUMMARY: + warning_flag = any([calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']]) + row[CompareConst.RESULT] = CompareConst.WARNING if warning_flag else "" + row[CompareConst.ERROR_MESSAGE] = "Need double check api accuracy." if warning_flag else "" + else: + row[CompareConst.COSINE] = '' + row[CompareConst.MAX_ABS_ERR] = '' + row[CompareConst.MAX_RELATIVE_ERR] = '' + row[CompareConst.ONE_THOUSANDTH_ERR_RATIO] = '' + row[CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = '' + row[CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES + row[CompareConst.ERROR_MESSAGE] = '' + return row[header] + + @classmethod + def make_result_table(cls, result, stack_mode, dump_mode): + header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode] + + if stack_mode: + header.append(CompareConst.STACK) + if dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, + 'op_name_y': CompareConst.BENCH_NAME, + 'dtype_x': CompareConst.NPU_DTYPE, + 'dtype_y': CompareConst.BENCH_DTYPE, + 'shape_x': CompareConst.NPU_SHAPE, + 'shape_y': CompareConst.BENCH_SHAPE, + 'md5_x': CompareConst.NPU_MD5, + 'md5_y': CompareConst.BENCH_MD5, + 'data_name_x': CompareConst.DATA_NAME, + 'stack_info_x': CompareConst.STACK}, inplace=True) + result[CompareConst.NPU_MAX] = result['summary_x'].apply(lambda row: row[0]) + result[CompareConst.NPU_MIN] = result['summary_x'].apply(lambda row: row[1]) + result[CompareConst.NPU_MEAN] = result['summary_x'].apply(lambda row: row[2]) + result[CompareConst.NPU_NORM] = result['summary_x'].apply(lambda row: row[3]) + result[CompareConst.BENCH_MAX] = result['summary_y'].apply(lambda row: row[0] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MIN] = result['summary_y'].apply(lambda row: row[1] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_MEAN] = result['summary_y'].apply(lambda row: row[2] if row != CompareConst.N_A else row) + result[CompareConst.BENCH_NORM] = result['summary_y'].apply(lambda row: row[3] if row != CompareConst.N_A else row) + result_df = pd.DataFrame(columns=header) + for h in header: + if h in result.columns: + result_df[h] = result[h] + result_df = result_df.apply(lambda row: cls.calc_accuracy(row, dump_mode, header), axis=1) + return result_df def load_internal_api(self): cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -66,40 +135,14 @@ class MSComparator(Comparator): return mapping_dict def process_cell_mapping(self, npu_op_name): - npu_op_name = [op_name.replace("Cell", "Module", 1) for op_name in npu_op_name] + npu_op_name = npu_op_name.replace("Cell", "Module", 1) if self.cell_mapping_dict: - for index, op_name in enumerate(npu_op_name): - # get cell name & class name from op_name - # Cell.fc1.Dense.forward.0.input.0 - cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] - if cell_name in self.cell_mapping_dict: - npu_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) + # get cell name & class name from op_name + # Cell.fc1.Dense.forward.0.input.0 + cell_name = npu_op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] + if cell_name in self.cell_mapping_dict: + npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) return npu_op_name - - def check_op(self, npu_dict, bench_dict, fuzzy_match): - npu_dict_new, bench_dict_new = copy.deepcopy(npu_dict), copy.deepcopy(bench_dict) - npu_op_name, bench_op_name = npu_dict_new.get(CompareConst.OP_NAME), bench_dict_new.get(CompareConst.OP_NAME) - if self.cell_mapping is not None: - npu_op_name = self.process_cell_mapping(npu_op_name) - if self.api_mapping is not None: - npu_op_name = self.process_internal_api_mapping(npu_op_name, bench_op_name) - if isinstance(self.api_mapping, str): - npu_dict_new, bench_dict_new, target_dict = self.transform_user_mapping_api(npu_dict_new, - bench_dict_new) - if target_dict: - bench_dict = self.reconstitution_bench_dict(npu_dict, copy.deepcopy(bench_dict_new), target_dict) - npu_op_name = npu_dict_new.get(CompareConst.OP_NAME) - bench_op_name = bench_dict_new.get(CompareConst.OP_NAME) - struct_match = check_struct_match(npu_dict_new, bench_dict_new, cross_frame=self.cross_frame) - if not fuzzy_match: - return npu_op_name == bench_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(npu_op_name, bench_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name)) - is_match = False - return is_match and struct_match def read_npy_data(self, dir_path, file_name, load_pt_file=False): data_path = os.path.join(dir_path, file_name) @@ -119,19 +162,17 @@ class MSComparator(Comparator): npu_op_name[idx] = npu_op_name[idx].replace(target, para) return npu_op_name - def process_internal_api_mapping(self, npu_op_name, bench_op_name): + def process_internal_api_mapping(self, npu_op_name): # get api name & class name from op_name # Functional.addcmul.0.forward.input.0 - npu_op_name, bench_op_name = npu_op_name.copy(), bench_op_name.copy() - ms_api_name = self.get_api_name(npu_op_name[0].split(Const.SEP)) - pt_api_name = self.get_api_name(bench_op_name[0].split(Const.SEP)) + ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) class_name = ms_api_name.split(Const.SEP)[0] if class_name == "Mint": - return self.api_replace(npu_op_name, "Mint", "Torch") + return npu_op_name.replace("Mint", "Torch") elif class_name == "MintFunctional": - return self.api_replace(npu_op_name, "MintFunctional", "Functional") - elif self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: - return self.api_replace(npu_op_name, ms_api_name, pt_api_name) + return npu_op_name.replace("MintFunctional", "Functional") + elif self.ms_to_pt_mapping.get(ms_api_name): + return npu_op_name.replace(ms_api_name, self.ms_to_pt_mapping.get(ms_api_name)) else: return npu_op_name @@ -148,108 +189,119 @@ class MSComparator(Comparator): raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error return api_name - def transform_user_mapping_api(self, new_npu_dict, new_bench_dict): - """ - Transform user mapping API based on new NPU and benchmark dictionaries. - Parameters: - new_npu_dict (dict): New NPU operation dictionary. - new_bench_dict (dict): New benchmark operation dictionary. - Returns: - tuple: Updated NPU and benchmark dictionaries, along with the target dictionary. - """ - npu_op_name, bench_op_name = new_npu_dict.get(CompareConst.OP_NAME), new_bench_dict.get(CompareConst.OP_NAME) - npu_struct_in = new_npu_dict.get(CompareConst.INPUT_STRUCT) - bench_struct_in = new_bench_dict.get(CompareConst.INPUT_STRUCT) - npu_struct_out = new_npu_dict.get(CompareConst.OUTPUT_STRUCT) - bench_struct_out = new_bench_dict.get(CompareConst.OUTPUT_STRUCT) - npu_summary, bench_summary = new_npu_dict.get(CompareConst.SUMMARY), new_bench_dict.get(CompareConst.SUMMARY) - npu_in_len, bench_in_len = len(npu_struct_in), len(bench_struct_in) - npu_out_len, bench_out_len = len(npu_struct_out), len(bench_struct_out) - ms_api_list, pt_api_list = npu_op_name[0].split(Const.SEP), bench_op_name[0].split(Const.SEP) - ms_api_name = self.get_api_name(ms_api_list) - pt_api_name = self.get_api_name(pt_api_list) - target_dict = {} - for api_dict in self.api_mapping_dict: - if api_dict.get("pt_api") == pt_api_name and api_dict.get("ms_api") == ms_api_name: - ms_user_args_len, pt_user_args_len = len(api_dict.get("ms_args")), len(api_dict.get("pt_args")) - ms_user_output_len, pt_user_output_len = len(api_dict.get("ms_output")), len(api_dict.get("pt_output")) - if ms_user_args_len != pt_user_args_len or ms_user_output_len != pt_user_output_len: - logger.warning("The user-defined mapping table is incorrect,\ - make sure that the number of parameters is equal") - break - ms_out_list = api_dict.get("ms_output", []) - for idx in reversed(range(npu_out_len)): - if idx not in ms_out_list: - del npu_struct_out[idx] - if idx + npu_in_len < len(npu_summary) and idx + npu_in_len < len(npu_op_name): - del npu_summary[idx + npu_in_len] - del npu_op_name[idx + npu_in_len] - pt_out_list = api_dict.get("pt_output", []) - for idx in reversed(range(bench_out_len)): - if idx not in pt_out_list: - del bench_struct_out[idx] - if idx + bench_in_len < len(bench_summary) and idx + bench_in_len < len(bench_op_name): - del bench_summary[idx + bench_in_len] - del bench_op_name[idx + bench_in_len] - ms_para_list = api_dict.get("ms_args", []) - for idx in reversed(range(npu_in_len)): - if idx not in ms_para_list: - self.remove_element(npu_op_name, npu_struct_in, npu_summary, idx) - pt_para_list = api_dict.get("pt_args", []) - for idx in reversed(range(bench_in_len)): - if idx not in pt_para_list: - self.remove_element(bench_op_name, bench_struct_in, bench_summary, idx) - npu_op_name = self.api_replace(npu_op_name, ms_api_name, pt_api_name) - if len(npu_op_name) != len(bench_op_name): - logger.warning( - "The total number of input and output parameters of \ - npu_op_name and bench_op_name are not equal.") - break - npu_op_name = self.para_sequence_update(npu_op_name, bench_op_name) - target_dict = api_dict - break - if target_dict: - new_npu_dict.update({CompareConst.OP_NAME: npu_op_name, CompareConst.INPUT_STRUCT: npu_struct_in, - CompareConst.OUTPUT_STRUCT: npu_struct_out, CompareConst.SUMMARY: npu_summary}) - new_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, - CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) - return new_npu_dict, new_bench_dict, target_dict + def compare_process(self, file_lists, stack_mode, fuzzy_match, dump_mode): + npu_json_path, bench_json_path, stack_json_path = file_lists + npu_json_data = load_json(npu_json_path) + bench_json_data = load_json(bench_json_path) + stack_json_data = load_json(stack_json_path) + + npu_df = self.gen_data_df(npu_json_data, stack_json_data, dump_mode) + bench_df = self.gen_data_df(bench_json_data, stack_json_data, dump_mode) + if self.cell_mapping: + npu_df['compare_key'] = npu_df.apply(lambda row: self.process_cell_mapping(row['op_name']), axis=1) + elif self.api_mapping: + npu_df['compare_key'] = npu_df.apply(lambda row: self.process_internal_api_mapping(row['op_name']), axis=1) + if isinstance(self.api_mapping, str): + self.modify_compare_data_with_user_mapping(npu_df, bench_df) + else: + npu_df['compare_key'] = npu_df['op_name'] + npu_df['compare_shape'] = npu_df[Const.SHAPE].apply(str) + bench_df['compare_shape'] = bench_df[Const.SHAPE].apply(str) + bench_df['compare_key'] = bench_df['op_name'] + match_result = pd.merge(npu_df, bench_df, on=['compare_key', 'compare_shape'], how='outer') + match_result = match_result[self.gen_compare_condition(match_result)].fillna(CompareConst.N_A) + return MSComparator.make_result_table(match_result, stack_mode, dump_mode) - def para_sequence_update(self, npu_op_name, bench_op_name): - for idx, _ in enumerate(npu_op_name): - bench_op_name_list = bench_op_name[idx].rsplit(Const.SEP, 1) - if len(bench_op_name_list) != 0: - npu_op_name[idx] = npu_op_name[idx].rsplit(Const.SEP, 1)[0] + Const.SEP + bench_op_name_list[-1] - return npu_op_name + def modify_compare_data_with_user_mapping(self, npu_df, bench_df): + def get_api_indices_dict(op_name_df): + api_indices_dict = {} + for op_index, name in enumerate(op_name_df['op_name']): + api = self.get_api_name(name.split(Const.SEP)) + if api in api_indices_dict: + api_indices_dict[api].append(op_index) + else: + api_indices_dict[api] = [op_index] + return api_indices_dict + + ms_api_indices_dict = get_api_indices_dict(npu_df) + pt_api_indices_dict = get_api_indices_dict(bench_df) + + for mapping_dict in self.api_mapping_dict: + if (len(mapping_dict.get('ms_args')) != len(mapping_dict.get('pt_args')) or + len(mapping_dict.get('ms_output')) != len(mapping_dict.get('pt_output'))): + logger.warning('The user-defined mapping table is incorrect,\ + make sure that the number of parameters is equal') + continue + ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') + if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: + continue + for index in ms_api_indices_dict.get(ms_api): + op_name = npu_df.loc[index, 'op_name'].replace(ms_api, pt_api, 1) + is_abandoned = True + if '.input.' in op_name: + for i, prefix in enumerate(mapping_dict.get('ms_args')): + if op_name.split('.input.')[1].startswith(str(prefix)): + npu_df.loc[index, 'compare_key'] = ( + op_name.replace('.input.' + str(prefix), + '.input.' + str(mapping_dict.get('pt_args')[i]))) + is_abandoned = False + else: + for i, prefix in enumerate(mapping_dict.get('ms_output')): + if op_name.split('.output.')[1].startswith(str(prefix)): + npu_df.loc[index, 'compare_key'] = ( + op_name.replace('.output.' + str(prefix), + '.output.' + str(mapping_dict.get('pt_output')[i]))) + is_abandoned = False + if is_abandoned: + npu_df.loc[index, 'compare_key'] = op_name + 'abandoned' + + def gen_compare_condition(self, match_result): + match_result['compare_dtype'] = match_result['dtype_x'] + if self.cross_frame: + match_result['compare_dtype'] = match_result['dtype_x'].apply(lambda row: dtype_mapping.get(row, row)) + return (match_result['compare_dtype'] == match_result['dtype_y']) | \ + ((match_result['compare_dtype'] == Const.FLOAT16) & ( + match_result['dtype_y'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.FLOAT16) & ( + match_result['compare_dtype'].isin([Const.FLOAT32, Const.BFLOAT16]))) | \ + ((match_result['compare_dtype'] == Const.TORCH_FLOAT16) & ( + match_result['dtype_y'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + ((match_result['dtype_y'] == Const.TORCH_FLOAT16) & ( + match_result['compare_dtype'].isin([Const.TORCH_FLOAT32, Const.TORCH_BFLOAT16]))) | \ + (match_result['op_name_x'].notna() & match_result['op_name_y'].isna()) - def reconstitution_bench_dict(self, npu_dict, del_bench_dict, api_dict): - ms_user_args_list = api_dict.get("ms_args", []) - ms_user_output_list = api_dict.get("ms_output", []) - npu_struct_in = npu_dict.get(CompareConst.INPUT_STRUCT) - npu_struct_out = npu_dict.get(CompareConst.OUTPUT_STRUCT) - npu_in_len = len(npu_struct_in) - npu_out_len = len(npu_struct_out) - if npu_in_len == len(ms_user_args_list) and npu_out_len == len(ms_user_output_list): - return del_bench_dict - ms_input_args_list = [i for i in range(npu_in_len)] - input_sub_list = list(set(ms_input_args_list) - set(ms_user_args_list)) - ms_output_args_list = [i for i in range(npu_out_len)] - output_sub_list = list(set(ms_output_args_list) - set(ms_user_output_list)) - bench_op_name = del_bench_dict.get(CompareConst.OP_NAME, []) - bench_struct_in = del_bench_dict.get(CompareConst.INPUT_STRUCT, []) - bench_struct_out = del_bench_dict.get(CompareConst.OUTPUT_STRUCT, []) - bench_summary = del_bench_dict.get(CompareConst.SUMMARY, []) - for idx in input_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(idx, CompareConst.N_A) - bench_struct_in.insert(idx, CompareConst.N_A) - bench_summary.insert(idx, CompareConst.N_A) - for idx in output_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(npu_in_len + idx, CompareConst.N_A) - bench_struct_out.insert(idx, CompareConst.N_A) - bench_summary.insert(npu_in_len + idx, CompareConst.N_A) - del_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, - CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) - return del_bench_dict + def gen_data_df(self, data_json, stack_json, dump_mode): + result = { + 'op_name': [], + Const.DTYPE: [], + Const.SHAPE: [], + Const.SUMMARY: [], + 'stack_info': [] + } + if dump_mode == Const.ALL: + result['data_name'] = [] + elif dump_mode == Const.MD5: + result[Const.MD5] = [] + for data in data_json['data']: + check_op_str_pattern_valid(data) + merge_list = self.gen_merge_list(data_json, data, stack_json, dump_mode) + if not merge_list: + continue + for op_name in merge_list['op_name']: + result['op_name'].append(op_name) + if Const.SEP + Const.INPUT + Const.SEP in op_name: + struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) + else: + struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) + result[Const.DTYPE].append(struct[0]) + result[Const.SHAPE].append(struct[1]) + if dump_mode == Const.MD5: + result[Const.MD5].append(struct[2]) + result[Const.SUMMARY].append(merge_list[Const.SUMMARY].pop(0)) + result['stack_info'].append(merge_list['stack_info'][0]) + if dump_mode == Const.ALL: + result['data_name'].append(merge_list['data_name'].pop(0)) + return pd.DataFrame(result) def check_cross_framework(bench_json_path):