diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index adcc171f6710b435e97e64313595bdbc74b4d256..dc0ce749e4b9c9b92627f3639c5d94a9f146b5c5 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -211,6 +211,7 @@ class Const: DTYPE = 'dtype' SHAPE = 'shape' + STACK_INFO = 'stack_info' MAX = 'Max' MIN = 'Min' MEAN = 'Mean' @@ -477,13 +478,6 @@ class CompareConst: Const.PARAMS_GRAD: PARAMS_GRAD_STRUCT } - STRUCT_COMPARE_KEY = [ - INPUT_STRUCT, - OUTPUT_STRUCT, - PARAMS_STRUCT, - PARAMS_GRAD_STRUCT - ] - # compare standard HUNDRED_RATIO_THRESHOLD = 0.01 THOUSAND_RATIO_THRESHOLD = 0.001 @@ -562,15 +556,33 @@ class CompareConst: MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None, MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None } + + API_MAPPING_KEYS_TO_COMPARE = [ + ('ms_args', 'pt_args'), + ('ms_outputs', 'pt_outputs'), + ('ms_parameters', 'pt_parameters'), + ('ms_parameters_grad', 'pt_parameters_grad') + ] + INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP PARAMS_PATTERN = Const.SEP + Const.PARAMS + Const.SEP PARAMS_GRAD_PATTERN = Const.SEP + Const.PARAMS_GRAD + Const.SEP - COMPARE_KEY = 'compare_key' - COMPARE_SHAPE = 'compare_shape' + CMP_KEY = 'compare_key' + CMP_SHAPE = 'compare_shape' + + MATCH_RESULT_COLUMNS = [ + 'op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + CMP_KEY, CMP_SHAPE, + 'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + ] + INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml' UNREADABLE = 'unreadable data' + NPU_DUMP_DATA_DIR = 'npu_dump_data_dir' + BENCH_DUMP_DATA_DIR = 'bench_dump_data_dir' + NO_REAL_DATA_FLAG = '-1' class FileCheckConst: diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index efd9518e8411669c4c9ca4d6e2a09e1dff1cf819..99270c7ababfcda321c37dacf4d5dfb1b6f60271 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -284,8 +284,8 @@ def set_dump_path(input_param): if not npu_path_valid or not bench_path_valid: logger.error(f"Please check the json path is valid and ensure that neither npu_path nor bench_path is None.") raise CompareException(CompareException.INVALID_PATH_ERROR) - input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA) - input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA) + input_param[CompareConst.NPU_DUMP_DATA_DIR] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA) + input_param[CompareConst.BENCH_DUMP_DATA_DIR] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA) def get_dump_mode(input_param): diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 3765b91ae95d3af01aeb9b746af321481a8e142b..abc2f87ffb9cf55ae7fc849617c30674076ea166 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -13,111 +13,224 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing import os import re -from copy import deepcopy +from dataclasses import dataclass +from collections import defaultdict +import numpy as np import pandas as pd from tqdm import tqdm from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import load_json, remove_path +from msprobe.core.common.file_utils import load_json, remove_path, create_directory from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, safe_get_value -from msprobe.core.compare.check import check_dump_json_str, check_graph_mode, check_stack_json_str, \ - check_struct_match, fuzzy_check_op -from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result -from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg -from msprobe.core.compare.utils import get_accuracy, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list +from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, \ + set_dump_path, get_dump_mode, check_compare_param, check_configuration_param +from msprobe.core.compare.check import check_dump_json_str, check_stack_json_str, cross_dtype_mapping +from msprobe.core.compare.utils import merge_tensor, print_compare_ends_info, read_op, \ + reorder_op_x_list, set_stack_json_path +from msprobe.core.compare.config import ModeConfig, MappingConfig, MappingDict +from msprobe.core.compare.multiprocessing_compute import CompareRealData +from msprobe.core.compare.highlight import HighLight + + +@dataclass +class ComparisonConfig: + dump_mode: str + stack_mode: bool + auto_analyze: bool + fuzzy_match: bool + data_mapping: dict + suffix: str + cell_mapping: dict + api_mapping: dict + layer_mapping: dict -class ModeConfig: - def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=None): - self.stack_mode = stack_mode - self.auto_analyze = auto_analyze - self.fuzzy_match = fuzzy_match - self.dump_mode = dump_mode +class Comparator: + def __init__(self, file_reader, mode_config: ModeConfig, mapping_config: MappingConfig, is_cross_framework=False): + self.file_reader = file_reader + self.mode_config = mode_config + self.mapping_config = mapping_config + if self.mapping_config.data_mapping: + self.cross_frame = is_cross_framework + else: + self.cross_frame = (self.mapping_config.cell_mapping is not None or + self.mapping_config.api_mapping is not None) -class Comparator: - def __init__(self, mode_config: ModeConfig): - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode + self.mapping_dict = MappingDict(mapping_config) @staticmethod - def get_result_md5_compare(ms_op_name, bench_op_name, npu_ops_all, bench_ops_all, *args): - npu_struct = npu_ops_all.get(ms_op_name).get('struct', []) - bench_struct = bench_ops_all.get(bench_op_name).get('struct', []) + def process_output_file(output_path, suffix): + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + if os.path.exists(file_path): + logger.warning(f"{file_path} will be deleted.") + remove_path(file_path) + return file_path - if len(npu_struct) < 3 or len(bench_struct) < 3: - logger.error(f"The length of npu_struct and bench_struct must be >= 3, " - f"but got npu_struct={len(npu_struct)} and bench_struct={len(bench_struct)}. Please check!") - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + def compare_core(self, input_param, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. - result_item = [ms_op_name, bench_op_name, npu_struct[0], bench_struct[0], - npu_struct[1], bench_struct[1], npu_struct[2], bench_struct[2], - CompareConst.PASS if npu_struct[2] == bench_struct[2] else CompareConst.DIFF] + Args: + input_param (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - dump_mode (str): ALL, SUMMARY, MD5. - if len(args) >= 2 and args[0]: - result_item.extend(args[1]) - else: - result_item.append(CompareConst.NONE) - return result_item + Returns: + """ + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - @staticmethod - def calculate_summary_data(npu_summary_data, bench_summary_data, result_item): - err_msg = "" - result_item, accuracy_check, err_msg = get_rela_diff_summary_mode(result_item, npu_summary_data, - bench_summary_data, err_msg) - result_item.append(accuracy_check) - result_item.append(err_msg) + # get kwargs or set default value + suffix = kwargs.get('suffix', '') - @staticmethod - def _generate_na_data(ops_all): - if not ops_all: - return {} - key = next(iter(ops_all)) - value = deepcopy(ops_all[key]) - for k, v in value.items(): - if isinstance(v, tuple): - value[k] = tuple(CompareConst.N_A for _ in range(len(v))) - elif isinstance(v, list): - value[k] = [CompareConst.N_A] * len(v) - else: - value[k] = CompareConst.N_A - return value + # process output file + file_path = self.process_output_file(output_path, suffix) + + # initialize the compare result table and compare general data(name, dtype, shape, statistics/md5, etc.) + npu_json = input_param.get("npu_json_path") + bench_json = input_param.get("bench_json_path") + stack_json = input_param.get("stack_json_path") + result_df = self.compare_statistics([npu_json, bench_json, stack_json]) + if not result_df.values.tolist(): + logger.warning("Can`t match any op.") + return - def make_result_table(self, result): - header = CompareConst.HEAD_OF_COMPARE_MODE[self.dump_mode][:] + # compare real data + if self.mode_config.dump_mode == Const.ALL: + compare_real_data = CompareRealData(self.file_reader, self.mode_config, self.cross_frame) + result_df = compare_real_data.do_multi_process(input_param, result_df) - if self.stack_mode: - header.append(CompareConst.STACK) - if self.dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) - else: - if self.dump_mode == Const.ALL: - for row in result: - del row[-2] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，真实数据时为倒数第2列 - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，非真实数据时为倒数第1列 - result_df = pd.DataFrame(result, columns=header, dtype='object') - return result_df + # highlight suspicious API + highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + highlight = HighLight(self.mode_config) + highlight.find_compare_result_error_rows(result_df, highlight_dict) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + + # output compare analysis suggestions + if self.mode_config.auto_analyze: + advisor = Advisor(result_df, output_path, suffix) + advisor.analysis() + + print_compare_ends_info() + + def compare_statistics(self, file_list): + # load and parse json data + parse_data = ParseData(self.mode_config) + npu_df, bench_df = parse_data.parse(file_list) + + npu_df[[Const.DTYPE, Const.SHAPE]] = npu_df[[Const.DTYPE, Const.SHAPE]].astype(str) + bench_df[[Const.DTYPE, Const.SHAPE]] = bench_df[[Const.DTYPE, Const.SHAPE]].astype(str) + + # create new columns for compare op_name and shape + # process npu_df's COMPARE_KEY whether same or different framework + process_df = ProcessDf(self.mode_config, self.mapping_config, self.mapping_dict) + npu_df, bench_df = process_df.process_compare_key_and_shape(npu_df, bench_df) + + # match npu and bench, match_result contains both npu_info and bench_info + match = Match(self.mode_config, self.mapping_config, self.cross_frame) + match_result = match.match_api_infos(npu_df, bench_df) + # 筛选出npu_name存在的行并填充筛选出行中的缺失值为N/A + match_result = match_result[match_result['op_name_x'].notna()].fillna(CompareConst.N_A) + bench_columns = [i + '_y' for i in bench_df.columns] + match_result.loc[~match.gen_dtype_condition(match_result), bench_columns] = CompareConst.N_A + + # organize compare result table by renaming columns + create_table = CreateTable(self.mode_config) + result_df, header = create_table.make_result_df(match_result) + + # calculate statistics diff + calc_stats_diff = CalcStatsDiff(self.mode_config) + return calc_stats_diff.calc_accuracy(result_df, header) + + +class ParseData: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config + + def parse(self, file_list): + npu_json_path, bench_json_path, stack_json_path = file_list + npu_json_data = load_json(npu_json_path) + bench_json_data = load_json(bench_json_path) + stack_json_data = load_json(stack_json_path) if self.mode_config.stack_mode else None + + # parse json data and generate df + npu_df = self.gen_data_df(npu_json_data, stack_json_data) + bench_df = self.gen_data_df(bench_json_data, stack_json_data) + + return npu_df, bench_df + + def gen_data_df(self, data_json, stack_json_data): + result = { + CompareConst.OP_NAME: [], + Const.DTYPE: [], + Const.SHAPE: [], + Const.SUMMARY: [], + Const.STACK_INFO: [] + } + if self.mode_config.dump_mode == Const.ALL: + result['data_name'] = [] + elif self.mode_config.dump_mode == Const.MD5: + result[Const.MD5] = [] + + api_nums = len(data_json['data']) + progress_bar = tqdm(total=api_nums, desc="API/Module Read Progress", unit="api/module", ncols=100) + + # 从json中循环解析API数据，遍历所有API + for data_name in data_json['data']: + check_op_str_pattern_valid(data_name) + merge_list = self.gen_merge_list(data_json, data_name, stack_json_data) + if not merge_list: + continue + + op_name_list = merge_list.get(CompareConst.OP_NAME) + summary_list = merge_list.get(Const.SUMMARY) + data_name_list = merge_list.get('data_name') + op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, + summary_list, + data_name_list) + # 遍历单个API的所有item + for index, op_name in enumerate(op_name_reorder): + result[CompareConst.OP_NAME].append(op_name) + if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name): + struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) + elif CompareConst.OUTPUT_PATTERN in op_name: + struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) + elif CompareConst.PARAMS_PATTERN in op_name: + struct = merge_list[CompareConst.PARAMS_STRUCT].pop(0) + else: + struct = merge_list[CompareConst.PARAMS_GRAD_STRUCT].pop(0) + result[Const.DTYPE].append(struct[0]) + result[Const.SHAPE].append(struct[1]) + if self.mode_config.dump_mode == Const.MD5: + result[Const.MD5].append(struct[2]) + result[Const.SUMMARY].append(summary_reorder.pop(0)) + result[Const.STACK_INFO].append( + merge_list[Const.STACK_INFO][0] if index == 0 and self.mode_config.stack_mode else None) + if self.mode_config.dump_mode == Const.ALL: + result['data_name'].append(data_name_reorder.pop(0)) + + progress_bar.update(1) + progress_bar.close() + return pd.DataFrame(result) def gen_merge_list(self, json_data, op_name, stack_json_data): op_data = json_data['data'][op_name] check_dump_json_str(op_data, op_name) op_parsed_list = read_op(op_data, op_name) - if self.stack_mode: + if self.mode_config.stack_mode: stack_info = stack_json_data.get(op_name) if stack_info is not None: check_stack_json_str(stack_info, op_name) @@ -127,392 +240,472 @@ class Comparator: 'full_info': stack_info }) - merge_list = merge_tensor(op_parsed_list, self.dump_mode) + merge_list = merge_tensor(op_parsed_list, self.mode_config.dump_mode) return merge_list - def check_op(self, npu_dict, bench_dict): - npu_op_name = npu_dict[CompareConst.OP_NAME] - bench_op_name = bench_dict[CompareConst.OP_NAME] - graph_mode = check_graph_mode(safe_get_value(npu_op_name, 0, "npu_op_name"), - safe_get_value(bench_op_name, 0, "bench_op_name")) - - frame_name = getattr(self, "frame_name") - if frame_name == "PTComparator": - from msprobe.pytorch.compare.match import graph_mapping - if graph_mode: - return graph_mapping.match(npu_op_name[0], bench_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not self.fuzzy_match: - name_match = npu_op_name == bench_op_name - return name_match and struct_match - try: - name_match = fuzzy_check_op(npu_op_name, bench_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name)) - name_match = False - return name_match and struct_match - def match_op(self, npu_queue, bench_queue): - for b_index, b_op in enumerate(bench_queue[0: -1]): - if self.check_op(npu_queue[-1], b_op): - return len(npu_queue) - 1, b_index - if self.check_op(npu_queue[-1], bench_queue[-1]): - return len(npu_queue) - 1, len(bench_queue) - 1 - for n_index, n_op in enumerate(npu_queue[0: -1]): - if self.check_op(n_op, bench_queue[-1]): - return n_index, len(bench_queue) - 1 - return -1, -1 +class ProcessDf: + def __init__(self, mode_config: ModeConfig, mapping_config: MappingConfig, mapping_dict: MappingDict): + self.mode_config = mode_config + self.mapping_config = mapping_config + self.mapping_dict = mapping_dict - def compare_process(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None + @staticmethod + def get_api_name(api_list): + try: + api_name = api_list[0] + Const.SEP + api_list[1] + except IndexError as error: + logger.error('Failed to retrieve API name, please check if the dump data is reasonable') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error + return api_name + + def process_compare_key_and_shape(self, npu_df, bench_df): + npu_df = self.assign_npu_df_compare_key(npu_df, bench_df) + npu_df[CompareConst.CMP_SHAPE] = npu_df[Const.SHAPE] + bench_df[CompareConst.CMP_KEY] = bench_df[CompareConst.OP_NAME] + bench_df[CompareConst.CMP_SHAPE] = bench_df[Const.SHAPE] + return npu_df, bench_df + + def assign_npu_df_compare_key(self, npu_df, bench_df): + """ + 处理 npu_df 的 COMPARE_KEY 赋值逻辑 - if self.fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + :param npu_df: DataFrame，NPU 对比数据 + :param bench_df: DataFrame，Bench 对比数据 + :return: compare_key(name)处理后的 npu_df + """ + # 处理api_mapping映射 + if self.mapping_config.api_mapping: + # 如果用户不传api_mapping.yaml，先使用内置api_mapping.yaml替换npu_op_name + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_internal_api_mapping) + # 如果用户传入api_mapping.yaml，再使用传入api_mapping.yaml进一步替换npu_op_name + if isinstance(self.mapping_config.api_mapping, str): + self.modify_compare_data_with_user_mapping(npu_df, bench_df) + # 处理cell_mapping映射 + elif self.mapping_config.cell_mapping: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_cell_mapping) + # 处理data_mapping映射 + elif self.mapping_config.data_mapping: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_data_mapping) + else: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME] + return npu_df + + def process_internal_api_mapping(self, npu_op_name): + # get api name & class name from op_name + ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) + class_name = ms_api_name.split(Const.SEP)[0] + if class_name == "Mint": + return npu_op_name.replace("Mint", "Torch") + elif class_name == "MintFunctional": + return npu_op_name.replace("MintFunctional", "Functional") + elif self.mapping_dict.ms_to_pt_mapping.get(ms_api_name): + return npu_op_name.replace(ms_api_name, self.mapping_dict.ms_to_pt_mapping.get(ms_api_name)) + else: + return npu_op_name + + def modify_compare_data_with_user_mapping(self, npu_df, bench_df): + def gen_input_compare_key(pattern, term): + is_unmatched = True + for i, prefix in enumerate(mapping_dict.get(f'ms_{term}')): + if op_name.split(pattern)[1].startswith(str(prefix)): + npu_df.loc[index, CompareConst.CMP_KEY] = ( + op_name.replace(pattern + str(prefix), + pattern + str(mapping_dict.get(f'pt_{term}')[i]))) + is_unmatched = False + return is_unmatched + + ms_api_indices_dict = self.get_api_indices_dict(npu_df) + pt_api_indices_dict = self.get_api_indices_dict(bench_df) + + for mapping_dict in self.mapping_dict.api_mapping_dict: + all_length_equal = True + for k1, k2 in CompareConst.API_MAPPING_KEYS_TO_COMPARE: + if len(mapping_dict.get(k1, [])) != len(mapping_dict.get(k2, [])): + all_length_equal = False + if not all_length_equal: + logger.warning('The user-defined mapping table is incorrect,\ + make sure that the number of parameters is equal') + continue - npu_ops_queue = [] - bench_ops_queue = [] - result = [] + ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') + if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: + continue + for index in ms_api_indices_dict.get(ms_api): + op_name = npu_df.loc[index, CompareConst.OP_NAME].replace(ms_api, pt_api, 1) + if CompareConst.INPUT_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.INPUT_PATTERN, 'args') + elif CompareConst.KWARGS_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.KWARGS_PATTERN, 'args') + elif CompareConst.OUTPUT_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.OUTPUT_PATTERN, 'output') + elif CompareConst.PARAMS_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.PARAMS_PATTERN, 'parameters') + elif CompareConst.PARAMS_GRAD_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.PARAMS_GRAD_PATTERN, 'parameters_grad') + else: + logger.error(f'Excepted op_name: {op_name}') + raise CompareException(CompareException.INVALID_DATA_ERROR) + if is_abandoned: + npu_df.loc[index, CompareConst.CMP_KEY] = op_name + 'abandoned' - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 + def get_api_indices_dict(self, op_name_df): + """ + 生成多个api对应的各自的所有的input、output等的index的键值对字典 + 示例： + {'Functional.conv2d': [0, 1, 2, 3], + 'Functional.batch_norm': [4, 5, 6, 7, 8] + } + """ + api_indices_dict = defaultdict(list) + for op_index, name in enumerate(op_name_df[CompareConst.OP_NAME]): + api_name = self.get_api_name(name.split(Const.SEP)) + api_indices_dict[api_name].append(op_index) + return api_indices_dict + + def process_cell_mapping(self, npu_op_name): + if not npu_op_name: + return CompareConst.N_A + param_grad_flag = Const.PARAMS_GRAD in npu_op_name.split(Const.SEP) + if not param_grad_flag and not re.search(Const.REGEX_FORWARD_BACKWARD, npu_op_name): + return CompareConst.N_A + npu_op_name = npu_op_name.replace("Cell", "Module", 1) + if self.mapping_dict.cell_mapping_dict: + # get cell name & class name from op_name + # Cell.fc1.Dense.forward.0.input.0 + cell_name = re.split(r'\.(?:forward|backward|parameters_grad)\.', npu_op_name.split(Const.SEP, 1)[-1])[0] + if cell_name in self.mapping_dict.cell_mapping_dict: + npu_op_name = npu_op_name.replace(cell_name, self.mapping_dict.cell_mapping_dict[cell_name], 1) + return npu_op_name + + def process_data_mapping(self, npu_op_name): + return self.mapping_dict.data_mapping_dict.get(npu_op_name, npu_op_name) + + +class Match: + def __init__(self, mode_config: ModeConfig, mapping_config: MappingConfig, cross_frame): + self.mode_config = mode_config + self.mapping_config = mapping_config + self.cross_frame = cross_frame - npu_api_nums = len(npu_json_data['data']) - progress_bar = tqdm(total=npu_api_nums, desc="API/Module Read Progress", unit="item", ncols=100) + @staticmethod + def put_unmatched_in_table(match_result, npu_op_item): + npu_columns = npu_op_item.index.tolist()[:-2] + new_columns = [name[:-1] + 'y' for name in npu_columns] + na_series = pd.Series([CompareConst.N_A] * len(new_columns), index=new_columns) + new_result_item = pd.concat([npu_op_item, na_series]).to_frame().T + new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS + match_result = pd.concat([match_result, new_result_item]) + return match_result - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - check_op_str_pattern_valid(op_name_npu) - npu_merge_list = self.gen_merge_list(npu_json_data, op_name_npu, stack_json_data) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - check_op_str_pattern_valid(op_name_bench) - bench_merge_list = self.gen_merge_list(bench_json_data, op_name_bench, stack_json_data) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False + @staticmethod + def put_matched_in_table(match_result, npu_op_item, bench_op_item): + head_len = len(CompareConst.MATCH_RESULT_COLUMNS) + new_result_item = pd.concat([npu_op_item, bench_op_item]).head(head_len).to_frame().T + new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS + match_result = pd.concat([match_result, new_result_item]) + return match_result - progress_bar.update(1) + @staticmethod + def rename_api(op_name): + """ + 原api： {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} + rename后： {api_type}.{api_name}.{API调用次数}.{input/output}.{参数序号} + """ + process = Const.FORWARD if Const.FORWARD in op_name else Const.BACKWARD + name_split = op_name.split(process) + try: + torch_func_index, in_out = name_split[0], name_split[1] + except IndexError as error: + logger.error(f'{op_name} can not be split with {process}, please check!') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error + torch_func_split = torch_func_index.rsplit(Const.SEP, 2) + torch_func = str(torch_func_split[0]) + Const.SEP + process + str(in_out) + return torch_func + + def check_op_item(self, npu_op_item, bench_op_item): + name_match = self.rename_api(npu_op_item[CompareConst.CMP_KEY]) == self.rename_api( + bench_op_item[CompareConst.CMP_KEY]) + shape_match = npu_op_item[CompareConst.CMP_SHAPE] == bench_op_item[CompareConst.CMP_SHAPE] + if name_match and shape_match: + return True + else: + npu_op_name = npu_op_item[CompareConst.OP_NAME] + bench_op_name = bench_op_item[CompareConst.OP_NAME] + check_op_str_pattern_valid(npu_op_name) + check_op_str_pattern_valid(bench_op_name) + logger.warning(f"{npu_op_name} and {bench_op_name} can not fuzzy match") + return False - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue + def match_api_infos(self, npu_df, bench_df): + """ + 正常匹配和模糊匹配 + """ + if self.mapping_config.data_mapping: + match_result = pd.merge(npu_df, bench_df, on=[CompareConst.CMP_KEY], how='outer') + elif not self.mode_config.fuzzy_match: + match_result = pd.merge(npu_df, bench_df, on=[CompareConst.CMP_KEY, CompareConst.CMP_SHAPE], + how='outer') + else: + match_result = self.process_fuzzy_match(npu_df, bench_df) + return match_result - # APIs in NPU and Bench models unconsistent judgment + def process_fuzzy_match(self, npu_df, bench_df): + """ + 模糊匹配通过循环方式匹配api + """ + npu_ops_queue = [] + bench_ops_queue = [] + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + + max_len = max(len(npu_df), len(bench_df)) + min_len = min(len(npu_df), len(bench_df)) + for i in range(max_len): + if i < min_len: + npu_ops_queue.append(npu_df.iloc[i]) + bench_ops_queue.append(bench_df.iloc[i]) + else: + try: + npu_ops_queue.append(npu_df.iloc[i]) + except IndexError: + pass + try: + bench_ops_queue.append(bench_df.iloc[i]) + except IndexError: + pass + + # 如果append之后queue状态不一致，则判断结束 if bool(npu_ops_queue) ^ bool(bench_ops_queue): - logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") break - n_match_point, b_match_point = self.match_op(npu_ops_queue, bench_ops_queue) + npu_match_point, bench_match_point = self.match_op(npu_ops_queue, bench_ops_queue) - # 如果没有匹配到，数据放到队列中，跳过，直到后面匹配到，把匹配之前的api放到不匹配中 - if n_match_point == -1 and b_match_point == -1: + # 如果没有匹配到，数据放到队列中，跳过。直到后面匹配到，把匹配之前的api放到不匹配中 + if npu_match_point == -1 and bench_match_point == -1: continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, self.dump_mode) - get_accuracy(result, n_match_data, b_match_data, self.dump_mode) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - progress_bar.close() + npu_op_item = npu_ops_queue[npu_match_point] + bench_op_item = bench_ops_queue[bench_match_point] + unmatched_data = npu_ops_queue[0: npu_match_point] + for op_item in unmatched_data: + match_result = self.put_unmatched_in_table(match_result, op_item) + match_result = self.put_matched_in_table(match_result, npu_op_item, bench_op_item) + del npu_ops_queue[0: npu_match_point + 1] + del bench_ops_queue[0: bench_match_point + 1] + if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, self.dump_mode) - - result_df = self.make_result_table(result) - return result_df - - def merge_data(self, json_data, stack_json_data): - ops_all = {} - for op_name in json_data.get('data', {}): - merge_list = self.gen_merge_list(json_data, op_name, stack_json_data) - if merge_list: - struct_to_index_mapping = { - CompareConst.INPUT_STRUCT: 0, - CompareConst.OUTPUT_STRUCT: 0, - CompareConst.PARAMS_STRUCT: 0, - CompareConst.PARAMS_GRAD_STRUCT: 0 - } - - op_name_list = merge_list.get(CompareConst.OP_NAME) - summary_list = merge_list.get(Const.SUMMARY) - data_name_list = merge_list.get('data_name') - op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for index, op_full_name in enumerate(op_name_reorder): - data_name = data_name_reorder[index] if data_name_reorder else None - - _, state = get_name_and_state(op_full_name) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - ops_all[op_full_name] = { - CompareConst.STRUCT: safe_get_value(merge_list, struct_to_index_mapping.get(struct_key), - "merge_list", key=struct_key), - CompareConst.SUMMARY: safe_get_value(summary_reorder, index, "summary_reorder"), - 'data_name': data_name, - 'stack_info': merge_list.get('stack_info') - } - struct_to_index_mapping[struct_key] += 1 - return ops_all - - def get_accuracy(self, npu_ops_all, bench_ops_all): - result = [] - bench_ops_all[CompareConst.N_A] = self._generate_na_data(bench_ops_all) - for ms_op_name, bench_op_name in self.data_mapping_dict.items(): - check_op_str_pattern_valid(ms_op_name) - check_op_str_pattern_valid(bench_op_name) - if ms_op_name in npu_ops_all and bench_op_name in bench_ops_all: - npu_stack_info = npu_ops_all.get(ms_op_name).get("stack_info", None) - bench_stack_info = bench_ops_all.get(bench_op_name).get("stack_info", None) - has_stack = npu_stack_info and bench_stack_info - if self.dump_mode == Const.MD5: - result.append(self.get_result_md5_compare(ms_op_name, bench_op_name, npu_ops_all, - bench_ops_all, has_stack, npu_stack_info)) - continue - - npu_struct = npu_ops_all.get(ms_op_name).get('struct', []) - bench_struct = bench_ops_all.get(bench_op_name).get('struct', []) - - if len(npu_struct) < 2 or len(bench_struct) < 2: - logger.error( - f"The length of npu_struct and bench_struct must be >= 2, " - f"but got npu_struct={len(npu_struct)} and bench_struct={len(bench_struct)}. " - f"Please check!" - ) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - base_result_item = [ - ms_op_name, bench_op_name, - npu_struct[0], - bench_struct[0], - npu_struct[1], - bench_struct[1] - ] - - if self.dump_mode == Const.SUMMARY: - result_item = base_result_item + [" "] * 8 # 8个统计量数据情况的比对指标 - else: - result_item = base_result_item + [" "] * 6 # 6个真实数据情况的比对指标 - - npu_summary_data = npu_ops_all.get(ms_op_name).get("summary") - result_item.extend(npu_summary_data) - bench_summary_data = bench_ops_all.get(bench_op_name).get("summary") - result_item.extend(bench_summary_data) - if self.dump_mode == Const.SUMMARY: - self.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - else: - result_item.append(CompareConst.ACCURACY_CHECK_YES) - result_item.append("") - if has_stack: - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if self.dump_mode == Const.ALL: - ms_data_name = npu_ops_all.get(ms_op_name).get("data_name", None) - pt_data_name = bench_ops_all.get(bench_op_name).get("data_name", None) - result_item.append([ms_data_name, pt_data_name]) - result.append(result_item) - logger.info(f"{ms_op_name}, {bench_op_name} compared.") - elif ms_op_name not in npu_ops_all: - logger.warning(f'Can not find npu op name : `{ms_op_name}` in npu dump json file.') - elif bench_op_name not in npu_ops_all: - logger.warning(f'Can not find bench op name : `{bench_op_name}` in bench dump json file.') - return result + for op_item in npu_ops_queue: + match_result = self.put_unmatched_in_table(match_result, op_item) - def compare_process_custom(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None - npu_ops_all = self.merge_data(npu_json_data, stack_json_data) - bench_ops_all = self.merge_data(bench_json_data, stack_json_data) + match_result.reset_index(drop=True, inplace=True) + return match_result - result = self.get_accuracy(npu_ops_all, bench_ops_all) - result_df = self.make_result_table(result) - return result_df + def match_op(self, npu_queue, bench_queue): + for b_index, b_op in enumerate(bench_queue[0: -1]): + if self.check_op_item(npu_queue[-1], b_op): + return len(npu_queue) - 1, b_index + if self.check_op_item(npu_queue[-1], bench_queue[-1]): + return len(npu_queue) - 1, len(bench_queue) - 1 + for n_index, n_op in enumerate(npu_queue[0: -1]): + if self.check_op_item(n_op, bench_queue[-1]): + return n_index, len(bench_queue) - 1 + return -1, -1 - def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): + def gen_dtype_condition(self, match_result): """ - :param npu_op_name: excel中的NPU_Name，例如：MintFunctional.conv2d.0.forward.input.3.0 - :param bench_op_name: excel中的Bench_Name，例如：Functional.conv2d.0.forward.input.3.0 - :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 - :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 - :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 - 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离 - 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 + dtype匹配条件为npu、bench的dtype一致或属于规定的映射关系 """ - error_file, relative_err, error_flag = None, None, False + # 如果使用了data_mapping，不校验dtype，返回全True的DataFrame + if self.mapping_config.data_mapping: + return pd.Series(True, index=match_result.index) + + npu_dtype = match_result['dtype_x'] + bench_dtype = match_result['dtype_y'] + npu_dtype = self.process_cross_frame_dtype(npu_dtype) + bench_dtype = self.process_cross_frame_dtype(bench_dtype) + + equal_condition = npu_dtype == bench_dtype + match_condition = ( + (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[0]) & bench_dtype.isin( + CompareConst.DTYPE_MATCH_GROUPS[0])) | + (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[1]) & bench_dtype.isin( + CompareConst.DTYPE_MATCH_GROUPS[1])) + ) + return equal_condition | match_condition - data_name_pair = op_name_mapping_dict.get(npu_op_name) - npu_data_name = data_name_pair[0] - bench_data_name = data_name_pair[1] + def process_cross_frame_dtype(self, dtype): + if self.cross_frame: + dtype = dtype.map(cross_dtype_mapping).fillna(dtype) + return dtype - if str(npu_data_name) == '-1': # 没有npu真实数据 - n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - elif str(bench_data_name) == '-1': # 没有bench真实数据 - n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - error_file = 'no_bench_data' - else: - npu_dir = input_param.get("npu_dump_data_dir") - bench_dir = input_param.get("bench_dump_data_dir") - try: - frame_name = getattr(self, "frame_name") - read_npy_data = getattr(self, "read_npy_data") - if frame_name == "MSComparator": - n_value = read_npy_data(npu_dir, npu_data_name) - if self.cross_frame: - b_value = read_npy_data(bench_dir, bench_data_name, load_pt_file=True) - else: - b_value = read_npy_data(bench_dir, bench_data_name) - else: - n_value = read_npy_data(npu_dir, npu_data_name) - b_value = read_npy_data(bench_dir, bench_data_name) - except IOError as error: - error_file = error.filename - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - except (FileCheckException, CompareException): - error_file = npu_data_name - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - - # 通过n_value, b_value同时得到错误标志和错误信息 - n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value, - error_flag=error_flag, error_file=error_file) - - result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg) - - if self.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: - err_msg += " Fuzzy matching data, the comparison accuracy may be affected." - result_list.append(err_msg) - return result_list - def compare_core(self, input_param, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_param (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - dump_mode (str): ALL, SUMMARY, MD5. - - Returns: - """ - # get kwargs or set default value - suffix = kwargs.get('suffix', '') +class CreateTable: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - if os.path.exists(file_path): - logger.warning(f"{file_path} will be deleted.") - remove_path(file_path) - highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + @staticmethod + def process_data_name(result): + result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) + return result - npu_json = input_param.get("npu_json_path") - bench_json = input_param.get("bench_json_path") - stack_json = input_param.get("stack_json_path") - if self.data_mapping: - result_df = self.compare_process_custom([npu_json, bench_json, stack_json]) - else: - result_df = self.compare_process([npu_json, bench_json, stack_json]) + @staticmethod + def set_summary(summary): + if summary == CompareConst.N_A: + return [CompareConst.N_A] * 4 # 4为统计值个数 + summary_list = [] + for i in summary: + if str(i).lower() == 'nan': + summary_list.append(CompareConst.NAN) + else: + summary_list.append(i) + return summary_list - if not result_df.values.tolist(): - logger.warning("Can`t match any op.") - return + def make_result_df(self, result): + # get header + header = CompareConst.HEAD_OF_COMPARE_MODE[self.mode_config.dump_mode][:] + if self.mode_config.stack_mode: + header.append(CompareConst.STACK) + if self.mode_config.dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) + result = self.process_data_name(result) + + # rename match_result columns + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, + 'op_name_y': CompareConst.BENCH_NAME, + 'dtype_x': CompareConst.NPU_DTYPE, + 'dtype_y': CompareConst.BENCH_DTYPE, + 'shape_x': CompareConst.NPU_SHAPE, + 'shape_y': CompareConst.BENCH_SHAPE, + 'md5_x': CompareConst.NPU_MD5, + 'md5_y': CompareConst.BENCH_MD5, + 'data_name_x': CompareConst.DATA_NAME, + 'stack_info_x': CompareConst.STACK}, inplace=True) + + # process summary data + npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] + bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, + CompareConst.BENCH_NORM] + result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist() + result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist() + + result_df = pd.DataFrame(columns=header) + for h in header: + if h in result.columns: + result_df[h] = result[h] + return result_df, header + + +class CalcStatsDiff: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - if self.dump_mode == Const.ALL: - result_df = self.do_multi_process(input_param, result_df) + @staticmethod + def type_check(val): + """ + 检查是否为数值或字符串形式的nan, 如果是返回True + """ + check_series = pd.Series(False, index=val.index) + val_str = val.astype(str) + check_series[pd.to_numeric(val_str, errors='coerce').notna() | val_str.str.lower().eq('nan')] = True + return check_series - find_compare_result_error_rows(result_df, highlight_dict, self.dump_mode) - highlight_rows_xlsx(result_df, highlight_dict, file_path) + @staticmethod + def get_number(val): + return pd.to_numeric(val.astype(str), errors='coerce') + + def calc_summary_diff(self, result_df, cond_no_bench, stats_index: str): + npu_val = result_df['NPU ' + stats_index] + bench_val = result_df['Bench ' + stats_index] + diff_name = stats_index.capitalize() + ' diff' + rel_err_name = ('norm' if stats_index == 'l2norm' else stats_index).capitalize() + 'RelativeErr' + + # npu、bench中统计量均为数字或nan + cond_num_nan = self.type_check(npu_val) & self.type_check(bench_val) + + # 如果统计量不是数字或nan，就赋值统计量差异为N/A + result_df.loc[~cond_num_nan, [diff_name, rel_err_name]] = CompareConst.N_A + cond_valid_stat = ~cond_no_bench & cond_num_nan # 有效统计条件：bench_name不是N/A，并且NPU和bench的统计量都是数字或nan + result_df.loc[cond_valid_stat, diff_name] = self.get_number(npu_val) - self.get_number(bench_val) + + cond_diff_nan = result_df[diff_name].isna() # 统计量差异是nan + cond_nan_diff = cond_valid_stat & cond_diff_nan + result_df.loc[cond_nan_diff, [diff_name, rel_err_name]] = CompareConst.NAN + + cond_not_nan_diff = cond_valid_stat & ~cond_diff_nan + condition_pt_zero = bench_val == 0 + result_df.loc[cond_not_nan_diff & condition_pt_zero, rel_err_name] = CompareConst.N_A + + # 相对误差转成百分比字符串 + cond_ref_err = cond_not_nan_diff & ~condition_pt_zero + result_df.loc[cond_ref_err, rel_err_name] = ( + result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err] * 100) + result_df.loc[cond_ref_err, rel_err_name] = (result_df.loc[cond_ref_err, rel_err_name].abs().astype(str) + '%') + + magnitude = self.get_number(result_df[diff_name]).abs() / (pd.Series( + np.maximum(self.get_number(npu_val), self.get_number(bench_val))).abs() + CompareConst.EPSILON) + return magnitude > CompareConst.MAGNITUDE + + def calc_accuracy(self, result_df, header): + # bench name N/A represents no bench data, err_msg adds "No bench data matched." + condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A + result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) + result_df.loc[condition_no_bench, CompareConst.ERROR_MESSAGE] = CompareConst.NO_BENCH + + if self.mode_config.dump_mode == Const.MD5: + condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5] + result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS + result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF + elif self.mode_config.dump_mode == Const.SUMMARY: + warning_list = [ + self.calc_summary_diff(result_df, condition_no_bench, stats_index) + for stats_index in ['max', 'min', 'mean', 'l2norm'] + ] + warning_flag = pd.DataFrame(warning_list).any() + result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' + result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING + result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.' + else: + fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST, + CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, + CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, + CompareConst.ERROR_MESSAGE] + result_df.loc[~condition_no_bench, fill_cols] = '' + result_df.loc[~condition_no_bench, CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES + + return result_df[header] + + +def setup_comparison(input_param, output_path, **kwargs) -> ComparisonConfig: + """公共的前置处理逻辑，返回封装后的 ComparisonConfig 对象""" + try: + config = ComparisonConfig( + dump_mode='', + stack_mode=False, + auto_analyze=kwargs.get('auto_analyze', True), + fuzzy_match=kwargs.get('fuzzy_match', False), + data_mapping=kwargs.get('data_mapping', {}), + suffix=kwargs.get('suffix', ''), + cell_mapping=kwargs.get('cell_mapping', {}), + api_mapping=kwargs.get('api_mapping', {}), + layer_mapping=kwargs.get('layer_mapping', {}), + ) - if self.auto_analyze: - advisor = Advisor(result_df, output_path, suffix) - advisor.analysis() + set_dump_path(input_param) + config.dump_mode = get_dump_mode(input_param) - print_compare_ends_info() + # set stack_mode and set "stack_json_path" in input_param + if 'stack_json_path' in input_param: + config.stack_mode = kwargs.get('stack_mode', False) + else: + config.stack_mode = set_stack_json_path(input_param) - def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): - cos_result = [] - euc_dist_result = [] - max_err_result = [] - max_relative_err_result = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - err_mess = [] - - is_print_compare_log = input_param.get("is_print_compare_log") - - for i in range(len(result_df)): - npu_op_name = result_df.iloc[i, 0] - bench_op_name = result_df.iloc[i, 1] - if is_print_compare_log: - logger.info("start compare: {}".format(npu_op_name)) - - cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg \ - = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param) - - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, \ - one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, - err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - euc_dist_result.append(euc_dist) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - err_mess.append(err_msg) - - cr = ComparisonResult( - cos_result=cos_result, - euc_dist_result=euc_dist_result, - max_err_result=max_err_result, - max_relative_err_result=max_relative_err_result, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result, - err_msgs=err_mess - ) + check_configuration_param(config.stack_mode, config.auto_analyze, config.fuzzy_match, + input_param.get('is_print_compare_log', True)) + create_directory(output_path) + check_compare_param(input_param, output_path, config.dump_mode, config.stack_mode) - return _save_cmp_result(idx, cr, result_df, lock) + return config - def do_multi_process(self, input_param, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_param, result_df, - multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + raise CompareException(error.code) from error diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index 9429d7ffa1a3c1feffb0bc68f5cde777e5f8d460..a88ddb8f5e088a9f72ef2d2b721b03dbc539c385 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -14,113 +14,46 @@ # limitations under the License. from msprobe.core.common.log import logger -from msprobe.core.compare.utils import rename_api from msprobe.core.common.utils import check_op_str_pattern_valid, CompareException -from msprobe.core.common.const import CompareConst, Const - -dtype_mapping = { - "Int8": "torch.int8", - "UInt8": "torch.uint8", - "Int16": "torch.int16", - "UInt16": "torch.uint16", - "Int32": "torch.int32", - "UInt32": "torch.uint32", - "Int64": "torch.int64", - "UInt64": "torch.uint64", - "Float16": "torch.float16", - "Float32": "torch.float32", - "Float64": "torch.float64", - "Bool": "torch.bool", - "BFloat16": "torch.bfloat16", - "Complex64": "torch.complex64", - "Complex128": "torch.complex128" +from msprobe.core.common.const import Const + +cross_dtype_mapping = { + "Int8": "int", + "torch.int8": "int", + "UInt8": "int", + "torch.uint8": "int", + "Int16": "int", + "torch.int16": "int", + "UInt16": "int", + "torch.uint16": "int", + "Int32": "int", + "torch.int32": "int", + "UInt32": "int", + "torch.uint32": "int", + "Int64": "int", + "torch.int64": "int", + "UInt64": "int", + "torch.uint64": "int", + + "Float16": "float", + "torch.float16": "float", + "Float32": "float", + "torch.float32": "float", + "Float64": "float", + "torch.float64": "float", + "BFloat16": "float", + "torch.bfloat16": "float", + + "Bool": "bool", + "torch.bool": "bool", + + "Complex64": "complex", + "torch.complex64": "complex", + "Complex128": "complex", + "torch.complex128": "complex", } -def compare_op_dict_struct(npu_dict, bench_dict): - return all(npu_dict.get(key) == bench_dict.get(key) for key in CompareConst.STRUCT_COMPARE_KEY) - - -def check_struct_match(npu_dict, bench_dict): - is_match = compare_op_dict_struct(npu_dict, bench_dict) - if not is_match: - struct_match_list = [] - try: - for i, key in enumerate(CompareConst.STRUCT_COMPARE_KEY): - # 首先额外检查input_struct是否空，input_struct不可能为空 - if i == 0 and (not npu_dict.get(key, []) or not bench_dict.get(key, [])): - return False - struct_match_list.append(check_type_shape_match(npu_dict.get(key, []), bench_dict.get(key, []))) - except CompareException as error: - err_msg = f'index out of bounds error occurs in npu or bench api, please check!\n' \ - f'npu_dict: {npu_dict}' \ - f'bench_dict: {bench_dict}' - logger.error(err_msg) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - is_match = all(struct_match_list) - return is_match - - -def check_type_shape_match(npu_struct, bench_struct): - """ - further check dtypes with a dtype mapping list when dtypes are not entirely consistent. - """ - if len(npu_struct) != len(bench_struct): - return False - if not npu_struct and not bench_struct: - return True - - struct_match = False - for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): - try: - npu_type = npu_type_shape[0] - npu_shape = npu_type_shape[1] - bench_type = bench_type_shape[0] - bench_shape = bench_type_shape[1] - except IndexError as error: - logger.error(f'length of npu_type_shape: {npu_type_shape} and bench_type_shape: {bench_type_shape} ' - f'should both be 2, please check!') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - shape_match = npu_shape == bench_shape - type_match = ((npu_type == bench_type) or - any(npu_type in group and bench_type in group for group in CompareConst.DTYPE_MATCH_GROUPS)) - struct_match = shape_match and type_match - if not struct_match: - return False - return struct_match - - -def check_graph_mode(a_op_name, b_op_name): - if Const.ATEN in a_op_name and Const.ATEN not in b_op_name: - return True - if Const.ATEN not in a_op_name and Const.ATEN in b_op_name: - return True - return False - - -def fuzzy_check_op(npu_name_list, bench_name_list): - # 先检查api里的item长度是否相等，如果不是parameters_grad, 必然有input或者output，长度不可能为0 - # 如果是parameters_grad, "parameters_grad"字段的字典不会是空字典，因此len>=1 - if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): - return False - is_match = True - for npu_name, bench_name in zip(npu_name_list, bench_name_list): - is_match = fuzzy_check_name(npu_name, bench_name) - if not is_match: - break - return is_match - - -def fuzzy_check_name(npu_name, bench_name): - if Const.FORWARD in npu_name and Const.FORWARD in bench_name: - is_match = rename_api(npu_name, Const.FORWARD) == rename_api(bench_name, Const.FORWARD) - elif Const.BACKWARD in npu_name and Const.BACKWARD in bench_name: - is_match = rename_api(npu_name, Const.BACKWARD) == rename_api(bench_name, Const.BACKWARD) - else: - is_match = npu_name == bench_name - return is_match - - def check_dump_json_str(op_data, op_name): input_list = op_data.get(Const.INPUT_ARGS, None) if op_data.get(Const.INPUT_ARGS, None) else op_data.get( Const.INPUT, None) diff --git a/debug/accuracy_tools/msprobe/core/compare/config.py b/debug/accuracy_tools/msprobe/core/compare/config.py new file mode 100644 index 0000000000000000000000000000000000000000..d50aa0ab7e869bd534ebbf22f9c93ce97e67110a --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/config.py @@ -0,0 +1,70 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.file_utils import load_yaml + + +class ModeConfig: + def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.SUMMARY): + self.stack_mode = stack_mode + self.auto_analyze = auto_analyze + self.fuzzy_match = fuzzy_match + self.dump_mode = dump_mode + + +class MappingConfig: + def __init__(self, cell_mapping=None, api_mapping=None, data_mapping=None): + self.cell_mapping = cell_mapping + self.api_mapping = api_mapping + self.data_mapping = data_mapping + + +class MappingDict: + def __init__(self, mapping_config: MappingConfig): + self.cell_mapping_dict = self.load_mapping_file(mapping_config.cell_mapping) + self.api_mapping_dict = self.load_mapping_file(mapping_config.api_mapping) + if mapping_config.api_mapping is not None: + self.ms_to_pt_mapping = self.load_internal_api() + self.data_mapping_dict = self.init_data_mapping(mapping_config.data_mapping) + + @staticmethod + def load_internal_api(): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.abspath(os.path.join(cur_path, CompareConst.INTERNAL_API_MAPPING_FILE)) + return load_yaml(yaml_path) + + @staticmethod + def load_mapping_file(mapping_file): + if isinstance(mapping_file, str): + mapping_dict = load_yaml(mapping_file) + else: + mapping_dict = {} + return mapping_dict + + def init_data_mapping(self, data_mapping): + """ + 初始化data_mapping_dict + """ + if isinstance(data_mapping, str) or data_mapping is None: + data_mapping_dict = self.load_mapping_file(data_mapping) + elif isinstance(data_mapping, dict): + data_mapping_dict = data_mapping + else: + raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " + f"{type(data_mapping)}") + return data_mapping_dict \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 1983313249f34680a8f25c3a2466d8871fe0a693..a5867a9da8ee05ff8292c1d63c6e1e5885521910 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -30,12 +30,7 @@ from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger from msprobe.core.common.utils import get_header_index, safe_get_value from msprobe.core.compare.utils import table_value_is_valid, get_name_and_state, CompareException - - -class HighlightCheck(abc.ABC): - @abc.abstractmethod - def apply(self, info, color_columns, dump_mode): - raise NotImplementedError +from msprobe.core.compare.config import ModeConfig def add_highlight_row_info(color_list, num, highlight_err_msg): @@ -46,6 +41,12 @@ def add_highlight_row_info(color_list, num, highlight_err_msg): color_list.append((num, [highlight_err_msg])) +class HighlightCheck(abc.ABC): + @abc.abstractmethod + def apply(self, info, color_columns, dump_mode): + raise NotImplementedError + + class CheckOrderMagnitude(HighlightCheck): """检查Max diff的数量级差异""" @@ -75,12 +76,12 @@ class CheckOneThousandErrorRatio(HighlightCheck): if (api_in[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_IN_RED and api_out[one_thousand_index] < CompareConst.ONE_THOUSAND_ERROR_OUT_RED): add_highlight_row_info(color_columns.red, num, - "The input/parameters's one thousandth err ratio exceeds 0.9, " + "The input/parameter's one thousandth err ratio exceeds 0.9, " "while the output's is below 0.6") elif api_in[one_thousand_index] - api_out[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_DIFF_YELLOW: add_highlight_row_info(color_columns.yellow, num, "The output's one thousandth err ratio decreases by more than 0.1 " - "compared to the input/parameters's") + "compared to the input/parameter's") class CheckCosineSimilarity(HighlightCheck): @@ -94,7 +95,7 @@ class CheckCosineSimilarity(HighlightCheck): if api_in[cosine_index] - api_out[cosine_index] > CompareConst.COSINE_DIFF_YELLOW: add_highlight_row_info(color_columns.yellow, num, "The output's cosine decreases by more than 0.1 " - "compared to the input/parameters's") + "compared to the input/parameter's") class CheckMaxRelativeDiff(HighlightCheck): @@ -117,7 +118,7 @@ class CheckMaxRelativeDiff(HighlightCheck): input_max_relative_diff < CompareConst.MAX_RELATIVE_IN_YELLOW): add_highlight_row_info(color_columns.yellow, num, "The output's maximum relative error exceeds 0.1, " - "while the input/parameters's is below 0.01") + "while the input/parameter's is below 0.01") class CheckOverflow(HighlightCheck): @@ -159,73 +160,6 @@ class HighlightRules: } -def check_indices_numeric(api_items, indices: list): - """检查指定索引处的值是否都为数字类型（int 或 float）""" - return all(isinstance(api_items[i], (float, int)) for i in indices) - - -def apply_comparison_rules(api_info, dump_mode, color_columns): - """output与input/params的比较""" - if dump_mode == Const.SUMMARY: - for rule in HighlightRules.summary_compare_rules.values(): - rule.apply(api_info, color_columns, dump_mode) - else: - for rule in HighlightRules.compare_rules.values(): - rule.apply(api_info, color_columns, dump_mode) - - -def find_error_rows(result, api_batch, highlight_dict, dump_mode): - """找到单个API中需要高亮的行""" - if dump_mode == Const.MD5: - return - npu_max_index = get_header_index(CompareConst.NPU_MAX, dump_mode) - bench_max_index = get_header_index(CompareConst.BENCH_MAX, dump_mode) - max_diff_index = get_header_index(CompareConst.MAX_DIFF if dump_mode == Const.SUMMARY - else CompareConst.MAX_ABS_ERR, dump_mode) - - red_lines, yellow_lines = [], [] - LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) - ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) - ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) - color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) - - api_batch_start = api_batch.start # result_df的input起始全局索引 - api_batch_params_end_index = api_batch.params_end_index # result_df的params结束全局索引 + 1 - api_batch_output_end_index = api_batch.output_end_index # result_df的output结束全局索引 + 1 - api_batch_params_slice_index_local = api_batch_params_end_index - api_batch_start # result的params结束局部切片索引 - api_batch_output_slice_index_local = api_batch_output_end_index - api_batch_start # result的output结束局部切片索引 - - # 对单行API的输入或输出进行误差判断 - for i, line in enumerate(result): - index = api_batch_start + i - line_info = LineInfo(line_data=line, num_pointer=index) - for rule in HighlightRules.basic_rules.values(): - rule.apply(line_info, color_columns, dump_mode) - - # 对API的输出与输入比较，进行误差判断 - for n, api_out in enumerate(result[api_batch_params_slice_index_local: api_batch_output_slice_index_local]): - index = api_batch_start + api_batch_params_slice_index_local + n - # 单行检查只有溢出检查（红色），如果已经溢出，不进一步检查 - if index in red_lines: - continue - if not check_indices_numeric(api_out, [npu_max_index, bench_max_index, max_diff_index]): - continue - - # input/parameters的比较检查, 这里api_in包括input、parameters - for _, api_in in enumerate(result[0: api_batch_params_slice_index_local]): - if not check_indices_numeric(api_in, [npu_max_index, bench_max_index, max_diff_index]): - continue - api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=index) - apply_comparison_rules(api_info, dump_mode, color_columns) - - red_lines_num_set = {x[0] for x in red_lines} - yellow_lines_num_set = {x[0] for x in yellow_lines} - highlight_dict.get('red_rows', set()).update(red_lines_num_set) - highlight_dict.get('yellow_rows', set()).update(yellow_lines_num_set - red_lines_num_set) - highlight_dict.get('red_lines', []).extend(red_lines) - highlight_dict.get('yellow_lines', []).extend(yellow_lines) - - class ApiBatch: def __init__(self, api_name: str, start: int): self.api_name = api_name @@ -259,159 +193,225 @@ class ApiBatch: self.params_grad_end_index += 1 -def api_batches_update(api_batches, api_name, state, index): - """ - 当一个api的所有item更新完后，input, output的索引范围： - input: [start: start+input_len] - output: [start+input_len: output_end_index] - params: [output_end_index: params_end_index] - """ - if not api_batches: - api_batches.append(ApiBatch(api_name, index)) - else: - api_batch = api_batches[-1] - if api_batch.api_name == api_name or ( - not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): - try: - api_batch.increment(state) - except ValueError as e: - logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") - raise CompareException(CompareException.INVALID_STATE_ERROR) from e - else: - api_batches.append(ApiBatch(api_name, index)) +class HighLight: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - -def find_compare_result_error_rows(result_df, highlight_dict, dump_mode): - """将dataframe根据API分组，并找到有误差的算子用于高亮""" - result = result_df.values - api_batches = [] - for i, res_i in enumerate(result): - api_full_name = safe_get_value(res_i, 0, "res_i") - api_name, state = get_name_and_state(api_full_name) - api_batches_update(api_batches, api_name, state, i) - with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: - for api_batch in api_batches: - find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, highlight_dict, - dump_mode) - progress_bar.update(1) - - -def value_check(value, api_name=None, i=None, result_df_columns=None): - if not table_value_is_valid(value): - if result_df_columns: - logger.error(f"Malicious value [{value}] at api_name [{api_name}], column [{result_df_columns[i]}], " - f"is not allowed to be written into the compare result xlsx.") + @staticmethod + def api_batches_update(api_batches, api_name, state, index): + """ + 当一个api的所有item更新完后，input, output的索引范围： + input: [start: start+input_len] + output: [start+input_len: output_end_index] + params: [output_end_index: params_end_index] + """ + if not api_batches: + api_batches.append(ApiBatch(api_name, index)) else: - logger.error(f"Malicious value [{value}] is not allowed to be written into the compare result xlsx.") - - -def df_malicious_value_check(df_chunk, result_df_columns): - for row in df_chunk.itertuples(index=False): - api_name = row[0] - for i, value in enumerate(row): - value_check(value, api_name, i, result_df_columns) - - -def handle_multi_process_malicious_value_check(func, result_df): - result_total_nums = len(result_df) - process_num = int((multiprocessing.cpu_count() + 1) / 2) - - if result_total_nums <= process_num: - process_num = 1 - chunks = [result_df] - else: - chunk_size = result_total_nums // process_num - chunks = [result_df.iloc[i: i + chunk_size] for i in range(0, result_total_nums, chunk_size)] - - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error("Multiprocessing malicious value check failed! Reason: {}".format(args)) - try: - pool.terminate() - except OSError: - logger.error("Pool terminate failed") - - result_df_columns = result_df.columns.tolist() - for column in result_df_columns: - value_check(column) - for df_chunk in chunks: - pool.apply_async(func, args=(df_chunk, result_df_columns,), error_callback=err_call) - - pool.close() - pool.join() - - -def compare_result_df_convert(value): - if not isinstance(value, (float, int)) or isinstance(value, bool): # bool类型或者非数字类型转str - value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else str(value) - if isinstance(value, float): - value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else value - return value - - -def highlight_rows_xlsx(result_df, highlight_dict, file_path): - """Write and highlight results in Excel""" + api_batch = api_batches[-1] + if api_batch.api_name == api_name or ( + not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): + try: + api_batch.increment(state) + except ValueError as e: + logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") + raise CompareException(CompareException.INVALID_STATE_ERROR) from e + else: + api_batches.append(ApiBatch(api_name, index)) + + @staticmethod + def check_indices_numeric(api_items, indices: list): + """检查指定索引处的值是否都为数字类型（int 或 float）""" + return all(isinstance(api_items[i], (float, int)) for i in indices) + + @staticmethod + def update_highlight_err_msg(result_df, highlight_dict): + if result_df.shape[1] <= 1: + return - update_highlight_err_msg(result_df, highlight_dict) # add highlight err_msg + if CompareConst.NPU_MD5 in result_df.columns: + return - wb = openpyxl.Workbook() - ws = wb.active + err_msg = result_df.get(CompareConst.ERROR_MESSAGE) + red_lines_num_set = highlight_dict.get('red_rows') + + for color in ['red', 'yellow']: + line_key = f'{color}_lines' + lines = highlight_dict.get(line_key, []) + for line_index, messages in lines: + if color == 'yellow' and line_index in red_lines_num_set: + continue # 如果是 yellow 行，且已被 red 行覆盖，跳过 + + for msg in messages: + if err_msg[line_index] == '': + err_msg[line_index] = msg + else: + err_msg[line_index] += '\n' + msg + + if color == 'red': + red_lines_num_set.add(line_index) + + result_df[CompareConst.ERROR_MESSAGE] = err_msg + + @staticmethod + def compare_result_df_convert(value): + if not isinstance(value, (float, int)) or isinstance(value, bool): # bool类型或者非数字类型转str + value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else str(value) + if isinstance(value, float): + value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else value + return value + + @staticmethod + def value_check(value, api_name=None, i=None, result_df_columns=None): + if not table_value_is_valid(value): + if result_df_columns: + logger.error(f"Malicious value [{value}] at api_name [{api_name}], column [{result_df_columns[i]}], " + f"is not allowed to be written into the compare result xlsx.") + else: + logger.error(f"Malicious value [{value}] is not allowed to be written into the compare result xlsx.") + + def find_compare_result_error_rows(self, result_df, highlight_dict): + """将dataframe根据API分组，并找到有误差的算子用于高亮""" + result = result_df.values + api_batches = [] + for i, res_i in enumerate(result): + api_full_name = safe_get_value(res_i, 0, "res_i") + api_name, state = get_name_and_state(api_full_name) + self.api_batches_update(api_batches, api_name, state, i) + with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: + for api_batch in api_batches: + self.find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, + highlight_dict) + progress_bar.update(1) + + def find_error_rows(self, result, api_batch, highlight_dict): + """找到单个API中需要高亮的行""" + if self.mode_config.dump_mode == Const.MD5: + return + npu_max_index = get_header_index(CompareConst.NPU_MAX, self.mode_config.dump_mode) + bench_max_index = get_header_index(CompareConst.BENCH_MAX, self.mode_config.dump_mode) + max_diff_index = get_header_index(CompareConst.MAX_DIFF if self.mode_config.dump_mode == Const.SUMMARY + else CompareConst.MAX_ABS_ERR, self.mode_config.dump_mode) + + red_lines, yellow_lines = [], [] + LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) + ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) + ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) + color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) + + api_batch_start = api_batch.start # result_df的input起始全局索引 + api_batch_params_end_index = api_batch.params_end_index # result_df的params结束全局索引 + 1 + api_batch_output_end_index = api_batch.output_end_index # result_df的output结束全局索引 + 1 + api_batch_params_slice_index_local = api_batch_params_end_index - api_batch_start # result的params结束局部切片索引 + api_batch_output_slice_index_local = api_batch_output_end_index - api_batch_start # result的output结束局部切片索引 + + # 对单行API的输入或输出进行误差判断 + for i, line in enumerate(result): + index = api_batch_start + i + line_info = LineInfo(line_data=line, num_pointer=index) + for rule in HighlightRules.basic_rules.values(): + rule.apply(line_info, color_columns, self.mode_config.dump_mode) + + # 对API的输出与输入比较，进行误差判断 + for n, api_out in enumerate(result[api_batch_params_slice_index_local: api_batch_output_slice_index_local]): + index = api_batch_start + api_batch_params_slice_index_local + n + # 单行检查只有溢出检查（红色），如果已经溢出，不进一步检查 + if index in red_lines: + continue + if not self.check_indices_numeric(api_out, [npu_max_index, bench_max_index, max_diff_index]): + continue - # write header - logger.info('Initializing Excel file.') + # input/parameters的比较检查, 这里api_in包括input、parameters + for api_in in result[0: api_batch_params_slice_index_local]: + if not self.check_indices_numeric(api_in, [npu_max_index, bench_max_index, max_diff_index]): + continue + api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=index) + self.apply_comparison_rules(api_info, color_columns) + + red_lines_num_set = {x[0] for x in red_lines} + yellow_lines_num_set = {x[0] for x in yellow_lines} + highlight_dict.get('red_rows', set()).update(red_lines_num_set) + highlight_dict.get('yellow_rows', set()).update(yellow_lines_num_set - red_lines_num_set) + highlight_dict.get('red_lines', []).extend(red_lines) + highlight_dict.get('yellow_lines', []).extend(yellow_lines) + + def apply_comparison_rules(self, api_info, color_columns): + """output与input/params的比较""" + if self.mode_config.dump_mode == Const.SUMMARY: + for rule in HighlightRules.summary_compare_rules.values(): + rule.apply(api_info, color_columns, self.mode_config.dump_mode) + else: + for rule in HighlightRules.compare_rules.values(): + rule.apply(api_info, color_columns, self.mode_config.dump_mode) - handle_multi_process_malicious_value_check(df_malicious_value_check, result_df) + def highlight_rows_xlsx(self, result_df, highlight_dict, file_path): + """Write and highlight results in Excel""" - result_df_convert = result_df.applymap(compare_result_df_convert) + self.update_highlight_err_msg(result_df, highlight_dict) # add highlight err_msg - for row in dataframe_to_rows(result_df_convert, index=False, header=True): - ws.append(row) + wb = openpyxl.Workbook() + ws = wb.active - # 对可疑数据标色 - logger.info('Coloring Excel in progress.') - col_len = len(result_df.columns) - red_fill = PatternFill( - start_color=CompareConst.RED, end_color=CompareConst.RED, fill_type="solid" - ) - yellow_fill = PatternFill( - start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid", - ) - for i in highlight_dict.get("red_rows", []): - for j in range(1, col_len + 1): - ws.cell(row=i + 2, column=j).fill = red_fill # 2因为ws.cell中的row或column需要>=1,数据从第2行开始 - for i in highlight_dict.get("yellow_rows", []): - for j in range(1, col_len + 1): - ws.cell(row=i + 2, column=j).fill = yellow_fill + # write header + logger.info('Initializing Excel file.') - logger.info('Saving Excel file to disk: %s' % file_path) - save_workbook(wb, file_path) + self.handle_multi_process_malicious_value_check(self.df_malicious_value_check, result_df) + result_df_convert = result_df.applymap(self.compare_result_df_convert) -def update_highlight_err_msg(result_df, highlight_dict): - if result_df.shape[1] <= 1: - return + for row in dataframe_to_rows(result_df_convert, index=False, header=True): + ws.append(row) - if CompareConst.NPU_MD5 in result_df.columns: - return + # 对可疑数据标色 + logger.info('Coloring Excel in progress.') + col_len = len(result_df.columns) + red_fill = PatternFill( + start_color=CompareConst.RED, end_color=CompareConst.RED, fill_type="solid" + ) + yellow_fill = PatternFill( + start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid", + ) + for i in highlight_dict.get("red_rows", []): + for j in range(1, col_len + 1): + ws.cell(row=i + 2, column=j).fill = red_fill # 2因为ws.cell中的row或column需要>=1,数据从第2行开始 + for i in highlight_dict.get("yellow_rows", []): + for j in range(1, col_len + 1): + ws.cell(row=i + 2, column=j).fill = yellow_fill - err_msg = result_df.get(CompareConst.ERROR_MESSAGE) - red_lines_num_set = highlight_dict.get('red_rows') + logger.info('Saving Excel file to disk: %s' % file_path) + save_workbook(wb, file_path) - for color in ['red', 'yellow']: - line_key = f'{color}_lines' - lines = highlight_dict.get(line_key, []) - for line_index, messages in lines: - if color == 'yellow' and line_index in red_lines_num_set: - continue # 如果是 yellow 行，且已被 red 行覆盖，跳过 + def handle_multi_process_malicious_value_check(self, func, result_df): + result_total_nums = len(result_df) + process_num = int((multiprocessing.cpu_count() + 1) / 2) - for msg in messages: - if err_msg[line_index] == '': - err_msg[line_index] = msg - else: - err_msg[line_index] += '\n' + msg + if result_total_nums <= process_num: + process_num = 1 + chunks = [result_df] + else: + chunk_size = result_total_nums // process_num + chunks = [result_df.iloc[i: i + chunk_size] for i in range(0, result_total_nums, chunk_size)] - if color == 'red': - red_lines_num_set.add(line_index) + pool = multiprocessing.Pool(process_num) - result_df[CompareConst.ERROR_MESSAGE] = err_msg + def err_call(args): + logger.error("Multiprocessing malicious value check failed! Reason: {}".format(args)) + try: + pool.terminate() + except OSError: + logger.error("Pool terminate failed") + + result_df_columns = result_df.columns.tolist() + for column in result_df_columns: + self.value_check(column) + for df_chunk in chunks: + pool.apply_async(func, args=(df_chunk, result_df_columns,), error_callback=err_call) + + pool.close() + pool.join() + + def df_malicious_value_check(self, df_chunk, result_df_columns): + for row in df_chunk.itertuples(index=False): + api_name = row[0] + for i, value in enumerate(row): + self.value_check(value, api_name, i, result_df_columns) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml b/debug/accuracy_tools/msprobe/core/compare/ms_to_pt_api.yaml similarity index 100% rename from debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml rename to debug/accuracy_tools/msprobe/core/compare/ms_to_pt_api.yaml diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index 71b0f29d64f717adc87b74cf48e891652e9e753f..9e293453739f012361d0424da22ec626969287c1 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -23,48 +23,20 @@ from tqdm import tqdm from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException from msprobe.core.common.const import CompareConst +from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg +from msprobe.core.compare.config import ModeConfig -def _handle_multi_process(func, input_param, result_df, lock): - process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1) - op_name_mapping_dict = read_dump_data(result_df) - - df_chunk_size = len(result_df) // process_num - if df_chunk_size > 0: - df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] - else: - df_chunks = [result_df] - - results = [] - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error('multiprocess compare failed! Reason: {}'.format(args)) - try: - pool.terminate() - except OSError as e: - logger.error("pool terminate failed") - - progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100) - - def update_progress(size, progress_lock, extra_param=None): - with progress_lock: - progress_bar.update(size) - - for process_idx, df_chunk in enumerate(df_chunks): - idx = df_chunk_size * process_idx - chunk_size = len(df_chunk) - result = pool.apply_async(func, - args=(idx, op_name_mapping_dict, df_chunk, lock, input_param), - error_callback=err_call, - callback=partial(update_progress, chunk_size, lock) - ) - results.append(result) - - final_results = [r.get() for r in results] - pool.close() - pool.join() - return pd.concat(final_results, ignore_index=True) +@dataclass +class ComparisonResult: + cos_result: list + euc_dist_result: list + max_err_result: list + max_relative_err_result: list + one_thousand_err_ratio_result: list + five_thousand_err_ratio_result: list + err_msgs: list def _ms_graph_handle_multi_process(func, result_df, mode): @@ -83,7 +55,7 @@ def _ms_graph_handle_multi_process(func, result_df, mode): try: pool.terminate() except OSError as e: - logger.error("pool terminate failed") + logger.error(f'pool terminate failed: {str(e)}') for df_chunk in df_chunks: result = pool.apply_async(func, args=(df_chunk, mode), error_callback=err_call) @@ -94,74 +66,6 @@ def _ms_graph_handle_multi_process(func, result_df, mode): return pd.concat(final_results, ignore_index=True) -def read_dump_data(result_df): - try: - npu_dump_name_list = result_df.iloc[0:, 0].tolist() - dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() - op_name_mapping_dict = {} - for index, _ in enumerate(npu_dump_name_list): - npu_dump_name = npu_dump_name_list[index] - dump_tensor_pair = dump_tensor_pair_list[index] - op_name_mapping_dict[npu_dump_name] = dump_tensor_pair - return op_name_mapping_dict - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - -@dataclass -class ComparisonResult: - cos_result: list - euc_dist_result: list - max_err_result: list - max_relative_err_result: list - one_thousand_err_ratio_result: list - five_thousand_err_ratio_result: list - err_msgs: list - - -def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): - """ - Save comparison results into the result DataFrame with thread safety. - Args: - offset: offset for index - result: data struct of ComparisonResult - result_df: result of DataFrame - lock: thread lock - - Returns: - comparison results in DataFrame - """ - - lock.acquire() - try: - for i, _ in enumerate(result.cos_result): - process_index = i + offset - result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] - result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i] - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = ( - result.one_thousand_err_ratio_result)[i] - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = ( - result.five_thousand_err_ratio_result)[i] - result_df.loc[process_index, CompareConst.ACCURACY] = ( - check_accuracy(result.cos_result[i], result.max_err_result[i])) - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - finally: - lock.release() - - def check_accuracy(cos, max_abs_err): if cos == CompareConst.SHAPE_UNMATCH: return CompareConst.ACCURACY_CHECK_UNMATCH @@ -179,3 +83,212 @@ def check_accuracy(cos, max_abs_err): if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: return CompareConst.ACCURACY_CHECK_NO return CompareConst.ACCURACY_CHECK_YES + + +class CompareRealData: + def __init__(self, file_reader, mode_config: ModeConfig, cross_frame): + self.file_reader = file_reader + self.mode_config = mode_config + self.cross_frame = cross_frame + + @staticmethod + def read_dump_data(result_df): + try: + npu_dump_name_list = result_df.iloc[0:, 0].tolist() + dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() + op_name_mapping_dict = {} + for index, npu_dump_name in enumerate(npu_dump_name_list): + dump_tensor_pair = dump_tensor_pair_list[index] + op_name_mapping_dict[npu_dump_name] = dump_tensor_pair + return op_name_mapping_dict + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + + @staticmethod + def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): + """ + Save comparison results into the result DataFrame with thread safety. + Args: + offset: offset for index + result: data struct of ComparisonResult + result_df: result of DataFrame + lock: thread lock + + Returns: + comparison results in DataFrame + """ + + lock.acquire() + try: + for i, cos_item in enumerate(result.cos_result): + process_index = i + offset + result_df.loc[process_index, CompareConst.COSINE] = cos_item + result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i] + result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] + result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] + result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = ( + result.one_thousand_err_ratio_result)[i] + result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = ( + result.five_thousand_err_ratio_result)[i] + result_df.loc[process_index, CompareConst.ACCURACY] = ( + check_accuracy(result.cos_result[i], result.max_err_result[i])) + result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + finally: + lock.release() + + def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): + """ + :param npu_op_name: excel中的NPU_Name，例如：MintFunctional.conv2d.0.forward.input.3.0 + :param bench_op_name: excel中的Bench_Name，例如：Functional.conv2d.0.forward.input.3.0 + :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 + :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 + :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 + 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离 + 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 + """ + error_file, relative_err, error_flag = None, None, False + + data_name_pair = op_name_mapping_dict.get(npu_op_name) + npu_data_name = data_name_pair[0] + bench_data_name = data_name_pair[1] + + if str(npu_data_name) == CompareConst.NO_REAL_DATA_FLAG: # 没有npu真实数据 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + elif str(bench_data_name) == CompareConst.NO_REAL_DATA_FLAG: # 没有bench真实数据 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + error_file = 'no_bench_data' + elif str(bench_data_name) == CompareConst.N_A: # bench没匹配 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + error_file = None + else: + npu_dir = input_param.get(CompareConst.NPU_DUMP_DATA_DIR) + bench_dir = input_param.get(CompareConst.BENCH_DUMP_DATA_DIR) + try: + n_value, b_value = self.file_reader(npu_dir, npu_data_name, bench_dir, bench_data_name, + self.cross_frame) + except IOError as error: + error_file = error.filename + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + except (FileCheckException, CompareException): + error_file = npu_data_name + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + + # 通过n_value, b_value同时得到错误标志和错误信息 + n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value, + error_flag=error_flag, error_file=error_file) + + result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg) + + if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: + err_msg += " Fuzzy matching data, the comparison accuracy may be affected." + result_list.append(err_msg) + return result_list + + def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): + cos_result = [] + euc_dist_result = [] + max_err_result = [] + max_relative_err_result = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + err_mess = [] + + is_print_compare_log = input_param.get("is_print_compare_log") + + for i in range(len(result_df)): + npu_op_name = result_df.iloc[i, 0] + bench_op_name = result_df.iloc[i, 1] + if is_print_compare_log: + logger.info("start compare: {}".format(npu_op_name)) + + cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg \ + = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param) + + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, \ + one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, + err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + euc_dist_result.append(euc_dist) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + err_mess.append(err_msg) + + cr = ComparisonResult( + cos_result=cos_result, + euc_dist_result=euc_dist_result, + max_err_result=max_err_result, + max_relative_err_result=max_relative_err_result, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result, + err_msgs=err_mess + ) + + return self._save_cmp_result(idx, cr, result_df, lock) + + def do_multi_process(self, input_param, result_df): + try: + result_df = self._handle_multi_process(self.compare_ops, input_param, result_df, + multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + def _handle_multi_process(self, func, input_param, result_df, lock): + process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1) + op_name_mapping_dict = self.read_dump_data(result_df) + + df_chunk_size = len(result_df) // process_num + if df_chunk_size > 0: + df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] + else: + df_chunks = [result_df] + + results = [] + pool = multiprocessing.Pool(process_num) + + def err_call(args): + logger.error('multiprocess compare failed! Reason: {}'.format(args)) + try: + pool.terminate() + except OSError: + logger.error("pool terminate failed") + + progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100) + + def update_progress(size, progress_lock, extra_param=None): + with progress_lock: + progress_bar.update(size) + + for process_idx, df_chunk in enumerate(df_chunks): + idx = df_chunk_size * process_idx + chunk_size = len(df_chunk) + result = pool.apply_async(func, + args=(idx, op_name_mapping_dict, df_chunk, lock, input_param), + error_callback=err_call, + callback=partial(update_progress, chunk_size, lock) + ) + results.append(result) + + final_results = [r.get() for r in results] + pool.close() + pool.join() + return pd.concat(final_results, ignore_index=True) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 035e7d1cda05968834429d6a6399b7044c3573b1..951a9d8a51d32808c492c52763157485de73f353 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -20,6 +20,7 @@ import zlib from dataclasses import dataclass import numpy as np +import pandas as pd from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger, safe_get_value @@ -81,22 +82,6 @@ def check_and_return_dir_contents(dump_dir, prefix): return contents -def rename_api(npu_name, process): - """ - 原api： {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} - rename后： {api_type}.{api_name}.{input/output}.{参数序号} - """ - npu_split = npu_name.split(process) - try: - torch_func_index, in_out = npu_split[0], npu_split[1] - except IndexError as error: - logger.error(f'{npu_name} can not be split with {process}, please check!') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - torch_func_split = torch_func_index.rsplit(Const.SEP, 2) - torch_func = str(torch_func_split[0]) + str(in_out) - return torch_func - - def read_op(op_data, op_name): if Const.PARAMS_GRAD in op_name.split(Const.SEP): op_parsed_list = op_item_parse(op_data, op_name) @@ -191,35 +176,146 @@ def gen_op_item(op_data, op_name): return op_item -def resolve_api_special_parameters(data_dict, full_op_name, item_list): +@dataclass +class ApiItemInfo: + name: str + struct: tuple + stack_info: list + + +def merge_tensor(tensor_list, dump_mode): + keys = [ + CompareConst.OP_NAME, + CompareConst.INPUT_STRUCT, + CompareConst.KWARGS_STRUCT, + CompareConst.OUTPUT_STRUCT, + CompareConst.PARAMS_STRUCT, + CompareConst.PARAMS_GRAD_STRUCT, + Const.SUMMARY, + Const.STACK_INFO + ] + op_dict = {key: [] for key in keys} + + if dump_mode == Const.ALL: + op_dict["data_name"] = [] + + for tensor in tensor_list: + # A dict(len=2) with 'full_op_name' and 'full_info' is added to the tensor only if self.stack_mode is True + if len(tensor) == 2: + op_dict[Const.STACK_INFO].append(tensor['full_info']) + break + + op_dict[CompareConst.OP_NAME].append(tensor['full_op_name']) + + _, state = get_name_and_state(tensor['full_op_name']) + struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) + if not struct_key: + continue + if dump_mode == Const.MD5: + op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE], tensor[Const.MD5])) + else: + op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE])) + op_dict[Const.SUMMARY].append([tensor[Const.MAX], tensor[Const.MIN], tensor[Const.MEAN], tensor[Const.NORM]]) + + if dump_mode == Const.ALL: + op_dict["data_name"].append(tensor['data_name']) + + if not op_dict[CompareConst.KWARGS_STRUCT]: + del op_dict[CompareConst.KWARGS_STRUCT] + return op_dict if op_dict[CompareConst.OP_NAME] else {} + + +def print_compare_ends_info(): + total_len = len(CompareConst.COMPARE_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS + logger.info('*' * total_len) + logger.info(f"*{CompareConst.COMPARE_ENDS_SUCCESSFULLY.center(total_len - 2)}*") + logger.info('*' * total_len) + + +def table_value_is_valid(value: str) -> bool: + if not isinstance(value, str): + return True + try: + # -1.00 or +1.00 should be considered as digit numbers + float(value) + except ValueError: + # otherwise, they will be considered as formular injections + return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value)) + return True + + +def get_name_and_state(name): """ - Function Description: - 解析下面格式的数据, 是api参数的一种特殊格式 - { - "last_hidden_state": { - "type": "torch.Tensor", - "dtype": "torch.bfloat16", - ... - }, - "loss": { - "type": "torch.Tensor", - "dtype": "torch.float32", - ... - } - } - Parameter: - data_dict: 字典格式的数据 - full_op_name: 参数的全名字符串 - item_list: 参数信息集合 + Get api/module name and state + example: + name = 'conv2d.forward.1.input.0' + return: ('conv2d.forward.1.', 'input') + + name = 'Functional.pad.0.backward.output.0' + return: ('Functional.pad.0.backward.', 'output') + + state type: input, output, kwargs, parameters, parameters_grad """ - for key, value in data_dict.items(): - if isinstance(value, dict): - parsed_item = value - parts = full_op_name.split(Const.SEP) - parts.insert(-1, key) - full_op_name_new = ".".join(parts) - parsed_item['full_op_name'] = full_op_name_new - item_list.append(parsed_item) + if not isinstance(name, str): + logger.error(f'Invalid name: {name}, type should be string, please check.') + raise CompareException(CompareException.INVALID_API_NAME_ERROR) + + if Const.PARAMS_GRAD in name.split(Const.SEP): + return name.split(Const.PARAMS_GRAD)[0], Const.PARAMS_GRAD + + split = re.split(Const.REGEX_FORWARD_BACKWARD, name) + if len(split) < 3: + logger.error(f'Invalid name string: {name}, can not be split by forward/backward, please check.') + raise CompareException(CompareException.INVALID_API_NAME_ERROR) + api = f'{split[0]}.{split[1]}.' + state_str = split[2] + match = re.match(r'^(\d+\.)?(input|output|kwargs|parameters)\..+$', state_str) + if not match: + raise CompareException(f'Invalid name string: {name}') + if match.group(1): + api = f'{api}{match.group(1)}' + state = match.group(2) + return api, state + + +def reorder_op_name_list(op_name_list): + if not op_name_list: + return op_name_list + + parameters = [] + output = [] + parameters_grad = [] + others = [] + for x in op_name_list: + state = get_name_and_state(x)[1] + if state == Const.PARAMS: + parameters.append(x) + elif state == Const.OUTPUT: + output.append(x) + elif state == Const.PARAMS_GRAD: + parameters_grad.append(x) + else: + others.append(x) + # 合并others, parameters, 和output，确保parameters排在output前面 + op_name_reorder = others + parameters + output + parameters_grad + return op_name_reorder + + +def reorder_op_x_list(op_name_list, summary_list, data_name_list): + """对op_name, summary, data_name重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理""" + if not op_name_list or not summary_list: + return op_name_list, summary_list, data_name_list + + index_map = {name: index for index, name in enumerate(op_name_list)} + + op_name_reorder = reorder_op_name_list(op_name_list) + summary_reorder = [summary_list[index_map.get(name)] for name in op_name_reorder] + if data_name_list: + data_name_reorder = [data_name_list[index_map.get(name)] for name in op_name_reorder] + else: + data_name_reorder = data_name_list + + return op_name_reorder, summary_reorder, data_name_reorder def process_summary_data(summary_data): @@ -407,204 +503,23 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): CompareConst.PARAMS_GRAD_STRUCT) -def append_stack_info(result_item, npu_stack_info, index): - """添加堆栈信息到 result_item""" - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - - -def get_un_match_accuracy(result, n_dict, dump_mode): - npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.N_A, CompareConst.N_A, CompareConst.N_A +def make_result_table(result, dump_mode, stack_mode): + header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode][:] - struct_to_index_mapping = { - CompareConst.INPUT_STRUCT: 0, - CompareConst.OUTPUT_STRUCT: 0, - CompareConst.PARAMS_STRUCT: 0, - CompareConst.PARAMS_GRAD_STRUCT: 0 - } - - op_name_list = n_dict.get(CompareConst.OP_NAME) - summary_list = n_dict.get(Const.SUMMARY) - data_name_list = n_dict.get('data_name') - op_name_reorder, summary_reorder, _ = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for index, n_name in enumerate(op_name_reorder): - _, state = get_name_and_state(n_name) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - n_struct = safe_get_value(n_dict, struct_to_index_mapping.get(struct_key), "n_dict", key=struct_key) - struct_to_index_mapping[struct_key] += 1 - - try: - result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] - except IndexError as e: - err_msg = "index out of bounds error occurs, please check!\n" \ - f"op_name of n_dict is {n_dict['op_name']}\n" \ - f"input_struct of n_dict is {n_dict[CompareConst.INPUT_STRUCT]}\n" \ - f"output_struct of n_dict is {n_dict[CompareConst.OUTPUT_STRUCT]}" - logger.error(err_msg) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - if dump_mode == Const.MD5: - result_item.extend([CompareConst.N_A] * 3) - append_stack_info(result_item, npu_stack_info, index) - result.append(result_item) - continue - if dump_mode == Const.SUMMARY: - result_item.extend([CompareConst.N_A] * 8) # 8个统计量数据情况的比对指标 + if stack_mode: + header.append(CompareConst.STACK) if dump_mode == Const.ALL: - result_item.extend([CompareConst.N_A] * 6) # 6个真实数据情况的比对指标 - - npu_summary_data = safe_get_value(summary_reorder, index, "summary_reorder") - bench_summary_data = [CompareConst.N_A] * 4 - result_item.extend(npu_summary_data) - result_item.extend(bench_summary_data) - err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.N_A - result_item.append(accuracy_check_res) - result_item.append(err_msg) - append_stack_info(result_item, npu_stack_info, index) - if dump_mode == Const.ALL and result_item[1] == CompareConst.N_A: - result_item.extend([["-1", "-1"]]) - result.append(result_item) - - -def merge_tensor(tensor_list, dump_mode): - op_dict = {} - op_dict["op_name"] = [] - op_dict[CompareConst.INPUT_STRUCT] = [] - op_dict[CompareConst.KWARGS_STRUCT] = [] - op_dict[CompareConst.OUTPUT_STRUCT] = [] - op_dict[CompareConst.PARAMS_STRUCT] = [] - op_dict[CompareConst.PARAMS_GRAD_STRUCT] = [] - op_dict[Const.SUMMARY] = [] - op_dict["stack_info"] = [] - - if dump_mode == Const.ALL: - op_dict["data_name"] = [] - - for tensor in tensor_list: - # A dict(len=2) with 'full_op_name' and 'full_info' is added to the tensor only if self.stack_mode is True - if len(tensor) == 2: - op_dict['stack_info'].append(tensor['full_info']) - break - - op_dict["op_name"].append(tensor['full_op_name']) - - _, state = get_name_and_state(tensor['full_op_name']) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - if dump_mode == Const.MD5: - op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE], tensor[Const.MD5])) - else: - op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE])) - op_dict[Const.SUMMARY].append([tensor[Const.MAX], tensor[Const.MIN], tensor[Const.MEAN], tensor[Const.NORM]]) - + header.append(CompareConst.DATA_NAME) + else: if dump_mode == Const.ALL: - op_dict["data_name"].append(tensor['data_name']) - - if not op_dict[CompareConst.KWARGS_STRUCT]: - del op_dict[CompareConst.KWARGS_STRUCT] - return op_dict if op_dict["op_name"] else {} - - -def print_compare_ends_info(): - total_len = len(CompareConst.COMPARE_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS - logger.info('*' * total_len) - logger.info(f"*{CompareConst.COMPARE_ENDS_SUCCESSFULLY.center(total_len - 2)}*") - logger.info('*' * total_len) - - -def table_value_is_valid(value: str) -> bool: - if not isinstance(value, str): - return True - try: - # -1.00 or +1.00 should be considered as digit numbers - float(value) - except ValueError: - # otherwise, they will be considered as formular injections - return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value)) - return True - - -def get_name_and_state(name): - """ - Get api/module name and state - example: - name = 'conv2d.forward.1.input.0' - return: ('conv2d.forward.1.', 'input') - - name = 'Functional.pad.0.backward.output.0' - return: ('Functional.pad.0.backward.', 'output') - - state type: input, output, kwargs, parameters, parameters_grad - """ - if not isinstance(name, str): - logger.error(f'Invalid name: {name}, type should be string, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - - if Const.PARAMS_GRAD in name.split(Const.SEP): - return name.split(Const.PARAMS_GRAD)[0], Const.PARAMS_GRAD - - split = re.split(Const.REGEX_FORWARD_BACKWARD, name) - if len(split) < 3: - logger.error(f'Invalid name string: {name}, can not be split by forward/backward, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - api = f'{split[0]}.{split[1]}.' - state_str = split[2] - match = re.match(r'^(\d+\.)?(input|output|kwargs|parameters)\..+$', state_str) - if not match: - raise CompareException(f'Invalid name string: {name}') - if match.group(1): - api = f'{api}{match.group(1)}' - state = match.group(2) - return api, state - - -def reorder_op_name_list(op_name_list): - if not op_name_list: - return op_name_list - - parameters = [] - output = [] - parameters_grad = [] - others = [] - for x in op_name_list: - state = get_name_and_state(x)[1] - if state == Const.PARAMS: - parameters.append(x) - elif state == Const.OUTPUT: - output.append(x) - elif state == Const.PARAMS_GRAD: - parameters_grad.append(x) + for row in result: + del row[-2] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，真实数据时为倒数第2列 + header.append(CompareConst.DATA_NAME) else: - others.append(x) - # 合并others, parameters, 和output，确保parameters排在output前面 - op_name_reorder = others + parameters + output + parameters_grad - return op_name_reorder - - -def reorder_op_x_list(op_name_list, summary_list, data_name_list): - """对op_name, summary, data_name重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理""" - if not op_name_list or not summary_list: - return op_name_list, summary_list, data_name_list - - index_map = {name: index for index, name in enumerate(op_name_list)} - - op_name_reorder = reorder_op_name_list(op_name_list) - summary_reorder = [summary_list[index_map.get(name)] for name in op_name_reorder] - if data_name_list: - data_name_reorder = [data_name_list[index_map.get(name)] for name in op_name_reorder] - else: - data_name_reorder = data_name_list - - return op_name_reorder, summary_reorder, data_name_reorder + for row in result: + del row[-1] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，非真实数据时为倒数第1列 + result_df = pd.DataFrame(result, columns=header, dtype='object') + return result_df def _compare_parser(parser): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index c0ef3bc66c3b25a3162630cc1e0fb4a1c4783310..0c8bb42a4faef2c62b76a12dbf483d43d73c426e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -13,418 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re -from collections import defaultdict - -import numpy as np -import pandas as pd - -from msprobe.core.common.const import CompareConst, Const -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory, load_json, load_npy, load_yaml -from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, \ - check_op_str_pattern_valid, get_dump_mode, set_dump_path, detect_framework_by_dump_json -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.check import dtype_mapping +from msprobe.core.compare.acc_compare import Comparator, ModeConfig, MappingConfig, setup_comparison from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping -from msprobe.core.compare.utils import set_stack_json_path, reorder_op_x_list - - -class MappingConfig: - def __init__(self, cell_mapping=None, api_mapping=None, data_mapping=None): - self.cell_mapping = cell_mapping - self.api_mapping = api_mapping - self.data_mapping = data_mapping - - -class MSComparator(Comparator): - """ - 用于mindspore动态图同框架/跨框架精度比对，支持md5/summary/all模式。 - cell_mapping: mindspore在cell级别(L0)dump数据和pytorch的module之间的映射关系； - api_mapping: mindspore在api级别(L1)dump数据和pytorch的api之间的映射关系； - data_mapping: mindspore的cell或api的入参/出参和pytorch之间的映射关系； - is_cross_framework: 是否跨框架。 - """ - def __init__(self, mode_config, mapping_config=None, is_cross_framework=False): - super().__init__(mode_config) - self.frame_name = MSComparator.__name__ - - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode - - if mapping_config: - self.cell_mapping = mapping_config.cell_mapping - self.api_mapping = mapping_config.api_mapping - self.data_mapping = mapping_config.data_mapping - - if self.data_mapping: - self.cross_frame = is_cross_framework - else: - self.cross_frame = self.cell_mapping is not None or self.api_mapping is not None - self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) - self.api_mapping_dict = self.load_mapping_file(self.api_mapping) - if self.api_mapping is not None: - self.ms_to_pt_mapping = self.load_internal_api() - - if isinstance(self.data_mapping, str) or self.data_mapping is None: - self.data_mapping_dict = self.load_mapping_file(self.data_mapping) - elif isinstance(self.data_mapping, dict): - self.data_mapping_dict = self.data_mapping - else: - raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " - f"{type(self.data_mapping)}") - - @staticmethod - def process_data_name(result): - result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) - return result - - def calc_accuracy(self, result_df, header): - condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A - result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) - result_df.loc[condition_no_bench, CompareConst.ERROR_MESSAGE] = CompareConst.NO_BENCH - - def calc_summary_diff(data_type: str): - def type_check(val): - check_series = pd.Series(False, index=val.index) - val_str = val.astype(str) - check_series[pd.to_numeric(val_str, errors='coerce').notna() | val_str.str.lower().eq('nan')] = True - return check_series - - def get_number(val): - return pd.to_numeric(val.astype(str), errors='coerce') - - ms_val = result_df['NPU ' + data_type] - pt_val = result_df['Bench ' + data_type] - diff_name = data_type.capitalize() + ' diff' - rel_err_name = ('norm' if data_type == 'l2norm' else data_type).capitalize() + 'RelativeErr' - condition_na = ~type_check(ms_val) | ~type_check(pt_val) - result_df.loc[condition_na, [diff_name, rel_err_name]] = CompareConst.N_A - result_df.loc[~(condition_no_bench | condition_na), diff_name] = get_number(ms_val) - get_number(pt_val) - condition_nan_diff = ~condition_no_bench & ~condition_na & result_df[diff_name].isna() - condition_not_nan_diff = ~condition_no_bench & ~condition_na & result_df[diff_name].notna() - result_df.loc[condition_nan_diff, [diff_name, rel_err_name]] = CompareConst.NAN - condition_pt_zero = pt_val == 0 - result_df.loc[condition_not_nan_diff & condition_pt_zero, rel_err_name] = CompareConst.NAN - condition_ref_err = condition_not_nan_diff & ~condition_pt_zero - result_df.loc[condition_ref_err, rel_err_name] = (result_df.loc[condition_ref_err, diff_name] / - pt_val[condition_ref_err] * 100) - result_df.loc[condition_ref_err, rel_err_name] = (result_df.loc[condition_ref_err, rel_err_name] - .abs().astype(str) + '%') - magnitude = get_number(result_df[diff_name]).abs() / ( - pd.Series(np.maximum(get_number(ms_val), get_number(pt_val))).abs() + CompareConst.EPSILON) - return magnitude > CompareConst.MAGNITUDE - - if self.dump_mode == Const.MD5: - condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5] - result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS - result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF - elif self.dump_mode == Const.SUMMARY: - warning_list = [calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']] - warning_flag = pd.DataFrame(warning_list).any() - result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' - result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING - result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.' - else: - fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST, - CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, - CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, - CompareConst.ERROR_MESSAGE] - result_df.loc[~condition_no_bench, fill_cols] = '' - result_df.loc[~condition_no_bench, CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES - return result_df[header] - - def make_result_df(self, result): - header = CompareConst.HEAD_OF_COMPARE_MODE[self.dump_mode][:] - - if self.stack_mode: - header.append(CompareConst.STACK) - if self.dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) - result = self.process_data_name(result) - - result.rename(columns={'op_name_x': CompareConst.NPU_NAME, - 'op_name_y': CompareConst.BENCH_NAME, - 'dtype_x': CompareConst.NPU_DTYPE, - 'dtype_y': CompareConst.BENCH_DTYPE, - 'shape_x': CompareConst.NPU_SHAPE, - 'shape_y': CompareConst.BENCH_SHAPE, - 'md5_x': CompareConst.NPU_MD5, - 'md5_y': CompareConst.BENCH_MD5, - 'data_name_x': CompareConst.DATA_NAME, - 'stack_info_x': CompareConst.STACK}, inplace=True) - - npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] - bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, - CompareConst.BENCH_NORM] - - def set_summary(summary): - if summary == CompareConst.N_A: - return [CompareConst.N_A] * 4 - summary_list = [] - for i in summary: - if i is None: - summary_list.append(CompareConst.N_A) - elif str(i).lower() == 'nan': - summary_list.append(CompareConst.NAN) - else: - summary_list.append(i) - return summary_list - - result[npu_summary] = result['summary_x'].apply(set_summary).tolist() - result[bench_summary] = result['summary_y'].apply(set_summary).tolist() - - result_df = pd.DataFrame(columns=header) - for h in header: - if h in result.columns: - result_df[h] = result[h] - return self.calc_accuracy(result_df, header) - - def load_internal_api(self): - cur_path = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.abspath(os.path.join(cur_path, CompareConst.INTERNAL_API_MAPPING_FILE)) - return load_yaml(yaml_path) - - def load_mapping_file(self, mapping_file): - if isinstance(mapping_file, str): - mapping_dict = load_yaml(mapping_file) - else: - mapping_dict = {} - return mapping_dict - - def process_cell_mapping(self, npu_op_name): - if not npu_op_name: - return CompareConst.N_A - param_grad_flag = Const.PARAMS_GRAD in npu_op_name.split(Const.SEP) - if not param_grad_flag and not re.search(Const.REGEX_FORWARD_BACKWARD, npu_op_name): - return CompareConst.N_A - npu_op_name = npu_op_name.replace("Cell", "Module", 1) - if self.cell_mapping_dict: - # get cell name & class name from op_name - # Cell.fc1.Dense.forward.0.input.0 - cell_name = re.split(r'\.(?:forward|backward|parameters_grad)\.', npu_op_name.split(Const.SEP, 1)[-1])[0] - if cell_name in self.cell_mapping_dict: - npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) - return npu_op_name - - def read_npy_data(self, dir_path, file_name, load_pt_file=False): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - if load_pt_file: - import torch - from msprobe.pytorch.common.utils import load_pt - data_value = load_pt(data_path, True).detach() - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - else: - data_value = load_npy(data_path) - return data_value - - def process_internal_api_mapping(self, npu_op_name): - # get api name & class name from op_name - # Functional.addcmul.0.forward.input.0 - ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) - class_name = ms_api_name.split(Const.SEP)[0] - if class_name == "Mint": - return npu_op_name.replace("Mint", "Torch") - elif class_name == "MintFunctional": - return npu_op_name.replace("MintFunctional", "Functional") - elif self.ms_to_pt_mapping.get(ms_api_name): - return npu_op_name.replace(ms_api_name, self.ms_to_pt_mapping.get(ms_api_name)) - else: - return npu_op_name - - def get_api_name(self, api_list): - try: - api_name = api_list[0] + Const.SEP + api_list[1] - except IndexError as error: - logger.error(f'Failed to retrieve API name, please check if the dump data is reasonable') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - return api_name - - def compare_process(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None - - npu_df = self.gen_data_df(npu_json_data, stack_json_data) - bench_df = self.gen_data_df(bench_json_data, stack_json_data) - if self.cell_mapping: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_cell_mapping) - elif self.api_mapping: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_internal_api_mapping) - if isinstance(self.api_mapping, str): - self.modify_compare_data_with_user_mapping(npu_df, bench_df) - else: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME] - npu_df[[Const.DTYPE, Const.SHAPE]] = npu_df[[Const.DTYPE, Const.SHAPE]].astype(str) - bench_df[[Const.DTYPE, Const.SHAPE]] = bench_df[[Const.DTYPE, Const.SHAPE]].astype(str) - npu_df[CompareConst.COMPARE_SHAPE] = npu_df[Const.SHAPE] - bench_df[CompareConst.COMPARE_KEY] = bench_df[CompareConst.OP_NAME] - bench_df[CompareConst.COMPARE_SHAPE] = bench_df[Const.SHAPE] - match_result = pd.merge(npu_df, bench_df, on=[CompareConst.COMPARE_KEY, CompareConst.COMPARE_SHAPE], - how='outer') - match_result = match_result[match_result['op_name_x'].notna()].fillna(CompareConst.N_A) - - def gen_dtype_condition(): - npu_dtype = match_result['dtype_x'] - bench_dtype = match_result['dtype_y'] - if self.cross_frame: - npu_dtype = npu_dtype.map(dtype_mapping).fillna(npu_dtype) - - equal_condition = npu_dtype == bench_dtype - match_condition = ( - (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[0]) & bench_dtype.isin( - CompareConst.DTYPE_MATCH_GROUPS[0])) | - (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[1]) & bench_dtype.isin( - CompareConst.DTYPE_MATCH_GROUPS[1])) - ) - return equal_condition | match_condition - - match_result.loc[~gen_dtype_condition(), [i + '_y' for i in bench_df.columns]] = CompareConst.N_A - return self.make_result_df(match_result) - - def modify_compare_data_with_user_mapping(self, npu_df, bench_df): - def get_api_indices_dict(op_name_df): - api_indices_dict = defaultdict(list) - for op_index, name in enumerate(op_name_df[CompareConst.OP_NAME]): - api = self.get_api_name(name.split(Const.SEP)) - api_indices_dict[api].append(op_index) - return api_indices_dict - - ms_api_indices_dict = get_api_indices_dict(npu_df) - pt_api_indices_dict = get_api_indices_dict(bench_df) - - def gen_input_compare_key(pattern, term): - flag = True - for i, prefix in enumerate(mapping_dict.get(f'ms_{term}')): - if op_name.split(pattern)[1].startswith(str(prefix)): - npu_df.loc[index, CompareConst.COMPARE_KEY] = ( - op_name.replace(pattern + str(prefix), - pattern + str(mapping_dict.get(f'pt_{term}')[i]))) - flag = False - return flag - - for mapping_dict in self.api_mapping_dict: - keys_to_compare = [ - ('ms_args', 'pt_args'), - ('ms_outputs', 'pt_outputs'), - ('ms_parameters', 'pt_parameters'), - ('ms_parameters_grad', 'pt_parameters_grad'), - ] - if not all(len(mapping_dict.get(k1, [])) == len(mapping_dict.get(k2, [])) for k1, k2 in keys_to_compare): - logger.warning('The user-defined mapping table is incorrect,\ - make sure that the number of parameters is equal') - continue - - ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') - if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: - continue - for index in ms_api_indices_dict.get(ms_api): - op_name = npu_df.loc[index, CompareConst.OP_NAME].replace(ms_api, pt_api, 1) - if CompareConst.INPUT_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.INPUT_PATTERN, 'args') - elif CompareConst.KWARGS_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.KWARGS_PATTERN, 'args') - elif CompareConst.OUTPUT_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.OUTPUT_PATTERN, 'output') - elif CompareConst.PARAMS_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.PARAMS_PATTERN, 'parameters') - elif CompareConst.PARAMS_GRAD_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.PARAMS_GRAD_PATTERN, 'parameters_grad') - else: - logger.error(f'Excepted op_name: {op_name}') - raise CompareException(CompareException.INVALID_DATA_ERROR) - if is_abandoned: - npu_df.loc[index, CompareConst.COMPARE_KEY] = op_name + 'abandoned' - - def gen_data_df(self, data_json, stack_json_data): - result = { - CompareConst.OP_NAME: [], - Const.DTYPE: [], - Const.SHAPE: [], - Const.SUMMARY: [], - 'stack_info': [] - } - if self.dump_mode == Const.ALL: - result['data_name'] = [] - elif self.dump_mode == Const.MD5: - result[Const.MD5] = [] - for data_name in data_json['data']: - check_op_str_pattern_valid(data_name) - merge_list = self.gen_merge_list(data_json, data_name, stack_json_data) - if not merge_list: - continue - - op_name_list = merge_list.get(CompareConst.OP_NAME) - summary_list = merge_list.get(Const.SUMMARY) - data_name_list = merge_list.get('data_name') - op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for op_name in op_name_reorder: - result[CompareConst.OP_NAME].append(op_name) - if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name): - struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) - elif CompareConst.OUTPUT_PATTERN in op_name: - struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) - elif CompareConst.PARAMS_PATTERN in op_name: - struct = merge_list[CompareConst.PARAMS_STRUCT].pop(0) - else: - struct = merge_list[CompareConst.PARAMS_GRAD_STRUCT].pop(0) - result[Const.DTYPE].append(struct[0]) - result[Const.SHAPE].append(struct[1]) - if self.dump_mode == Const.MD5: - result[Const.MD5].append(struct[2]) - result[Const.SUMMARY].append(summary_reorder.pop(0)) - result['stack_info'].append(merge_list['stack_info'][0] if self.stack_mode else None) - if self.dump_mode == Const.ALL: - result['data_name'].append(data_name_reorder.pop(0)) - return pd.DataFrame(result) +from msprobe.mindspore.compare.utils import read_npy_data, check_cross_framework +from msprobe.pytorch.compare.utils import read_pt_data -def check_cross_framework(bench_json_path): - framework = detect_framework_by_dump_json(bench_json_path) - if framework == Const.PT_FRAMEWORK: - return True +def read_real_data(npu_dir, npu_data_name, bench_dir, bench_data_name, cross_frame) -> tuple: + n_value = read_npy_data(npu_dir, npu_data_name) + if cross_frame: + b_value = read_pt_data(bench_dir, bench_data_name) else: - return False + b_value = read_npy_data(bench_dir, bench_data_name) + return n_value, b_value def ms_compare(input_param, output_path, **kwargs): - try: - auto_analyze = kwargs.get('auto_analyze', True) - fuzzy_match = kwargs.get('fuzzy_match', False) - cell_mapping = kwargs.get('cell_mapping', None) - api_mapping = kwargs.get('api_mapping', None) - data_mapping = kwargs.get('data_mapping', None) - layer_mapping = kwargs.get('layer_mapping', None) - suffix = kwargs.get('suffix', '') + config = setup_comparison(input_param, output_path, **kwargs) - set_dump_path(input_param) - dump_mode = get_dump_mode(input_param) - if 'stack_json_path' in input_param: - stack_mode = kwargs.get('stack_mode', False) - else: - stack_mode = set_stack_json_path(input_param) # set stack_mode and set "stack_json_path" in input_param - check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) - create_directory(output_path) - check_compare_param(input_param, output_path, dump_mode, stack_mode) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - raise CompareException(error.code) from error - if layer_mapping: - data_mapping = generate_data_mapping_by_layer_mapping(input_param, layer_mapping, output_path) + if config.layer_mapping: + config.data_mapping = generate_data_mapping_by_layer_mapping(input_param, config.layer_mapping, output_path) - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(cell_mapping, api_mapping, data_mapping) is_cross_framework = check_cross_framework(input_param.get('bench_json_path')) - ms_comparator = MSComparator(mode_config, mapping_config, is_cross_framework) - ms_comparator.compare_core(input_param, output_path, suffix=suffix) + mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match, config.dump_mode) + mapping_config = MappingConfig(config.cell_mapping, config.api_mapping, config.data_mapping) + ms_comparator = Comparator(read_real_data, mode_config, mapping_config, is_cross_framework) + ms_comparator.compare_core(input_param, output_path, suffix=config.suffix) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/utils.py b/debug/accuracy_tools/msprobe/mindspore/compare/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7a9c78e8f74426c23982723fcf90f729fc9e694c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/utils.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.const import Const +from msprobe.core.common.file_utils import load_npy, FileChecker, FileCheckConst +from msprobe.core.common.utils import detect_framework_by_dump_json + + +def read_npy_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = load_npy(data_path) + return data_value + + +def check_cross_framework(bench_json_path): + framework = detect_framework_by_dump_json(bench_json_path) + return framework == Const.PT_FRAMEWORK diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index de62af421b5a37e39140a9836fb16853443740d7..a484ad5ceed06fd7e8ecd8c1ada7b9b7060260ab 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,14 +15,10 @@ import os -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ - set_dump_path -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json, set_stack_json_path +from msprobe.core.common.utils import CompareException +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json from msprobe.pytorch.common.log import logger -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.pytorch.compare.pt_compare import compare def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 308a82b3d6e9beb67a669ea05b83d7b8a6eddc90..16f0dedb9eea111fdfe090b68b9e7716df9f961d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,92 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os.path +from msprobe.core.compare.acc_compare import Comparator, ModeConfig, MappingConfig, setup_comparison +from msprobe.pytorch.compare.utils import read_pt_data -import torch -from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileChecker, create_directory, load_yaml -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ - set_dump_path -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.utils import set_stack_json_path -from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import load_pt - - -class PTComparator(Comparator): - def __init__(self, mode_config, data_mapping=None): - super().__init__(mode_config) - - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode - - self.frame_name = PTComparator.__name__ - self.data_mapping = data_mapping - if isinstance(self.data_mapping, str) or self.data_mapping is None: - self.data_mapping_dict = self.load_mapping_file(self.data_mapping) - elif isinstance(self.data_mapping, dict): - self.data_mapping_dict = self.data_mapping - else: - raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " - f"{type(self.data_mapping)}") - - @staticmethod - def load_mapping_file(mapping_file): - if isinstance(mapping_file, str): - mapping_dict = load_yaml(mapping_file) - else: - mapping_dict = {} - return mapping_dict - - def read_npy_data(self, dir_path, file_name): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - try: - # detach because numpy can not process gradient information - data_value = load_pt(data_path, to_cpu=True).detach() - except RuntimeError as e: - # 这里捕获 load_pt 中抛出的异常 - logger.error(f"Failed to load the .pt file at {data_path}.") - raise CompareException(CompareException.INVALID_FILE_ERROR) from e - except AttributeError as e: - # 这里捕获 detach 方法抛出的异常 - logger.error(f"Failed to detach the loaded tensor.") - raise CompareException(CompareException.DETACH_ERROR) from e - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - return data_value +def read_real_data(npu_dir, npu_data_name, bench_dir, bench_data_name, _) -> tuple: + n_value = read_pt_data(npu_dir, npu_data_name) + b_value = read_pt_data(bench_dir, bench_data_name) + return n_value, b_value def compare(input_param, output_path, **kwargs): - try: - auto_analyze = kwargs.get('auto_analyze', True) - fuzzy_match = kwargs.get('fuzzy_match', False) - data_mapping = kwargs.get('data_mapping', None) - suffix = kwargs.get('suffix', '') - - set_dump_path(input_param) - dump_mode = get_dump_mode(input_param) - if "stack_json_path" in input_param: - stack_mode = kwargs.get('stack_mode', False) - else: - stack_mode = set_stack_json_path(input_param) # set stack_mode and set "stack_json_path" in input_param - check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) - create_directory(output_path) - check_compare_param(input_param, output_path, dump_mode, stack_mode) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - raise CompareException(error.code) from error + config = setup_comparison(input_param, output_path, **kwargs) - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - pt_comparator = PTComparator(mode_config, data_mapping) - pt_comparator.compare_core(input_param, output_path, suffix=suffix) + mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match, config.dump_mode) + mapping_config = MappingConfig(data_mapping=config.data_mapping) + pt_comparator = Comparator(read_real_data, mode_config, mapping_config) + pt_comparator.compare_core(input_param, output_path, suffix=config.suffix) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/utils.py b/debug/accuracy_tools/msprobe/pytorch/compare/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..16473ff386d89de5f3bbb269e69837c07a950ea5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/utils.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch + +from msprobe.core.common.utils import logger, CompareException +from msprobe.core.common.file_utils import FileChecker, FileCheckConst +from msprobe.pytorch.common.utils import load_pt + + +def read_pt_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + try: + # detach because numpy can not process gradient information + data_value = load_pt(data_path, to_cpu=True).detach() + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py index 94244be326e9954c700339abec2db16a2ab31b07..6d39ffd8998c0d00863813203d8fd455305be707 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -6,15 +6,49 @@ import threading import unittest from unittest.mock import patch +import numpy as np import pandas as pd import torch +from msprobe.core.common.file_utils import load_json from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.highlight import find_error_rows, find_compare_result_error_rows, ApiBatch -from msprobe.core.compare.utils import get_accuracy -from msprobe.pytorch.compare.pt_compare import PTComparator +from msprobe.core.compare.acc_compare import ModeConfig, MappingConfig, MappingDict, Comparator, ParseData, ProcessDf, \ + Match, CreateTable, CalcStatsDiff + +npu_op_item_data_fuzzy = { + 'op_name': 'Functional.conv2d.0.forward.input.0', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.0.forward.input.0.pt', + 'compare_key': 'Functional.conv2d.0.forward.input.0', + 'compare_shape': [1, 1, 28, 28], +} +npu_op_item_fuzzy = pd.Series(npu_op_item_data_fuzzy) +npu_op_item_data_fuzzy_2 = { + 'op_name': 'Functional.conv2d.0.forward.input.1', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.0.forward.input.1.pt', + 'compare_key': 'Functional.conv2d.0.forward.input.1', + 'compare_shape': [1, 1, 28, 28], +} +npu_op_item_fuzzy_2 = pd.Series(npu_op_item_data_fuzzy_2) +bench_op_item_data_fuzzy = { + 'op_name': 'Functional.conv2d.1.forward.input.0', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.1.forward.input.0.pt', + 'compare_key': 'Functional.conv2d.1.forward.input.0', + 'compare_shape': [1, 1, 28, 28], +} +bench_op_item_fuzzy = pd.Series(bench_op_item_data_fuzzy) npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], @@ -159,7 +193,8 @@ aten_result = [ -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward.output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', + ' ', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', + 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward.output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', @@ -173,40 +208,6 @@ aten_result = [ highlight_dict = {'red_rows': [], 'yellow_rows': []} -num_0, num_1, num_2, num_3 = 0, 1, 2, 3 -summary_line_input = ['Functional_batch_norm_0_forward.input.0', 'Functional_batch_norm_0_forward.input.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -summary_line_1 = ['Functional_batch_norm_0_forward.output.0', 'Functional_batch_norm_0_forward.output.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -summary_line_2 = ['Functional_batch_norm_0_forward.output.1', 'Functional_batch_norm_0_forward.output.1', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, - 'Warning', ''] -summary_line_3 = ['Functional_batch_norm_0_forward.output.2', 'Functional_batch_norm_0_forward.output.2', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -line_input = ['Functional.batch.norm.0.forward.input.0', 'Functional.batch.norm.0.forward.input.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 0.5, 1, 1, 0.95, 1, - 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -line_1 = ['Functional.batch.norm.0.forward.output.0', 'Functional.batch.norm.0.forward.output.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1, 1, 0.59, 1, - 'nan', 0, 1, 1, 19, 1, 1, 1, - 'Yes', ''] -line_2 = ['Functional.batch.norm.0.forward.output.1', 'Functional.batch.norm.0.forward.output.1', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 0.5, 1, 1, 0.8, 1, - 0, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] -line_3 = ['Functional.batch.norm.0.forward.output.2', 'Functional.batch.norm.0.forward.output.2', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1.1e+10, 1, 0.85, 1, - 9, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] - op_data = { 'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, @@ -267,6 +268,33 @@ def generate_dump_json(base_dir): json.dump(data, json_file) +def generate_dump_json_md5(base_dir): + data_path = os.path.join(base_dir, 'dump_md5.json') + data = { + 'task': 'statistics', + 'level': 'L1', + 'dump_data_dir': '', + 'data': { + 'Functional.linear.0.forward': { + 'input_args': [ + {'type': 'torch.Tensor', + 'dtype': 'torch.float32', + 'shape': [2, 2], + 'Max': 2, + 'Min': 0, + 'Mean': 1, + 'Norm': 1, + 'requires_grad': False, + 'md5': 123456 + } + ] + } + } + } + with open(data_path, 'w') as json_file: + json.dump(data, json_file) + + def generate_stack_json(base_dir): data_path = os.path.join(base_dir, 'stack.json') data = {'Functional.linear.0.forward': ['File']} @@ -300,145 +328,6 @@ class TestUtilsMethods(unittest.TestCase): if os.path.exists(base_dir3): shutil.rmtree(base_dir3) - def test_get_accuracy_graph_mode(self): - result = [] - get_accuracy(result, npu_dict_aten, bench_dict_functional, dump_mode=Const.SUMMARY) - self.assertEqual(result, aten_result) - - def test_find_error_rows(self): - api_batch = ApiBatch("Functional_batch_norm_0_forward", 0) - api_batch.input_len = 1 - api_batch.output_end_index = 4 - api_batch.params_end_index = 4 - summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] - highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} - find_error_rows(summary_result, api_batch, highlight_dict_test, dump_mode=Const.SUMMARY) - self.assertEqual(highlight_dict_test, - {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}) - - def test_find_compare_result_error_rows(self): - result = [line_input, line_1, line_2, line_3] - result_df = pd.DataFrame(result) - highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} - find_compare_result_error_rows(result_df, highlight_dict_test, dump_mode=Const.ALL) - self.assertEqual(highlight_dict_test, { - "red_rows": {1, 3}, - "yellow_rows": {2}, - "red_lines": [ - (1, ["maximum or minimum is nan, -inf, or inf"]), - (3, ["maximum absolute error exceeds 1e+10"]) - ], - "yellow_lines": [ - (2, ["The output's one thousandth err ratio decreases by more than 0.1 compared to the input/parameters's"]), - (3, [ - "maximum absolute error of both input/parameters and output exceed 1, " - "with the output larger by an order of magnitude", - "The output's cosine decreases by more than 0.1 compared to the input/parameters's"]) - ] - }) - - def test_calculate_summary_data(self): - npu_summary_data = [1, 1, 1, 1] - bench_summary_data = [2, 2, 2, 2] - result_item = ['', '', '', '', '', '', '', '', '', '', '', '', '', ''] - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - comparator = Comparator(mode_config) - comparator.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - self.assertEqual(result_item, - ['', '', '', '', '', '', -1, -1, -1, -1, '50.0%', '50.0%', '50.0%', '50.0%', '', '']) - - bench_summary_data = [0, 0, 0, 0] - result_item = ['', '', '', '', '', '', '', '', '', '', '', '', '', ''] - - comparator.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - self.assertEqual(result_item, ['', '', '', '', '', '', 1, 1, 1, 1, 'N/A', 'N/A', 'N/A', 'N/A', 'Warning', - 'Need double check api accuracy.']) - - def test_make_result_table_stack_mode_True(self): - result_md5 = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', 'File']] - result_summary = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', 'File']] - result_all = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', 'File', '-1']] - columns_md5_stack_mode_true = CompareConst.MD5_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - result_table_md5_true = pd.DataFrame(result_md5, columns=columns_md5_stack_mode_true, dtype=object) - columns_summary_stack_mode_true = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - result_table_summary_true = pd.DataFrame(result_summary, columns=columns_summary_stack_mode_true, dtype=object) - columns_all_stack_mode_true = CompareConst.COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + ['Data_name'] - result_table_all_true = pd.DataFrame(result_all, columns=columns_all_stack_mode_true, dtype=object) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - - dump_mode = Const.MD5 - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_md5) - self.assertTrue(result_df.equals(result_table_md5_true)) - - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_summary) - self.assertTrue(result_df.equals(result_table_summary_true)) - - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_all) - self.assertTrue(result_df.equals(result_table_all_true)) - - def test_make_result_table_stack_mode_False(self): - result_md5_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '']] - result_md5 = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '']] - result_summary_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '']] - result_summary = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '']] - result_all_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '', '-1']] - result_all = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '-1']] - columns_md5_stack_mode_true = CompareConst.MD5_COMPARE_RESULT_HEADER - result_table_md5_true = pd.DataFrame(result_md5, columns=columns_md5_stack_mode_true, dtype='object') - columns_summary_stack_mode_true = CompareConst.SUMMARY_COMPARE_RESULT_HEADER - result_table_summary_true = pd.DataFrame(result_summary, columns=columns_summary_stack_mode_true, - dtype='object') - columns_all_stack_mode_true = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] - result_table_all_true = pd.DataFrame(result_all, columns=columns_all_stack_mode_true, dtype='object') - - stack_mode = False - auto_analyze = True - fuzzy_match = False - - dump_mode = Const.MD5 - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_md5_test) - self.assertTrue(result_df.equals(result_table_md5_true)) - - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_summary_test) - self.assertTrue(result_df.equals(result_table_summary_true)) - - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_all_test) - self.assertTrue(result_df.equals(result_table_all_true)) - def test_gen_merge_list(self): op_data = { 'input_args': [ @@ -469,32 +358,396 @@ class TestUtilsMethods(unittest.TestCase): dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result = Comparator(mode_config).gen_merge_list(json_data, op_name, stack_json_data) + result = ParseData(mode_config).gen_merge_list(json_data, op_name, stack_json_data) self.assertEqual(result, merge_list) - def test_check_op_fuzzy_false(self): + def test_check_op_item_fuzzy(self): stack_mode = False auto_analyze = True dump_mode = Const.SUMMARY - fuzzy_match = False + fuzzy_match = True mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - result = pt_comparator.check_op(npu_dict, bench_dict) + match = Match(mode_config, mapping_config, cross_frame=False) + result = match.check_op_item(npu_op_item_fuzzy, bench_op_item_fuzzy) self.assertEqual(result, True) - def test_check_op_fuzzy_true(self): - stack_mode = False + def test_compare_statistics(self): + generate_dump_json(base_dir) + generate_stack_json(base_dir) + file_list = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), + os.path.join(base_dir, 'stack.json')] + + stack_mode = True auto_analyze = True + fuzzy_match = False dump_mode = Const.SUMMARY - - fuzzy_match = True mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - result = pt_comparator.check_op(npu_dict2, bench_dict) - self.assertEqual(result, True) + from msprobe.pytorch.compare.pt_compare import read_real_data + comparator = Comparator(read_real_data, mode_config, mapping_config) + result = comparator.compare_statistics(file_list) + o_data = [ + ['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', '[2, 2]', '[2, 2]', 0, 0, 0, 0, '0.0%', 'N/A', '0.0%', '0.0%', + 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'] + ] + ] + columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + o_result = pd.DataFrame(o_data, columns=columns, dtype=object) + self.assertTrue(np.array_equal(result.to_numpy(), o_result.to_numpy())) + + +class TestParseData(unittest.TestCase): + + def setUp(self): + os.makedirs(base_dir, mode=0o750, exist_ok=True) + generate_dump_json(base_dir) + generate_dump_json_md5(base_dir) + generate_stack_json(base_dir) + + self.lock = threading.Lock() + + def tearDown(self): + if os.path.exists(base_dir): + shutil.rmtree(base_dir) + + def test_parse(self): + file_list = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), + os.path.join(base_dir, 'stack.json')] + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + npu_df, bench_df = parse_data.parse(file_list) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + self.assertTrue(npu_df.equals(target_df)) + self.assertTrue(bench_df.equals(target_df)) + + def test_gen_data_df_summary(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_data_df_all(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=Const.ALL) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'Functional.linear.0.forward.input.0.pt']], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_data_df_md5(self): + npu_json_path = os.path.join(base_dir, 'dump_md5.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=Const.MD5) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 123456]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'md5'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_merge_list(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + merge_list = parse_data.gen_merge_list(npu_json_data, 'Functional.linear.0.forward', stack_json_data) + + target_dict = { + 'input_struct': [('torch.float32', [2, 2])], + 'op_name': ['Functional.linear.0.forward.input.0'], + 'output_struct': [], + 'params_grad_struct': [], + 'params_struct': [], + 'stack_info': [['File']], + 'summary': [[2, 0, 1, 1]] + } + self.assertEqual(merge_list, target_dict) + + +class TestProcessDf(unittest.TestCase): + + def test_get_api_name_success(self): + api_list = ['Functional', 'linear', '0', 'forward', 'input', '0'] + + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + api_name = process_df.get_api_name(api_list) + + target_api_name = 'Functional.linear' + self.assertEqual(api_name, target_api_name) + + @patch('msprobe.core.compare.acc_compare.logger') + def test_get_api_name_index_error(self, mock_logger): + api_list = ['Functional'] + with self.assertRaises(CompareException) as context: + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + api_name = process_df.get_api_name(api_list) + self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + mock_logger.error.assert_called_once_with('Failed to retrieve API name, please check if the dump data is reasonable') + + def test_process_compare_key_and_shape(self): + npu_df_o = bench_df_o = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + npu_df, bench_df = process_df.process_compare_key_and_shape(npu_df_o, bench_df_o) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'Functional.linear.0.forward.input.0', [2, 2]]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key', 'compare_shape'] + ) + self.assertTrue(npu_df.equals(target_df)) + self.assertTrue(bench_df.equals(target_df)) + + def test_process_internal_api_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + # mint to torch + npu_op_name = 'Mint.mean.0.input.0' + target_name = 'Torch.mean.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # mintfunctional to functional + npu_op_name = 'MintFunctional.mean.0.input.0' + target_name = 'Functional.mean.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # inner mapping exists + npu_op_name = 'Functional.abs.0.input.0' + mapping_dict.ms_to_pt_mapping = {'Functional.abs': 'Torch.abs'} + target_name = 'Torch.abs.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # inner mapping not found + npu_op_name = 'Functional.abs.0.input.0' + mapping_dict.ms_to_pt_mapping = {} + target_name = 'Functional.abs.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + def test_modify_compare_data_with_user_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + mapping_dict.api_mapping_dict = [{ + 'ms_api': 'Functional.conv2d', + 'pt_api': 'Torch.conv2d', + 'ms_args': [0], + 'pt_args': [0] + }] + + npu_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0'], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + bench_df = pd.DataFrame([ + ['Torch.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Torch.conv2d.0.forward.input.0'], + ['Torch.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Torch.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + + process_df.modify_compare_data_with_user_mapping(npu_df, bench_df) + + def test_get_api_indices_dict(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + op_name_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0'], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + + api_indices_dict = process_df.get_api_indices_dict(op_name_df) + expected = { + 'Functional.conv2d': [0], + 'Functional.amax': [1] + } + self.assertEqual(api_indices_dict, expected) + + def test_process_cell_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + # not name + npu_op_name = None + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, CompareConst.N_A) + + # not params_grad + npu_op_name = 'MintFunctional.embedding.0.input.0' + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, CompareConst.N_A) + + # default replace + npu_op_name = 'Cell.network_with_loss.module.GPTModel.forward.1.input.0' + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, 'Module.network_with_loss.module.GPTModel.forward.1.input.0') + + # mapping_dict + npu_op_name = 'Cell.fc1.Dense.forward.0.input.0' + mapping_dict.cell_mapping_dict = {'fc1.Dense': 'module.name'} + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, 'Module.module.name.forward.0.input.0') + + def test_process_data_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + npu_op_name = 'Functional.flash_attention_score.4.forward.input.0' + mapping_dict.data_mapping_dict = {'Functional.flash_attention_score.4.forward.input.0': 'NPU.npu_fusion_attention.4.forward.input.0'} + name = process_df.process_data_mapping(npu_op_name) + self.assertEqual(name, 'NPU.npu_fusion_attention.4.forward.input.0') + + +class TestMatch(unittest.TestCase): + + def test_put_unmatched_in_table(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + match_result = match.put_unmatched_in_table(match_result, npu_op_item) + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], + 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']], + columns=CompareConst.MATCH_RESULT_COLUMNS) + self.assertTrue(match_result.equals(target_match_result)) + + def test_put_matched_in_table(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + 'compare_key', 'compare_shape'] + ) + match_result = match.put_matched_in_table(match_result, npu_op_item, bench_op_item) + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], + 'op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name']], + columns=CompareConst.MATCH_RESULT_COLUMNS) + self.assertTrue(match_result.equals(target_match_result)) + + def test_rename_api(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + op_name = 'Functional.linear.0.forward.input.0' + result = match.rename_api(op_name) + self.assertTrue(result, 'Functional.linear.input.0') + + def test_check_op_item(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'Functional.linear.0.forward.input.0', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'Functional.linear.1.forward.input.0', [1, 2]], + index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + 'compare_key', 'compare_shape'] + ) + result = match.check_op_item(npu_op_item, bench_op_item) + self.assertTrue(result) + + def test_process_fuzzy_match(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + npu_df = pd.DataFrame([ + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2]], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + bench_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt', 'Functional.conv2d.0.forward.input.0', [1, 2]], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.0.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + + match_result = match.process_fuzzy_match(npu_df, bench_df) + expected = pd.DataFrame( + [ + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2], 'Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt'], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2], 'Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt'] + ] + , columns=CompareConst.MATCH_RESULT_COLUMNS) + + self.assertTrue(match_result.equals(expected)) def test_match_op_both_last_element(self): stack_mode = False @@ -502,9 +755,10 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict], [bench_dict]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy], [bench_op_item_fuzzy]) self.assertEqual(a, 0) self.assertEqual(b, 0) @@ -514,9 +768,10 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict], [bench_dict, 1]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy], [bench_op_item_fuzzy, 1]) self.assertEqual(a, 0) self.assertEqual(b, 0) @@ -526,217 +781,102 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict, npu_dict2], [bench_dict]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy, npu_op_item_data_fuzzy_2], [bench_op_item_fuzzy]) self.assertEqual(a, 0) self.assertEqual(b, 0) - def test_compare_process(self): - generate_dump_json(base_dir) - generate_stack_json(base_dir) - file_lists = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), - os.path.join(base_dir, 'stack.json')] + def test_gen_dtype_condition(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=True) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + # data mapping + mapping_config.data_mapping = True + match_result = pd.DataFrame([1, 2, 3]) + result = match.gen_dtype_condition(match_result) + expected = pd.Series([True, True, True]) + self.assertTrue(result.equals(expected)) - result = PTComparator(mode_config).compare_process(file_lists) - o_data = [ - ['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], 0, 0, 0, 0, '0.0%', 'N/A', '0.0%', '0.0%', - 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'] - ] - ] - columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - o_result = pd.DataFrame(o_data, columns=columns, dtype=object) - self.assertTrue(result.equals(o_result)) + # normal + mapping_config.data_mapping = None + match_result = pd.DataFrame([['Float16', 'Float32'], ['torch.float32', 'torch.bfloat16']], columns=['dtype_x', 'dtype_y']) + result = match.gen_dtype_condition(match_result) + expected = pd.Series([True, True]) + self.assertTrue(result.equals(expected)) - def test_merge_data(self): - op_data = { - 'input_args': [ - { - 'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [2, 2], - 'Max': 1, 'Min': 1, 'Mean': 1, 'Norm': 1, 'requires_grad': False, - 'data_name': 'Functional.linear.0.forward.input.0.pt', - 'full_op_name': 'Functional.linear.0.forward.input.0' - } - ] - } - json_data = {'data': {'Functional.linear.0.forward': op_data}} - stack_json_data = {'Functional.linear.0.forward': ['File']} - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - result = Comparator(mode_config).merge_data(json_data, stack_json_data) - ops_all = { - 'Functional.linear.0.forward.input.0': { - 'data_name': None, 'stack_info': [['File']], - 'struct': ('torch.float32', [2, 2]), 'summary': [1, 1, 1, 1] - } - } - self.assertEqual(result, ops_all) - - def test_compare_core_basic(self): - generate_dump_json(base_dir2) - generate_stack_json(base_dir2) - input_params = { - "npu_json_path": os.path.join(base_dir2, "dump.json"), - "bench_json_path": os.path.join(base_dir2, "dump.json"), - "stack_json_path": os.path.join(base_dir2, "stack.json"), - } - output_path = base_dir2 - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - PTComparator(mode_config).compare_core(input_params, output_path) - - output_files = os.listdir(output_path) - self.assertTrue(any(f.endswith(".xlsx") for f in output_files)) - - def test_compare_ops(self): - generate_dump_json(base_dir3) - generate_stack_json(base_dir3) - generate_pt(pt_dir) - dump_path = os.path.join(base_dir3, 'dump.json') - stack_path = os.path.join(base_dir3, 'stack.json') - input_param = {'npu_json_path': dump_path, 'bench_json_path': dump_path, 'stack_json_path': stack_path, - 'is_print_compare_log': True, 'npu_dump_data_dir': pt_dir, 'bench_dump_data_dir': pt_dir} - dump_path_dict = {'Functional.linear.0.forward.input.0': ['Functional.linear.0.forward.input.0.pt', - 'Functional.linear.0.forward.input.0.pt']} - result_df = pd.DataFrame({ - 'NPU Name': ['Functional.linear.0.forward.input.0'], - 'Bench Name': ['Functional.linear.0.forward.input.0'] - }) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - updated_df = pt_comparator.compare_ops(idx=0, dump_path_dict=dump_path_dict, result_df=result_df, - lock=self.lock, input_param=input_param) - - self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 1.0) - self.assertEqual(updated_df.loc[0, CompareConst.MAX_ABS_ERR], 0) - - def test_do_multi_process(self): - data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], - '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', ['-1', '-1']]] - o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], - 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', ['-1', '-1']]] - columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] - result_df = pd.DataFrame(data, columns=columns) - o_result = pd.DataFrame(o_data, columns=columns) - generate_dump_json(base_dir) - input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} + def test_process_cross_frame_dtype(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=True) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + dtype_o = pd.Series(['Int8', 'Float16', 'torch.bool', 'Complex64', 'unknown']) + dtype = match.process_cross_frame_dtype(dtype_o) + self.assertTrue(dtype.equals(pd.Series(['int', 'float', 'bool', 'complex', 'unknown']))) - comparator = Comparator(mode_config) - result = comparator.do_multi_process(input_param, result_df) - self.assertTrue(result.equals(o_result)) - def test_compare_by_op_1(self): - npu_op_name = 'Functional.linear.0.forward.input.0' - bench_op_name = 'N/A' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [-1, -1]} - input_param = {} +class TestCreateTable(unittest.TestCase): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + def test_process_data_name(self): + mode_config = ModeConfig() + create_table = CreateTable(mode_config) - pt_comparator = PTComparator(mode_config) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + data = { + 'data_name_x': ['A', 'B', 'C'], + 'data_name_y': ['X', 'Y', 'Z'] + } + result_o = pd.DataFrame(data) + result = create_table.process_data_name(result_o) + target_data = { + 'data_name_x': [['A', 'X'], ['B', 'Y'], ['C', 'Z']], + 'data_name_y': ['X', 'Y', 'Z'] + } + target_result = pd.DataFrame(target_data) + self.assertTrue(result.equals(target_result)) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'No bench data matched.']) + def test_set_summary(self): + mode_config = ModeConfig() + create_table = CreateTable(mode_config) - def test_compare_by_op_2(self): - npu_op_name = 'Functional.linear.0.forward.input.0' - bench_op_name = 'Functional.linear.0.forward.input.0' + # all nan + result = create_table.set_summary(['nan', 'NaN', 'nAn']) + expected = [CompareConst.NAN, CompareConst.NAN, CompareConst.NAN] + self.assertEqual(result, expected) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + # mixed values + result = create_table.set_summary([1, 'nan', 2.0, 'NaN']) + expected = [1, CompareConst.NAN, 2.0, CompareConst.NAN] + self.assertEqual(result, expected) - pt_comparator = PTComparator(mode_config) + # NA case + result = create_table.set_summary(CompareConst.N_A) + expected = [CompareConst.N_A, CompareConst.N_A, CompareConst.N_A, CompareConst.N_A] + self.assertEqual(result, expected) - pt_name = '-1' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} - input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'No bench data matched.']) + # empty input + result = create_table.set_summary([]) + expected = [] + self.assertEqual(result, expected) - pt_name = 'Functional.linear.0.forward.input.0.pt' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} - input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found.']) - generate_pt(base_dir) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, '']) +class TestCalcStatsDiff(unittest.TestCase): + def test_type_check(self): + mode_config = ModeConfig() + calc_stats_diff = CalcStatsDiff(mode_config) -class TestComparator(unittest.TestCase): - def setUp(self): - mode_config = ModeConfig(dump_mode=Const.MD5) - self.comparator = Comparator(mode_config=mode_config) - self.npu_ops_all = { - 'op1': {'struct': ['float32', [1, 96, 2], '83dcefb7']}, - } - self.bench_ops_all = { - 'op1': {'struct': ['float32', [1, 96, 2], '83dcefb7']}, - } + series = pd.Series([float('nan'), 5, 'nan', 10, 'abc', None]) + result = calc_stats_diff.type_check(series) + expected = pd.Series([True, True, True, True, False, False]) + self.assertTrue(result.equals(expected)) - def test_normal(self): - expected_result = ['op1', 'op1', 'float32', 'float32', [1, 96, 2], [1, 96, 2], '83dcefb7', '83dcefb7', - CompareConst.PASS, CompareConst.NONE] - result = self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all) - self.assertEqual(result, expected_result) + def test_get_number(self): + mode_config = ModeConfig() + calc_stats_diff = CalcStatsDiff(mode_config) - @patch('msprobe.core.compare.acc_compare.logger') - def test_length_exception(self, mock_logger): - self.npu_ops_all['op1']['struct'] = ['npu_val1', 'npu_val2'] - with self.assertRaises(CompareException) as context: - self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - mock_logger.error.assert_called_once_with("The length of npu_struct and bench_struct must be >= 3, " - "but got npu_struct=2 and bench_struct=3. Please check!") - - def test_with_extra_args(self): - expected_result = ['op1', 'op1', 'float32', 'float32', [1, 96, 2], [1, 96, 2], '83dcefb7', '83dcefb7', - CompareConst.PASS, 'extra_data'] - result = self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all, True, ['extra_data']) - self.assertEqual(result, expected_result) + series = pd.Series([1, '2', 3.5, 'text', None]) + result = calc_stats_diff.get_number(series) + expected = pd.Series([1, 2, 3.5, float('nan'), float('nan')]) + self.assertTrue(result.equals(expected)) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py index a1e5f8eee1bce9b170e6f4f7fdfeda65d47252c9..1a0a33f799724ffefe73bf8f024e0146b2925464 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py @@ -1,7 +1,6 @@ # coding=utf-8 import unittest -from msprobe.core.compare.check import check_struct_match, check_type_shape_match, check_graph_mode, fuzzy_check_op, \ - fuzzy_check_name, check_dump_json_str, check_json_key_value, valid_key_value, check_stack_json_str +from msprobe.core.compare.check import check_dump_json_str, check_json_key_value, valid_key_value, check_stack_json_str from msprobe.core.common.utils import CompareException @@ -65,87 +64,6 @@ op_name = 'Functional.conv2d.0.backward.input.0' class TestUtilsMethods(unittest.TestCase): - - def test_check_struct_match_success(self): - result = check_struct_match(npu_dict, bench_dict) - self.assertTrue(result) - - def test_check_struct_match_fail(self): - npu_dict2 = {'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])] - } - - bench_dict2 = {'input_struct': [('torch.float32', [2, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])] - } - result = check_struct_match(npu_dict2, bench_dict2) - self.assertFalse(result) - - def test_check_struct_index_error(self): - npu_dict3 = {'input_struct': [('a'), ('torch.float32'), - ('torch.float32')], - 'output_struct': [('torch.float32')] - } - - bench_dict3 = {'input_struct': [('torch.float32'), ('torch.float32'), - ('torch.float32')], - 'output_struct': [('torch.float32')] - } - with self.assertRaises(CompareException) as context: - result = check_struct_match(npu_dict3, bench_dict3) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - def test_check_type_shape_match_success(self): - result = check_type_shape_match(npu_struct, bench_struct) - self.assertTrue(result) - - def test_check_type_shape_match_index_error(self): - npu_struct2 = [('a'), ('torch.float32'), ('torch.float32')] - bench_struct2 = [('torch.float32'), ('torch.float32'), ('torch.float32')] - with self.assertRaises(CompareException) as context: - result = check_type_shape_match(npu_struct2, bench_struct2) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - def test_check_graph_mode(self): - op1 = "Aten" - op2 = "torch" - self.assertTrue(check_graph_mode(op1, op2)) - self.assertTrue(check_graph_mode(op2, op1)) - self.assertFalse(check_graph_mode(op1, op1)) - self.assertFalse(check_graph_mode(op2, op2)) - - def test_fuzzy_check_op_1(self): - npu_name_list = [] - bench_name_list = [] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertFalse(result) - - def test_fuzzy_check_op_2(self): - npu_name_list = [] - bench_name_list = ['Functional.conv2d.0.forward.input.0'] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertFalse(result) - - def test_fuzzy_check_op_3(self): - npu_name_list = ['Functional.conv2d.0.forward.input.0'] - bench_name_list = ['Functional.conv2d.1.forward.input.0'] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertTrue(result) - - def test_fuzzy_check_name_1(self): - npu_name = 'Functional.conv2d.0.backward.input.0' - bench_name = 'Functional.conv2d.1.backward.input.0' - result = fuzzy_check_name(npu_name, bench_name) - self.assertTrue(result) - - def test_fuzzy_check_name_2(self): - npu_name = 'Functional.conv2d.0.backward.input.0' - bench_name = 'Functional.conv2d.1.backward.input.1' - result = fuzzy_check_name(npu_name, bench_name) - self.assertFalse(result) - def test_check_dump_json_str(self): with self.assertRaises(CompareException) as context: check_dump_json_str(op_data, op_name) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index bf23f4de1dac73a44a2497e1a927ba30e5440715..11d88eee940f08a3a3276679fcb5bc490e2da02e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -12,9 +12,8 @@ import numpy as np from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.compare.utils import ApiItemInfo, _compare_parser, check_and_return_dir_contents, extract_json, \ - count_struct, get_accuracy, append_stack_info, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - op_item_parse, read_op, rename_api, resolve_api_special_parameters, result_item_init, stack_column_process, \ - table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item + count_struct, get_accuracy, get_rela_diff_summary_mode, merge_tensor, op_item_parse, read_op, result_item_init, \ + stack_column_process, table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item # test_read_op_1 op_data = { @@ -350,18 +349,6 @@ class TestUtilsMethods(unittest.TestCase): result = check_and_return_dir_contents(base_dir2, 'rank') self.assertEqual(set(result), set(['rank0', 'rank1'])) - def test_rename_api_1(self): - test_name_1 = "Distributed.broadcast.0.forward.input.0" - expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = rename_api(test_name_1, "forward") - self.assertEqual(actual_name_1, expect_name_1) - - def test_rename_api_2(self): - test_name_2 = "Torch.sum.0.backward.output.0" - expect_name_2 = "Torch.sum.output.0" - actual_name_2 = rename_api(test_name_2, "backward") - self.assertEqual(actual_name_2, expect_name_2) - def test_read_op(self): result = read_op(op_data, op_name) self.assertEqual(result, op_result) @@ -379,11 +366,6 @@ class TestUtilsMethods(unittest.TestCase): op_item_parse(parse_item, parse_op_name, depth=11) self.assertEqual(context.exception.code, CompareException.RECURSION_LIMIT_ERROR) - def test_resolve_api_special_parameters(self): - item_list = [] - resolve_api_special_parameters(data_dict, full_op_name, item_list) - self.assertEqual(item_list, o_result_api_special) - def test_get_rela_diff_summary_mode_float_or_int(self): result_item = [0] * 14 err_msg = '' @@ -449,57 +431,6 @@ class TestUtilsMethods(unittest.TestCase): get_accuracy(result, npu_dict, bench_dict, dump_mode=Const.SUMMARY) self.assertEqual(result, o_result) - def test_append_stack_info_stack_exist_index_0(self): - result_item = ['item1'] - npu_stack_info = ['stack_info1'] - index = 0 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', 'stack_info1']) - - def test_append_stack_info_stack_exist_index_not_0(self): - result_item = ['item1'] - npu_stack_info = ['stack_info1'] - index = 1 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_append_stack_info_stack_empty_index_0(self): - result_item = ['item1'] - npu_stack_info = [] - index = 0 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_append_stack_info_stack_empty_index_not_0(self): - result_item = ['item1'] - npu_stack_info = [] - index = 1 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_get_un_match_accuracy_md5(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.MD5) - self.assertEqual(result, o_result_unmatch_1) - - def test_get_un_match_accuracy_summary(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.SUMMARY) - self.assertEqual(result, o_result_unmatch_2) - - def test_get_un_match_accuracy_all(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.ALL) - self.assertEqual(result, o_result_unmatch_3) - def test_merge_tensor_summary(self): op_dict = merge_tensor(tensor_list, dump_mode=Const.SUMMARY) self.assertEqual(op_dict, result_op_dict) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py index 3261bce5d6d0a15d8e46c7d9fc22df0cf64c9e4d..5ffc0013fad8cfa289a79f5aaf39219b31b77c07 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py @@ -12,12 +12,44 @@ import openpyxl from openpyxl import load_workbook from openpyxl.styles import PatternFill - from msprobe.core.common.const import CompareConst, Const from msprobe.core.compare.highlight import ApiBatch, CheckMaxRelativeDiff, CheckOrderMagnitude, \ - CheckOneThousandErrorRatio, CheckCosineSimilarity, add_highlight_row_info, compare_result_df_convert, \ - df_malicious_value_check, find_error_rows, highlight_rows_xlsx, update_highlight_err_msg, value_check - + CheckOneThousandErrorRatio, CheckCosineSimilarity, add_highlight_row_info, HighLight +from msprobe.core.compare.config import ModeConfig + + +summary_line_input = ['Functional_batch_norm_0_forward.input.0', 'Functional_batch_norm_0_forward.input.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +summary_line_1 = ['Functional_batch_norm_0_forward.output.0', 'Functional_batch_norm_0_forward.output.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +summary_line_2 = ['Functional_batch_norm_0_forward.output.1', 'Functional_batch_norm_0_forward.output.1', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +summary_line_3 = ['Functional_batch_norm_0_forward.output.2', 'Functional_batch_norm_0_forward.output.2', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +line_input = ['Functional.batch.norm.0.forward.input.0', 'Functional.batch.norm.0.forward.input.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 0.5, 1, 1, 0.95, 1, + 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +line_1 = ['Functional.batch.norm.0.forward.output.0', 'Functional.batch.norm.0.forward.output.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1, 1, 0.59, 1, + 'nan', 0, 1, 1, 19, 1, 1, 1, + 'Yes', ''] +line_2 = ['Functional.batch.norm.0.forward.output.1', 'Functional.batch.norm.0.forward.output.1', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 0.5, 1, 1, 0.8, 1, + 0, 0.12, 0, 1, 1, 0.1, 1, 1, + 'Yes', ''] +line_3 = ['Functional.batch.norm.0.forward.output.2', 'Functional.batch.norm.0.forward.output.2', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1.1e+10, 1, 0.85, 1, + 9, 0.12, 0, 1, 1, 0.1, 1, 1, + 'Yes', ''] base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_highlight') @@ -161,7 +193,7 @@ class TestUtilsMethods(unittest.TestCase): num = 1 info = (api_in, api_out, num) CheckMaxRelativeDiff().apply(info, color_columns, dump_mode=Const.SUMMARY) - red_lines, yellow_lines = [], [(1, ["The output's maximum relative error exceeds 0.1, while the input/parameters's is below 0.01"])] + red_lines, yellow_lines = [], [(1, ["The output's maximum relative error exceeds 0.1, while the input/parameter's is below 0.01"])] target_color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) self.assertEqual(color_columns, target_color_columns) @@ -178,45 +210,6 @@ class TestUtilsMethods(unittest.TestCase): result = CheckMaxRelativeDiff().apply(info, color_columns, dump_mode=Const.SUMMARY) self.assertEqual(result, None) - def test_find_error_rows_normal(self): - compare_result = np.array([ - ["Functional.linear.0.forward.input.0", "Functional.linear.0.forward.input.0", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.input.1", "Functional.linear.0.forward.input.1", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.input.2", "Functional.linear.0.forward.input.2", - "torch.float32", "torch.float32", [2], [2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.output.0", "Functional.linear.0.forward.output.0", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ], dtype=object) - api_batch = ApiBatch("Functional.linear.0.forward", 0) - api_batch.input_len = 3 - api_batch.output_end_index = 4 - api_batch.params_end_index = 4 - highlight_dict = {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()} - dump_mode = Const.ALL - - find_error_rows(compare_result, api_batch, highlight_dict, dump_mode) - - self.assertEqual(highlight_dict, {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()}) - - def test_find_error_rows_md5(self): - compare_result = [] - api_batch = ApiBatch("", 0) - api_batch.input_len = 0 - api_batch.output_end_index = 1 - api_batch.params_end_index = 1 - highlight_dict = {} - dump_mode = Const.MD5 - - result = find_error_rows(compare_result, api_batch, highlight_dict, dump_mode) - - self.assertEqual(result, None) - def test_ApiBatch_increment_input(self): api_name = "functional.conv2d" start = 2 @@ -297,6 +290,48 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(api_batch.output_end_index, 5) self.assertEqual(api_batch.params_grad_end_index, 5) + + def test_find_error_rows_normal(self): + compare_result = np.array([ + ["Functional.linear.0.forward.input.0", "Functional.linear.0.forward.input.0", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.input.1", "Functional.linear.0.forward.input.1", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.input.2", "Functional.linear.0.forward.input.2", + "torch.float32", "torch.float32", [2], [2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.output.0", "Functional.linear.0.forward.output.0", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ], dtype=object) + api_batch = ApiBatch("Functional.linear.0.forward", 0) + api_batch.input_len = 3 + api_batch.output_end_index = 4 + api_batch.params_end_index = 4 + highlight_dict = {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()} + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.find_error_rows(compare_result, api_batch, highlight_dict) + + self.assertEqual(highlight_dict, {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()}) + + def test_find_error_rows_md5(self): + compare_result = [] + api_batch = ApiBatch("", 0) + api_batch.input_len = 0 + api_batch.output_end_index = 1 + api_batch.params_end_index = 1 + highlight_dict = {} + + mode_config = ModeConfig(dump_mode=Const.MD5) + highlight = HighLight(mode_config) + result = highlight.find_error_rows(compare_result, api_batch, highlight_dict) + + self.assertEqual(result, None) + @patch("msprobe.core.compare.highlight.logger") def test_value_check(self, mock_logger): value = "=functional.conv2d" @@ -304,7 +339,9 @@ class TestUtilsMethods(unittest.TestCase): i = 1 result_df_columns = CompareConst.COMPARE_RESULT_HEADER - value_check(value, api_name, i, result_df_columns) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + highlight.value_check(value, api_name, i, result_df_columns) mock_logger.error.assert_called_once_with( "Malicious value [=functional.conv2d] at api_name [=functional.conv2d], column [Bench Name], " @@ -319,11 +356,15 @@ class TestUtilsMethods(unittest.TestCase): ] result_df = pd.DataFrame(data, columns=columns) - df_malicious_value_check(result_df, columns) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.df_malicious_value_check(result_df, columns) def test_compare_result_df_convert(self): value = float("nan") - result = compare_result_df_convert(value) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + result = highlight.compare_result_df_convert(value) self.assertEqual(result, "nan\t") def test_highlight_rows_xlsx_red(self): @@ -335,7 +376,11 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {'red_rows': [0]} file_path = os.path.join(base_dir, 'result.xlsx') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + generate_result_xlsx(base_dir) self.assertTrue(compare_excel_files_with_highlight(file_path, os.path.join(base_dir, 'target_result.xlsx'))) @@ -348,7 +393,11 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {'yellow_rows': [0]} file_path = os.path.join(base_dir, 'result.xlsx') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + generate_result_xlsx(base_dir) self.assertTrue(compare_excel_files_with_highlight(file_path, os.path.join(base_dir, 'target_result_yellow.xlsx'))) @@ -366,7 +415,9 @@ class TestUtilsMethods(unittest.TestCase): temp_output_file = 'temp_output.txt' sys.stdout = open(temp_output_file, 'w') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) with open(temp_output_file, 'r') as f: output = f.read() @@ -391,7 +442,9 @@ class TestUtilsMethods(unittest.TestCase): temp_output_file = 'temp_output.txt' sys.stdout = open(temp_output_file, 'w') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) with open(temp_output_file, 'r') as f: output = f.read() @@ -429,7 +482,10 @@ class TestUtilsMethods(unittest.TestCase): 'red_lines': [(0, ['a', 'b'])], 'yellow_lines': [(0, ['c']), (1, ['d'])] } - update_highlight_err_msg(result_df, highlight_dict) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.update_highlight_err_msg(result_df, highlight_dict) t_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], @@ -449,7 +505,9 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {} - result = update_highlight_err_msg(result_df, highlight_dict) + mode_config = ModeConfig(dump_mode=Const.MD5) + highlight = HighLight(mode_config) + result = highlight.update_highlight_err_msg(result_df, highlight_dict) self.assertEqual(result, None) @@ -466,5 +524,43 @@ class TestUtilsMethods(unittest.TestCase): 'red_lines': [(0, ['a', 'b'])], 'yellow_lines': [(0, ['c']), (1, ['d'])] } - result = update_highlight_err_msg(result_df, highlight_dict) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + result = highlight.update_highlight_err_msg(result_df, highlight_dict) self.assertEqual(result, None) + + def test_find_error_rows(self): + api_batch = ApiBatch("Functional_batch_norm_0_forward", 0) + api_batch.input_len = 1 + api_batch.output_end_index = 4 + api_batch.params_end_index = 4 + summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] + highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + mode_config = ModeConfig() + highlight = HighLight(mode_config) + highlight.find_error_rows(summary_result, api_batch, highlight_dict_test) + self.assertEqual(highlight_dict_test, + {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}) + + def test_find_compare_result_error_rows(self): + result = [line_input, line_1, line_2, line_3] + result_df = pd.DataFrame(result) + highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.find_compare_result_error_rows(result_df, highlight_dict_test) + self.assertEqual(highlight_dict_test, { + "red_rows": {1, 3}, + "yellow_rows": {2}, + "red_lines": [ + (1, ["maximum or minimum is nan, -inf, or inf"]), + (3, ["maximum absolute error exceeds 1e+10"]) + ], + "yellow_lines": [ + (2, ["The output's one thousandth err ratio decreases by more than 0.1 compared to the input/parameter's"]), + (3, [ + "maximum absolute error of both input/parameters and output exceed 1, " + "with the output larger by an order of magnitude", + "The output's cosine decreases by more than 0.1 compared to the input/parameter's"]) + ] + }) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py index 49f084ce07c8e90afb2aa1c3340bb4c3965c8fa7..0180c08e87f6cb78c392223830214fccffb8c149 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py @@ -7,12 +7,12 @@ import unittest import pandas as pd -from msprobe.core.common.const import CompareConst, Const +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result, \ - check_accuracy, read_dump_data -from test_acc_compare import generate_dump_json +from msprobe.core.compare.acc_compare import ModeConfig +from msprobe.core.compare.multiprocessing_compute import check_accuracy, CompareRealData, ComparisonResult +from msprobe.pytorch.compare.pt_compare import read_real_data +from test_acc_compare import generate_dump_json, generate_pt, generate_stack_json data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], @@ -28,10 +28,49 @@ columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] result_df = pd.DataFrame(data, columns=columns) o_result = pd.DataFrame(o_data, columns=columns) base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_cmp_multiprocessing_compute') +base_dir3 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_acc_compare_data3') +pt_dir = os.path.join(base_dir3, f'dump_data_dir') class TestUtilsMethods(unittest.TestCase): + def test_check_accuracy(self): + max_abs_err = '' + + cos_1 = CompareConst.SHAPE_UNMATCH + result_1 = check_accuracy(cos_1, max_abs_err) + self.assertEqual(result_1, CompareConst.ACCURACY_CHECK_UNMATCH) + + cos_2 = CompareConst.NONE + result_2 = check_accuracy(cos_2, max_abs_err) + self.assertEqual(result_2, CompareConst.NONE) + + cos_3 = 'N/A' + result_3 = check_accuracy(cos_3, max_abs_err) + self.assertEqual(result_3, CompareConst.ACCURACY_CHECK_NO) + + cos_4 = '' + result_4 = check_accuracy(cos_4, max_abs_err) + self.assertEqual(result_4, CompareConst.NONE) + + cos_5 = 0.95 + max_abs_err = 0.002 + result_5 = check_accuracy(cos_5, max_abs_err) + self.assertEqual(result_5, CompareConst.ACCURACY_CHECK_NO) + + cos_6 = 0.85 + max_abs_err = 2 + result_6 = check_accuracy(cos_6, max_abs_err) + self.assertEqual(result_6, CompareConst.ACCURACY_CHECK_NO) + + cos_7 = 0.95 + max_abs_err = 0.001 + result_7 = check_accuracy(cos_7, max_abs_err) + self.assertEqual(result_7, CompareConst.ACCURACY_CHECK_YES) + + +class TestCompareRealData(unittest.TestCase): + def setUp(self): self.result_df = pd.DataFrame(columns=[ CompareConst.COSINE, CompareConst.EUC_DIST, CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, @@ -39,35 +78,39 @@ class TestUtilsMethods(unittest.TestCase): CompareConst.ACCURACY, CompareConst.ERROR_MESSAGE ]) os.makedirs(base_dir, mode=0o750, exist_ok=True) + os.makedirs(base_dir3, mode=0o750, exist_ok=True) + os.makedirs(pt_dir, mode=0o750, exist_ok=True) self.lock = threading.Lock() def tearDown(self): if os.path.exists(base_dir): shutil.rmtree(base_dir) - - def test_handle_multi_process(self): - stack_mode = False - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - func = Comparator(mode_config).compare_ops - generate_dump_json(base_dir) - input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} - lock = multiprocessing.Manager().RLock() - result = _handle_multi_process(func, input_param, result_df, lock) - self.assertTrue(result.equals(o_result)) + if os.path.exists(pt_dir): + shutil.rmtree(pt_dir) + if os.path.exists(base_dir3): + shutil.rmtree(base_dir3) def test_read_dump_data(self): - result = read_dump_data(result_df) + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + # normal + result = compare_real_data.read_dump_data(result_df) self.assertEqual(result, {'Functional.linear.0.forward.input.0': ['-1', '-1']}) + # index error with self.assertRaises(CompareException) as context: - result = read_dump_data(pd.DataFrame()) + result = compare_real_data.read_dump_data(pd.DataFrame()) self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) def test_save_cmp_result_success(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + comparison_result = ComparisonResult( cos_result=[0.99, 0.98], max_err_result=[0.01, 0.02], @@ -78,13 +121,18 @@ class TestUtilsMethods(unittest.TestCase): err_msgs=['', 'Error in comparison'] ) offset = 0 - updated_df = _save_cmp_result(offset, comparison_result, self.result_df, self.lock) + updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock) self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 0.99) self.assertEqual(updated_df.loc[1, CompareConst.COSINE], 0.98) self.assertEqual(updated_df.loc[1, CompareConst.ERROR_MESSAGE], 'Error in comparison') def test_save_cmp_result_index_error(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + comparison_result = ComparisonResult( cos_result=[0.99], max_err_result=[], @@ -95,39 +143,108 @@ class TestUtilsMethods(unittest.TestCase): err_msgs=[''] ) with self.assertRaises(CompareException) as context: - _save_cmp_result(0, comparison_result, self.result_df, self.lock) + compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock) self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - def test_check_accuracy(self): - max_abs_err = '' - - cos_1 = CompareConst.SHAPE_UNMATCH - result_1 = check_accuracy(cos_1, max_abs_err) - self.assertEqual(result_1, CompareConst.ACCURACY_CHECK_UNMATCH) - - cos_2 = CompareConst.NONE - result_2 = check_accuracy(cos_2, max_abs_err) - self.assertEqual(result_2, CompareConst.NONE) + def test_compare_by_op_bench_normal(self): + npu_op_name = 'Functional.linear.0.forward.input.0' + bench_op_name = 'Functional.linear.0.forward.input.0' + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + pt_name = '-1' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} + input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'No bench data matched.']) + + pt_name = 'Functional.linear.0.forward.input.0.pt' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} + input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found.']) + + generate_pt(base_dir) + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, '']) + + def test_compare_by_op_bench_na(self): + npu_op_name = 'Functional.linear.0.forward.input.0' + bench_op_name = 'N/A' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [-1, -1]} + input_param = {} + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'No bench data matched.']) + + def test_compare_ops(self): + generate_dump_json(base_dir3) + generate_stack_json(base_dir3) + generate_pt(pt_dir) + dump_path = os.path.join(base_dir3, 'dump.json') + stack_path = os.path.join(base_dir3, 'stack.json') + input_param = {'npu_json_path': dump_path, 'bench_json_path': dump_path, 'stack_json_path': stack_path, + 'is_print_compare_log': True, 'npu_dump_data_dir': pt_dir, 'bench_dump_data_dir': pt_dir} + dump_path_dict = {'Functional.linear.0.forward.input.0': ['Functional.linear.0.forward.input.0.pt', + 'Functional.linear.0.forward.input.0.pt']} + result_df = pd.DataFrame({ + 'NPU Name': ['Functional.linear.0.forward.input.0'], + 'Bench Name': ['Functional.linear.0.forward.input.0'] + }) + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + updated_df = compare_real_data.compare_ops(idx=0, dump_path_dict=dump_path_dict, result_df=result_df, + lock=self.lock, input_param=input_param) + + self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 1.0) + self.assertEqual(updated_df.loc[0, CompareConst.MAX_ABS_ERR], 0) + + def test_do_multi_process(self): + data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', [2, 2], [2, 2], + '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', ['-1', '-1']]] + o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', [2, 2], [2, 2], + 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', ['-1', '-1']]] + columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] + result_df = pd.DataFrame(data, columns=columns) + o_result = pd.DataFrame(o_data, columns=columns) + generate_dump_json(base_dir) + input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} - cos_3 = 'N/A' - result_3 = check_accuracy(cos_3, max_abs_err) - self.assertEqual(result_3, CompareConst.ACCURACY_CHECK_NO) + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) - cos_4 = '' - result_4 = check_accuracy(cos_4, max_abs_err) - self.assertEqual(result_4, CompareConst.NONE) - - cos_5 = 0.95 - max_abs_err = 0.002 - result_5 = check_accuracy(cos_5, max_abs_err) - self.assertEqual(result_5, CompareConst.ACCURACY_CHECK_NO) + result = compare_real_data.do_multi_process(input_param, result_df) + self.assertTrue(result.equals(o_result)) - cos_6 = 0.85 - max_abs_err = 2 - result_6 = check_accuracy(cos_6, max_abs_err) - self.assertEqual(result_6, CompareConst.ACCURACY_CHECK_NO) + def test_handle_multi_process(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) - cos_7 = 0.95 - max_abs_err = 0.001 - result_7 = check_accuracy(cos_7, max_abs_err) - self.assertEqual(result_7, CompareConst.ACCURACY_CHECK_YES) + func = compare_real_data.compare_ops + generate_dump_json(base_dir) + input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} + lock = multiprocessing.Manager().RLock() + result = compare_real_data._handle_multi_process(func, input_param, result_df, lock) + self.assertTrue(result.equals(o_result)) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 6f7377894002e60add41dc7b2d3c1d3d68391e0b..b0e49fd545393bac6fbc3d7af554c91e24119e69 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -1,20 +1,10 @@ # coding=utf-8 -import json -import os + import random -import shutil -import tempfile import unittest from unittest.mock import patch -import numpy as np -import pandas as pd -import torch -import yaml - -from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.mindspore.compare.ms_compare import MappingConfig, MSComparator, check_cross_framework +from msprobe.mindspore.compare.ms_compare import check_cross_framework from msprobe.core.common.const import Const npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', @@ -190,169 +180,9 @@ def gen_data(is_ms=True): } -def gen_api_mapping_test_data(need_user_mapping=False): - result_npu = json_data_template.copy() - result_bench = json_data_template.copy() - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - ms_comparator = MSComparator(mode_config, mapping_config) - - api_mapping = ms_comparator.load_internal_api() - ms_api_list = np.random.choice(list(api_mapping.keys()), size=5, replace=False).astype(str).tolist() - ms_api_data = {} - pt_api_data = {} - user_mapping = [] - for api in ms_api_list: - call_num = random.randint(1, 10) - direction = random.choice(['forward', 'backward']) - data_name_ms = api + '.' + str(call_num) + '.' + direction - data_name_pt = api_mapping.get(api) + '.' + str(call_num) + '.' + direction - input_num = random.randint(1, 5) - output_num = random.randint(1, 5) - ms_data = {'input_args': [gen_data(True) for _ in range(input_num)], - 'output': [gen_data(True) for _ in range(output_num)]} - pt_data = {'input_args': [gen_data(False) for _ in range(input_num)], - 'output': [gen_data(False) for _ in range(output_num)]} - ms_api_data[data_name_ms] = ms_data - pt_api_data[data_name_pt] = pt_data - if need_user_mapping: - compare_num_input = random.randint(1, input_num) - compare_num_output = random.randint(1, output_num) - user_mapping_item = {'ms_api': api, - 'pt_api': api_mapping.get(api), - 'ms_args': sorted(np.random.choice(list(range(input_num)), size=compare_num_input, - replace=False).astype(int).tolist()), - 'pt_args': sorted(np.random.choice(list(range(input_num)), size=compare_num_input, - replace=False).astype(int).tolist()), - 'ms_output': sorted(np.random.choice(list(range(output_num)), size=compare_num_output, - replace=False).astype(int).tolist()), - 'pt_output': sorted(np.random.choice(list(range(output_num)), size=compare_num_output, - replace=False).astype(int).tolist())} - user_mapping.append(user_mapping_item) - ms_api_key_list = list(ms_api_data.keys()) - random.shuffle(ms_api_key_list) - result_npu['data'] = {k: ms_api_data.get(k) for k in ms_api_key_list} - pt_api_key_list = list(pt_api_data.keys()) - random.shuffle(pt_api_key_list) - result_bench['data'] = {k: pt_api_data.get(k) for k in pt_api_key_list} - return result_npu, result_bench, user_mapping - - class TestUtilsMethods(unittest.TestCase): - def test_check_op_ms(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - result = ms_comparator.check_op(npu_dict, bench_dict) - self.assertTrue(result) - - def test_data_mapping(self): - stack_json_data = {} - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(data_mapping=data_mapping) - ms_comparator = MSComparator(mode_config, mapping_config) - - npu_ops_all = ms_comparator.merge_data(npu_json_data, stack_json_data) - npu_ops_all_correct = { - 'Functional.flash_attention_score.4.forward.input.0': { - 'struct': ('BFloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625], - 'data_name': None, - 'stack_info': [None] - }, - 'Functional.flash_attention_score.4.forward.output.0': { - 'struct': ('BFloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625], - 'data_name': None, - 'stack_info': [None] - } - } - self.assertDictEqual(npu_ops_all, npu_ops_all_correct) - - bench_ops_all = ms_comparator.merge_data(bench_json_data, stack_json_data) - bench_ops_all_correct = { - 'NPU.npu_fusion_attention.4.forward.input.0': { - 'struct': ('torch.bfloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.553794860839844e-05, 2320.0], - 'data_name': None, - 'stack_info': [None] - }, - 'NPU.npu_fusion_attention.4.forward.output.0': { - 'struct': ('torch.bfloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.553794860839844e-05, 2320.0], - 'data_name': None, - 'stack_info': [None] - } - } - self.assertDictEqual(bench_ops_all, bench_ops_all_correct) - - result = ms_comparator.get_accuracy(npu_ops_all, bench_ops_all) - result_correct = [['Functional.flash_attention_score.4.forward.input.0', - 'NPU.npu_fusion_attention.4.forward.input.0', - 'BFloat16', 'torch.bfloat16', [4096, 1, 2048], [4096, 1, 2048], 0.0, 0.0, - 3.512832336127758e-08, -3.620849609375, '0.0%', '0.0%', '0.07714076816099476%', - '0.1560711038523707%', 4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625, - 4.1875, -4.4375, -4.553794860839844e-05, 2320.0, '', '', None], - ['Functional.flash_attention_score.4.forward.output.0', - 'NPU.npu_fusion_attention.4.forward.output.0', - 'BFloat16', 'torch.bfloat16', [4096, 1, 2048], [4096, 1, 2048], 0.0, 0.0, - 3.512832336127758e-08, -3.620849609375, '0.0%', '0.0%', '0.07714076816099476%', - '0.1560711038523707%', 4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625, - 4.1875, -4.4375, -4.553794860839844e-05, 2320.0, '', '', None] - ] - self.assertListEqual(result, result_correct) - - def test_dm_tensor_task(self): - self.compare_process_custom(dump_mode=Const.ALL) - - def compare_process_custom(self, dump_mode): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - - with open(npu_dump_path, 'w') as n_d_f: - json.dump(npu_json_data, n_d_f) - with open(bench_dump_path, 'w') as b_d_f: - json.dump(bench_json_data, b_d_f) - with open(npu_stack_path, 'w') as n_s_f: - json.dump({}, n_s_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process_custom((npu_dump_path, bench_dump_path, npu_stack_path)) - self.assertListEqual(result_df.values.tolist(), []) - finally: - shutil.rmtree(data_path) - - @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.utils.detect_framework_by_dump_json') def test_check_cross_framework_valid_pytorch(self, mock_detect_framework): mock_detect_framework.return_value = Const.PT_FRAMEWORK @@ -360,203 +190,10 @@ class TestUtilsMethods(unittest.TestCase): self.assertTrue(result) - @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.utils.detect_framework_by_dump_json') def test_check_cross_framework_invalid_framework(self, mock_detect_framework): mock_detect_framework.return_value = Const.MS_FRAMEWORK result = check_cross_framework("dummy_path") self.assertFalse(result) - - def test_comapre_process(self): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - - npu_data, bench_data, _ = gen_api_mapping_test_data() - with open(npu_dump_path, 'w', encoding='utf8') as n_d_f: - json.dump(npu_data, n_d_f) - with open(bench_dump_path, 'w', encoding='utf8') as b_d_f: - json.dump(bench_data, b_d_f) - with open(npu_stack_path, 'w', encoding='utf8') as n_s_f: - json.dump({}, n_s_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=True) - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process((npu_dump_path, bench_dump_path, npu_stack_path)) - self.assertTrue((result_df['Bench Name'] != 'N/A').all()) - finally: - shutil.rmtree(data_path) - - def test_compare_process_with_customize_api_mapping(self): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - user_mapping_path = os.path.join(data_path, 'user_mapping.yaml') - - npu_data, bench_data, user_mapping = gen_api_mapping_test_data(True) - with open(npu_dump_path, 'w', encoding='utf8') as n_d_f: - json.dump(npu_data, n_d_f) - with open(bench_dump_path, 'w', encoding='utf8') as b_d_f: - json.dump(bench_data, b_d_f) - with open(npu_stack_path, 'w', encoding='utf8') as n_s_f: - json.dump({}, n_s_f) - with open(user_mapping_path, 'w', encoding='utf8') as u_m_f: - yaml.safe_dump(user_mapping, u_m_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=user_mapping_path) - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process((npu_dump_path, bench_dump_path, npu_stack_path)) - - user_mapping_dict = {} - for i in user_mapping: - user_mapping_dict[i.get('ms_api')] = {'input': i.get('ms_args'), 'output': i.get('ms_output')} - match_set = set() - for key in npu_data.get('data').keys(): - matched_dict = user_mapping_dict.get(key.rsplit('.', 2)[0]) - match_set.update({key + '.input.' + str(i) for i in matched_dict.get('input')}) - match_set.update({key + '.output.' + str(i) for i in matched_dict.get('output')}) - - self.assertTrue((result_df.loc[result_df['NPU Name'].isin(match_set), 'Bench Name'] != 'N/A').all()) - self.assertTrue((result_df.loc[~result_df['NPU Name'].isin(match_set), 'Bench Name'] == 'N/A').all()) - finally: - shutil.rmtree(data_path) - - def test_load_internal_api(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - api_dict = ms_comparator.load_internal_api() - self.assertEqual(api_dict['Functional.abs'], 'Torch.abs') - - def test_process_cell_mapping(self): - self.base_test_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) - self.input_dir = os.path.join(self.base_test_dir, 'resources') - cell_mapping_path = os.path.join(self.input_dir, 'common', 'cell_mapping.yaml') - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(cell_mapping=cell_mapping_path) - - ms_comparator = MSComparator(mode_config, mapping_config) - npu_op_name = ms_comparator.process_cell_mapping(npu_cell_dict.get('op_name')[0]) - self.assertEqual(npu_op_name, 'Module.fc1.Linear.forward.0.input.0') - - def test_read_npy_data(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.pt') - tensor = torch.Tensor([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - torch.save(tensor, self.temp_file.name) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=True) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.npy') - tensor = np.array([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - np.save(self.temp_file.name, tensor) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=False) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() - - def test_process_internal_api_mapping(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=1) - - ms_comparator = MSComparator(mode_config, mapping_config) - - npu_op_name = "Mint.addcmul.0.forward.input.0" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Torch.addcmul.0.forward.input.0") - - npu_op_name = "MintFunctional.addcmul.0.forward.input.0" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Functional.addcmul.0.forward.input.0") - - npu_op_name = "Functional.abs" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Torch.abs") - - def test_get_api_name(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - - api_list = ["Functional", "absolute", "0", "forward", "input", "0"] - result = ms_comparator.get_api_name(api_list) - self.assertEqual(result, "Functional.absolute") - - api_list = ["Mint"] - with self.assertRaises(CompareException): - ms_comparator.get_api_name(api_list) - - def test_process_data_name(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - ms_comparator = MSComparator(mode_config, mapping_config) - - data = pd.DataFrame({ - 'data_name_x': ['A', 'B', 'C'], - 'data_name_y': ['X', 'Y', 'Z'] - }) - - result = ms_comparator.process_data_name(data.copy()) - - expected = pd.DataFrame({ - 'data_name_x': [['A', 'X'], ['B', 'Y'], ['C', 'Z']], - 'data_name_y': ['X', 'Y', 'Z'] - }) - - pd.testing.assert_frame_equal(result, expected) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7fb5e38fb82b309caf3ab2a1b621655d7babc86 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py @@ -0,0 +1,24 @@ +import unittest +from unittest.mock import patch + +import numpy as np + +from msprobe.core.common.file_utils import FileCheckConst +from msprobe.mindspore.compare.utils import read_npy_data + + +class TestReadNpyData(unittest.TestCase): + + @patch('msprobe.mindspore.compare.utils.load_npy') + @patch('msprobe.mindspore.compare.utils.FileChecker') + @patch('os.path.join', return_value='/fake/path/to/file.npy') + def test_read_real_data_ms(self, mock_os, mock_file_checker, mock_load_npy): + mock_file_checker.return_value.common_check.return_value = '/fake/path/to/file.npy' + + mock_load_npy.return_value = np.array([1.0, 2.0, 3.0]) + + result = read_npy_data('/fake/dir', 'file_name.npy') + + mock_file_checker.assert_called_once_with('/fake/path/to/file.npy', FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.NUMPY_SUFFIX, False) + mock_load_npy.assert_called_once_with('/fake/path/to/file.npy') + self.assertTrue(np.array_equal(result, np.array([1.0, 2.0, 3.0]))) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py deleted file mode 100644 index ac28e994e9c8e77f8ae675fec3322eaf64a64321..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ /dev/null @@ -1,20 +0,0 @@ -# coding=utf-8 -import unittest -from msprobe.pytorch.compare import match - - -class TestMatch(unittest.TestCase): - def test_graph_mapping(self): - op1 = "Aten_convolution_1_forward_0.input.0" - op2 = "Torch_conv2d_0_forward_0.input.0" - op3 = "Torch_batch_norm_0_forward_0.input.0" - op4 = "Aten_convolution.default_1_forward_0.input.0" - op5 = "Aten_foo_1_forward_0.input.0" - self.assertTrue(match.graph_mapping.match(op1, op2)) - self.assertTrue(match.graph_mapping.match(op2, op1)) - self.assertTrue(match.graph_mapping.match(op4, op2)) - self.assertTrue(match.graph_mapping.match(op2, op4)) - self.assertFalse(match.graph_mapping.match(op1, op3)) - self.assertFalse(match.graph_mapping.match(op3, op1)) - self.assertFalse(match.graph_mapping.match(op5, op2)) - self.assertFalse(match.graph_mapping.match(op2, op5)) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py index b079e646c4a8f4098bb233e3e6259ef3ebea9c94..e4c8b722b182b8c0a4e82ba1b0eeb1a6ed847ee2 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py @@ -3,16 +3,12 @@ import os import shutil import unittest -import numpy as np import torch -from msprobe.core.common.const import Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.pytorch.compare.pt_compare import compare from msprobe.test.core_ut.compare.test_acc_compare import generate_dump_json, generate_stack_json - base_dir1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare1') base_dir2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare2') @@ -40,36 +36,6 @@ class TestUtilsMethods(unittest.TestCase): if os.path.exists(base_dir2): shutil.rmtree(base_dir2) - def test_read_npy_data_bf16(self): - generate_bf16_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - result = pt_comparator.read_npy_data(base_dir1, 'bf16.pt') - - target_result = torch.tensor([1, 2, 3, 4], dtype=torch.float32).numpy() - self.assertTrue(np.array_equal(result, target_result)) - - def test_read_npy_data_dict(self): - generate_dict_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - - with self.assertRaises(CompareException) as context: - result = pt_comparator.read_npy_data(base_dir1, 'dict.pt') - self.assertEqual(context.exception.code, CompareException.DETACH_ERROR) - def test_compare(self): generate_dump_json(base_dir2) generate_stack_json(base_dir2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d967c5e314cc872da7972a37ac72fa0ffdb04a23 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py @@ -0,0 +1,34 @@ +import os +import shutil +import threading +import unittest + +import numpy as np + +from msprobe.pytorch.compare.utils import read_pt_data +from msprobe.test.core_ut.compare.test_acc_compare import generate_pt + + +base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare_utils_data') +pt_dir = os.path.join(base_dir, f'dump_data_dir') + + +class TestReadPtData(unittest.TestCase): + + def setUp(self): + os.makedirs(base_dir, mode=0o750, exist_ok=True) + os.makedirs(pt_dir, mode=0o750, exist_ok=True) + + self.lock = threading.Lock() + + def tearDown(self): + if os.path.exists(pt_dir): + shutil.rmtree(pt_dir) + if os.path.exists(base_dir): + shutil.rmtree(base_dir) + + def test_read_pt_data(self): + generate_pt(pt_dir) + result = read_pt_data(pt_dir, 'Functional.linear.0.forward.input.0.pt') + expected = np.array([1.0, 2.0, 3.0, 4.0]) + self.assertTrue(np.array_equal(result, expected)) diff --git a/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py b/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py index 751006f3e527726ce049e054ab9cbb2ad87de064..ef39b3fcf8424f95d7c859d5c49b810dca1c105c 100644 --- a/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py +++ b/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,12 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import re -from msprobe.core.compare.acc_compare import read_op, merge_tensor, get_accuracy + +from msprobe.core.compare.acc_compare import ModeConfig +from msprobe.core.compare.multiprocessing_compute import CompareRealData +from msprobe.core.compare.utils import read_op, merge_tensor, get_accuracy, make_result_table from msprobe.core.common.utils import set_dump_path, get_dump_mode from msprobe.visualization.utils import GraphConst from msprobe.core.common.const import Const -from msprobe.core.compare.acc_compare import ModeConfig + # 用于将节点名字解析成对应的NodeOp的规则 op_patterns = [ @@ -53,13 +57,11 @@ def run_real_data(dump_path_param, csv_path, framework, is_cross_frame=False): mode_config = ModeConfig(stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.ALL) if framework == Const.PT_FRAMEWORK: - from msprobe.pytorch.compare.pt_compare import PTComparator - return PTComparator(mode_config).do_multi_process(dump_path_param, csv_path) + from msprobe.pytorch.compare.pt_compare import read_real_data + return CompareRealData(read_real_data, mode_config, is_cross_frame).do_multi_process(dump_path_param, csv_path) else: - from msprobe.mindspore.compare.ms_compare import MSComparator, MappingConfig - ms_comparator = MSComparator(mode_config, MappingConfig()) - ms_comparator.cross_frame = is_cross_frame - return ms_comparator.do_multi_process(dump_path_param, csv_path) + from msprobe.mindspore.compare.ms_compare import read_real_data + return CompareRealData(read_real_data, mode_config, is_cross_frame).do_multi_process(dump_path_param, csv_path) def get_input_output(node_data, node_id): @@ -226,3 +228,12 @@ def _format_data(data_dict): if all_null: data_dict.clear() data_dict[GraphConst.VALUE] = GraphConst.NULL + + +def get_csv_df(stack_mode, csv_data, compare_mode): + """ + 调用acc接口写入csv + """ + + dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode) + return make_result_table(csv_data, dump_mode, stack_mode) diff --git a/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py b/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py index f2ba394bcd3c5ae96ce2eadb62cafe50e488a2d6..0f98a79cfc691b6ff1444f73e48fb93732ccf038 100644 --- a/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py +++ b/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +14,8 @@ # limitations under the License. import re -from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data -from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file, get_csv_df +from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data, get_csv_df +from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file from msprobe.visualization.graph.graph import Graph, NodeOp from msprobe.visualization.compare.mode_adapter import ModeAdapter from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/visualization/utils.py b/debug/accuracy_tools/msprobe/visualization/utils.py index 5f428697bdef1bc1de72a3eb6da1c4c5761eae2a..f20ed75765612de4fe4eaf569f57e3b8384b57a0 100644 --- a/debug/accuracy_tools/msprobe/visualization/utils.py +++ b/debug/accuracy_tools/msprobe/visualization/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,6 @@ import re import json from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import CompareConst, Const -from msprobe.core.compare.acc_compare import Comparator, ModeConfig def load_json_file(file_path): @@ -42,15 +41,6 @@ def load_data_json_file(file_path): return load_json_file(file_path).get(GraphConst.DATA_KEY, {}) -def get_csv_df(stack_mode, csv_data, compare_mode): - """ - 调用acc接口写入csv - """ - dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode) - mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=dump_mode) - return Comparator(mode_config).make_result_table(csv_data) - - def str2float(percentage_str): """ 百分比字符串转换转换为浮点型